"""The turn runner: one employee turn, with a scripted model and a fake sandbox. What these tests defend: * the write → run → report loop really works: a file is written inside the sandbox, a command runs there, and the report ends the turn * the sandbox is the boundary: a path escape, a ``.git`` path and a git command are refused *inside the tool*, with a message the model can read * a turn is bounded: the tool-call cap and the turn timeout both stop it * errors are classified, so a rate limit is retried later and an ungranted model is not * every model call writes usage events — including the ones a manager chat makes, which agybot forgot * no secret from the settings ever reaches a prompt """ from __future__ import annotations from types import SimpleNamespace import pytest import pytest_asyncio from sqlalchemy import select from app.application.engine.runner import ( TurnRequest, TurnRunner, classify_error, ) from app.application.ports import CommandResult from app.core.config import settings from app.core.errors import ProviderUnavailableError from app.domain.ai_control.models import AiModel from app.domain.base import new_id from app.domain.workforce.engine import ErrorKind from app.domain.workforce.models import RoleKey from app.domain.workforce.pipeline import Mode, Stage from app.infrastructure.db.models import ( AiProviderRow, EmployeeRow, OrganizationRow, RunRow, TaskRow, UsageEventRow, UserRow, ) from app.infrastructure.repositories.ai_control import ( AiModelRepository, ModelCredentialRepository, RateCardRepository, ) from app.infrastructure.repositories.audit import UsageLedgerRepository from app.infrastructure.repositories.engine import ( EngineTurnRepository, PipelineStateRepository, ) from app.infrastructure.repositories.workforce import ( EmployeeRepository, RunRepository, RunStepRepository, TaskRepository, ToolCallRepository, ) from tests.engine_fakes import ( FakeGrants, FakeSandbox, RecordingMemory, ScriptedModel, ScriptedReply, ) DEVELOPER = { "role": RoleKey.DEVELOPER, "mode": Mode.WORK, "stage": Stage.PENDING_IMPLEMENTATION, } # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @pytest_asyncio.fixture async def _db(session_factory, seeded): """A tenant, a task, a run, a developer employee and a granted model.""" async with session_factory() as session: user = UserRow(email=f"runner-{new_id()[:8]}@example.com", password_hash="x") session.add(user) await session.flush() org = OrganizationRow(name="سازمان اجرا", slug=f"run-{user.id[:6]}", owner_user_id=user.id) session.add(org) await session.flush() # The seeded provider row: the model must hang off a real one. provider_id = (await session.execute(select(AiProviderRow.id).limit(1))).scalar_one() model = await AiModelRepository(session).create( AiModel( provider_id=provider_id, internal_name="scripted-model", display_name_for_admin="مدل فیلم‌نویسی‌شده", capabilities=["text", "code"], is_active=True, ) ) task = TaskRow(organization_id=org.id, title="ساخت صفحهٔ ورود") session.add(task) await session.flush() run = RunRow(task_id=task.id) session.add(run) employee = EmployeeRow(organization_id=org.id, name="سارا", role_key="developer") session.add(employee) await session.commit() yield SimpleNamespace( session_factory=session_factory, organization_id=org.id, task_id=task.id, run_id=run.id, employee_id=employee.id, model_id=model.id, ) def _runner(session, *, model, sandbox, grants, feed=None) -> TurnRunner: return TurnRunner( provider=model, models=AiModelRepository(session), credentials=ModelCredentialRepository(session), rate_cards=RateCardRepository(session), ledger=UsageLedgerRepository(session), grants=grants, sandbox=sandbox, memory=RecordingMemory(context="این سازمان تست‌ها را با pytest اجرا می‌کند."), turns=EngineTurnRepository(session), run_steps=RunStepRepository(session), tool_calls=ToolCallRepository(session), runs=RunRepository(session), states=PipelineStateRepository(session), feed=feed, ) async def _request(db, *, sandbox_root="/workspace", session_id="sess-1", **kwargs) -> TurnRequest: async with db.session_factory() as session: task = await TaskRepository(session).get(db.task_id) employee = await EmployeeRepository(session).get(db.employee_id) return TurnRequest( organization_id=db.organization_id, task=task, run_id=db.run_id, employee=employee, model_id=db.model_id, sandbox_root=sandbox_root, session_external_id=session_id, **kwargs, ) # --------------------------------------------------------------------------- # The loop # --------------------------------------------------------------------------- async def test_write_then_run_then_report_ends_the_turn(_db, session_factory): """The whole agent loop: one file, one command, one report.""" scripted = ScriptedModel( [ ScriptedReply( content='[[WRITE_FILE path="app/login.py"]]print("login")[[/WRITE_FILE]]' ), ScriptedReply(content='[[RUN cmd="pytest -q" timeout="30"/]]'), ScriptedReply(content="[[WORK_REPORT]]صفحهٔ ورود نوشده و تست‌ها سبز شد.[[/WORK_REPORT]]"), ] ) sandbox = FakeSandbox( commands={"pytest": [CommandResult(exit_code=0, stdout="3 passed")]}, ) async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id]) ) request = await _request(_db, **DEVELOPER) result = await runner.run_turn(request) await session.commit() assert result.ok is True assert result.model_calls == 3 assert result.tool_calls == 2 assert result.tags[0].name.value == "WORK_REPORT" # The file really was written inside the sandbox, rooted at the worktree. assert sandbox.writes == [{"path": "/workspace/app/login.py", "content": 'print("login")'}] assert [call["command"] for call in sandbox.executed] == ["pytest -q"] assert scripted.calls == 3 async def test_tool_results_feed_back_into_the_next_prompt(_db, session_factory): """The model must see what its own command printed.""" scripted = ScriptedModel( [ ScriptedReply(content='[[RUN cmd="pytest -q"/]]'), ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"), ] ) sandbox = FakeSandbox( commands={"pytest": [CommandResult(exit_code=1, stdout="1 failed", stderr="boom")]} ) async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id]) ) result = await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() assert result.ok is True last_prompt = scripted.seen[-1] feedback = "\n".join(m.content for m in last_prompt["messages"]) assert "1 failed" in feedback and "boom" in feedback async def test_each_tool_call_writes_a_tool_calls_row(_db, session_factory): """The approvals UI finally sees real executions, not intentions.""" scripted = ScriptedModel( [ ScriptedReply(content='[[READ_FILE path="app/x.py"/]]'), ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"), ] ) sandbox = FakeSandbox(files={"/workspace/app/x.py": "print(1)"}) async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id]) ) await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() calls = await ToolCallRepository(session).list_for_run(_db.run_id) assert len(calls) == 1 assert calls[0].tool_key == "sandbox.read_file" assert calls[0].status.value == "succeeded" assert calls[0].result_json["ok"] is True async def test_a_failed_tool_is_recorded_as_failed(_db, session_factory): scripted = ScriptedModel( [ ScriptedReply(content='[[READ_FILE path="app/missing.py"/]]'), ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"), ] ) sandbox = FakeSandbox() async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id]) ) result = await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() calls = await ToolCallRepository(session).list_for_run(_db.run_id) assert result.ok is True assert calls[0].status.value == "failed" async def test_the_turn_leaves_an_engine_turns_row(_db, session_factory): scripted = ScriptedModel([ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]")]) async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=FakeSandbox(), grants=FakeGrants([_db.model_id]) ) result = await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() turns = await EngineTurnRepository(session).list_for_task(_db.task_id) assert result.turn_id assert len(turns) == 1 assert turns[0].ok is True assert turns[0].role is RoleKey.DEVELOPER assert turns[0].mode is Mode.WORK assert turns[0].stage is Stage.PENDING_IMPLEMENTATION assert turns[0].prompt_chars > 0 assert "[[WORK_REPORT]]" in turns[0].reply_excerpt assert "تمام" in turns[0].reply_excerpt assert turns[0].tags == {"WORK_REPORT": 1} assert turns[0].finished_at is not None # --------------------------------------------------------------------------- # The sandbox boundary # --------------------------------------------------------------------------- @pytest.mark.parametrize( "path", ["../../etc/passwd", "/etc/passwd", ".git/config", "app/../../secret"], ) async def test_a_path_escape_is_refused(_db, session_factory, path): scripted = ScriptedModel( [ ScriptedReply(content=f'[[READ_FILE path="{path}"/]]'), ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"), ] ) sandbox = FakeSandbox(files={"/etc/passwd": "root"}) async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id]) ) result = await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() # The refusal is fed back to the model, and nothing was read. assert result.ok is True assert sandbox.reads == [] assert "مجاز نیست" in scripted.all_text() @pytest.mark.parametrize( "command", ["git push origin dev", "git remote add x y", "git config user.email a@b.c"] ) async def test_the_model_cannot_run_git(_db, session_factory, command): scripted = ScriptedModel( [ ScriptedReply(content=f'[[RUN cmd="{command}"/]]'), ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"), ] ) sandbox = FakeSandbox() async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id]) ) result = await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() assert result.ok is True assert sandbox.executed == [] assert "پلتفرم" in scripted.all_text() async def test_edit_file_applies_a_search_replace_block(_db, session_factory): scripted = ScriptedModel( [ ScriptedReply( content=( "[[EDIT_FILE path=\"app/x.py\"]]<<>>[[/EDIT_FILE]]" ) ), ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"), ] ) sandbox = FakeSandbox(files={"/workspace/app/x.py": "before\nold line\nafter"}) async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id]) ) result = await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() assert result.ok is True assert sandbox.files["/workspace/app/x.py"] == "before\nnew line\nafter" async def test_edit_file_fails_when_the_old_text_is_not_unique(_db, session_factory): scripted = ScriptedModel( [ ScriptedReply( content=( "[[EDIT_FILE path=\"app/x.py\"]]<<>>[[/EDIT_FILE]]" ) ), ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"), ] ) sandbox = FakeSandbox(files={"/workspace/app/x.py": "line\nline"}) async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id]) ) result = await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() assert result.ok is True assert sandbox.files["/workspace/app/x.py"] == "line\nline" assert "دقیقاً یک بار" in scripted.all_text() # --------------------------------------------------------------------------- # Bounds: the tool cap and the timeout # --------------------------------------------------------------------------- async def test_the_tool_call_cap_stops_a_runaway_turn(_db, session_factory, monkeypatch): monkeypatch.setattr(settings, "engine_max_tool_calls_per_turn", 2) scripted = ScriptedModel([ScriptedReply(content='[[LIST_DIR path="."/]]') for _ in range(5)]) sandbox = FakeSandbox() async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id]) ) result = await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() assert result.ok is False assert result.incomplete is True assert result.tool_calls == 2 assert result.error_kind == "protocol" async def test_the_turn_timeout_stops_a_slow_loop(_db, session_factory, monkeypatch): monkeypatch.setattr(settings, "engine_turn_timeout_seconds", 0) scripted = ScriptedModel([ScriptedReply(content='[[LIST_DIR path="."/]]')] * 3) async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=FakeSandbox(), grants=FakeGrants([_db.model_id]) ) result = await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() assert result.ok is False assert result.error_kind == "timeout" async def test_a_cancelled_run_stops_between_steps(_db, session_factory): scripted = ScriptedModel( [ ScriptedReply(content='[[LIST_DIR path="."/]]'), ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"), ] ) async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=FakeSandbox(), grants=FakeGrants([_db.model_id]) ) request = await _request(_db, **DEVELOPER) # The hook cancels after the first model call has been made. calls = {"n": 0} async def should_cancel() -> bool: calls["n"] += 1 return calls["n"] > 1 request.should_cancel = should_cancel result = await runner.run_turn(request) await session.commit() assert result.ok is False assert result.error_kind == "cancelled" # --------------------------------------------------------------------------- # Errors: transient versus fatal # --------------------------------------------------------------------------- async def test_a_rate_limit_is_transient_and_reported(_db, session_factory): scripted = ScriptedModel([ScriptedReply(raises=ProviderUnavailableError("429 rate limited"))]) async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=FakeSandbox(), grants=FakeGrants([_db.model_id]) ) result = await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() assert result.ok is False assert result.error_kind == "rate_limit" assert result.retryable is True async def test_an_ungranted_model_is_refused_before_any_call(_db, session_factory): scripted = ScriptedModel([ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]")]) grants = FakeGrants(["some-other-model"]) async with session_factory() as session: runner = _runner(session, model=scripted, sandbox=FakeSandbox(), grants=grants) result = await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() assert result.ok is False assert result.error_kind == "auth" assert result.retryable is False # The model was never called: an ungranted employee spends nothing. assert scripted.calls == 0 assert grants.checks == [_db.model_id] def test_error_classification(): assert classify_error(ProviderUnavailableError("429 rate limit")) is ErrorKind.RATE_LIMIT assert classify_error(ProviderUnavailableError("auth failed")) is ErrorKind.AUTH assert classify_error(ProviderUnavailableError("connection refused")) is ErrorKind.NETWORK assert classify_error(TimeoutError()) is ErrorKind.TIMEOUT assert classify_error(ProviderUnavailableError("boom")) is ErrorKind.UNAVAILABLE # --------------------------------------------------------------------------- # Billing and secrets # --------------------------------------------------------------------------- async def test_every_model_call_writes_usage_events(_db, session_factory): """agybot forgot the manager chat; the engine does not.""" scripted = ScriptedModel( [ ScriptedReply(content='[[LIST_DIR path="."/]]', input_tokens=11, output_tokens=7), ScriptedReply( content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]", input_tokens=13, output_tokens=9 ), ] ) from sqlalchemy import select async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=FakeSandbox(), grants=FakeGrants([_db.model_id]) ) await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() rows = ( (await session.execute(select(UsageEventRow).where(UsageEventRow.run_id == _db.run_id))) .scalars() .all() ) assert len(rows) == 4 # two calls, input and output each assert sum(r.quantity for r in rows if r.unit == "model_input_tokens") == 24 assert sum(r.quantity for r in rows if r.unit == "model_output_tokens") == 16 async def test_no_secret_from_settings_reaches_the_model(_db, session_factory, monkeypatch): """The prompt is built by the engine; the settings stay on the server.""" monkeypatch.setattr(settings, "jwt_secret", "super-secret-jwt-value") monkeypatch.setattr(settings, "database_url", "postgresql+asyncpg://u:p@h/db") scripted = ScriptedModel( [ ScriptedReply(content='[[READ_FILE path="app/x.py"/]]'), ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"), ] ) sandbox = FakeSandbox(files={"/workspace/app/x.py": "print(1)"}) async with session_factory() as session: runner = _runner( session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id]) ) await runner.run_turn(await _request(_db, **DEVELOPER)) await session.commit() seen = scripted.all_text() assert "super-secret-jwt-value" not in seen assert "postgresql+asyncpg://u:p@h/db" not in seen # The model still got the rules and the role it plays. assert "قوانین پلتفرم" in seen assert "WORK_REPORT" in seen