feat(engine): the turn runner — prompt builder, sandbox tools, agent loop
Phase 2 of TODO-agent-2: one employee, one turn, real tools in the sandbox. - engine/prompt.py: the stable, cache-friendly prefix in the plan's order (rules → role protocol → toolkit → app profile → memory), a task brief capped at 1200 chars, role-specific stage context (spec for the developer, the diff only for the reviewer, test output for the tester) and a hard prompt cap that drops the oldest tool results first. - engine/tools.py: READ/WRITE/EDIT/LIST_DIR/SEARCH/RUN executed inside the sandbox through SandboxProviderPort, rooted at the worktree's sandbox path. Path escapes, .. and .git are refused; git push/remote/config are refused with "the platform does git"; RUN clamps its timeout (120s default, 600s max) and keeps the tail of its output; every execution writes a tool_calls row so the approvals UI shows real work. - engine/runner.py: the loop — model call, parse tags, execute tool tags, feed results back — until a terminal tag, the tool cap, the turn timeout or a cancellation. The employee's model is checked against ModelGrantPort before the first call (ungranted → error_kind=auth, no spend), errors are classified transient vs fatal, and every model call writes usage events through the one shared record_model_usage path (extracted from WorkforceService so manager chat counts too). Writes engine_turns, run_steps and tool_calls. - engine/events.py: RunFeed batches run.log (≤1 event / 500 ms / stream, ≤200 lines) and emits run.tool and task.gate through the generic envelope. - tests: engine_fakes.py (ScriptedModel, FakeSandbox, FakeGrants, RecordingMemory) plus 54 new tests in test_engine_runner, test_engine_prompt and test_engine_tools. Suite: 648 passed, ruff clean, mypy clean.
This commit is contained in:
mjalmoudy94
@@ -0,0 +1,551 @@
|
||||
"""The turn runner: one employee turn, with a scripted model and a fake sandbox.
|
||||
|
||||
What these tests defend:
|
||||
|
||||
* the write → run → report loop really works: a file is written inside the
|
||||
sandbox, a command runs there, and the report ends the turn
|
||||
* the sandbox is the boundary: a path escape, a ``.git`` path and a git
|
||||
command are refused *inside the tool*, with a message the model can read
|
||||
* a turn is bounded: the tool-call cap and the turn timeout both stop it
|
||||
* errors are classified, so a rate limit is retried later and an ungranted
|
||||
model is not
|
||||
* every model call writes usage events — including the ones a manager chat
|
||||
makes, which agybot forgot
|
||||
* no secret from the settings ever reaches a prompt
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.application.engine.runner import (
|
||||
TurnRequest,
|
||||
TurnRunner,
|
||||
classify_error,
|
||||
)
|
||||
from app.application.ports import CommandResult
|
||||
from app.core.config import settings
|
||||
from app.core.errors import ProviderUnavailableError
|
||||
from app.domain.ai_control.models import AiModel
|
||||
from app.domain.base import new_id
|
||||
from app.domain.workforce.engine import ErrorKind
|
||||
from app.domain.workforce.models import RoleKey
|
||||
from app.domain.workforce.pipeline import Mode, Stage
|
||||
from app.infrastructure.db.models import (
|
||||
AiProviderRow,
|
||||
EmployeeRow,
|
||||
OrganizationRow,
|
||||
RunRow,
|
||||
TaskRow,
|
||||
UsageEventRow,
|
||||
UserRow,
|
||||
)
|
||||
from app.infrastructure.repositories.ai_control import (
|
||||
AiModelRepository,
|
||||
ModelCredentialRepository,
|
||||
RateCardRepository,
|
||||
)
|
||||
from app.infrastructure.repositories.audit import UsageLedgerRepository
|
||||
from app.infrastructure.repositories.engine import (
|
||||
EngineTurnRepository,
|
||||
PipelineStateRepository,
|
||||
)
|
||||
from app.infrastructure.repositories.workforce import (
|
||||
EmployeeRepository,
|
||||
RunRepository,
|
||||
RunStepRepository,
|
||||
TaskRepository,
|
||||
ToolCallRepository,
|
||||
)
|
||||
from tests.engine_fakes import (
|
||||
FakeGrants,
|
||||
FakeSandbox,
|
||||
RecordingMemory,
|
||||
ScriptedModel,
|
||||
ScriptedReply,
|
||||
)
|
||||
|
||||
DEVELOPER = {
|
||||
"role": RoleKey.DEVELOPER,
|
||||
"mode": Mode.WORK,
|
||||
"stage": Stage.PENDING_IMPLEMENTATION,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def _db(session_factory, seeded):
|
||||
"""A tenant, a task, a run, a developer employee and a granted model."""
|
||||
async with session_factory() as session:
|
||||
user = UserRow(email=f"runner-{new_id()[:8]}@example.com", password_hash="x")
|
||||
session.add(user)
|
||||
await session.flush()
|
||||
org = OrganizationRow(name="سازمان اجرا", slug=f"run-{user.id[:6]}", owner_user_id=user.id)
|
||||
session.add(org)
|
||||
await session.flush()
|
||||
# The seeded provider row: the model must hang off a real one.
|
||||
provider_id = (await session.execute(select(AiProviderRow.id).limit(1))).scalar_one()
|
||||
|
||||
model = await AiModelRepository(session).create(
|
||||
AiModel(
|
||||
provider_id=provider_id,
|
||||
internal_name="scripted-model",
|
||||
display_name_for_admin="مدل فیلمنویسیشده",
|
||||
capabilities=["text", "code"],
|
||||
is_active=True,
|
||||
)
|
||||
)
|
||||
task = TaskRow(organization_id=org.id, title="ساخت صفحهٔ ورود")
|
||||
session.add(task)
|
||||
await session.flush()
|
||||
run = RunRow(task_id=task.id)
|
||||
session.add(run)
|
||||
employee = EmployeeRow(organization_id=org.id, name="سارا", role_key="developer")
|
||||
session.add(employee)
|
||||
await session.commit()
|
||||
yield SimpleNamespace(
|
||||
session_factory=session_factory,
|
||||
organization_id=org.id,
|
||||
task_id=task.id,
|
||||
run_id=run.id,
|
||||
employee_id=employee.id,
|
||||
model_id=model.id,
|
||||
)
|
||||
|
||||
|
||||
def _runner(session, *, model, sandbox, grants, feed=None) -> TurnRunner:
|
||||
return TurnRunner(
|
||||
provider=model,
|
||||
models=AiModelRepository(session),
|
||||
credentials=ModelCredentialRepository(session),
|
||||
rate_cards=RateCardRepository(session),
|
||||
ledger=UsageLedgerRepository(session),
|
||||
grants=grants,
|
||||
sandbox=sandbox,
|
||||
memory=RecordingMemory(context="این سازمان تستها را با pytest اجرا میکند."),
|
||||
turns=EngineTurnRepository(session),
|
||||
run_steps=RunStepRepository(session),
|
||||
tool_calls=ToolCallRepository(session),
|
||||
runs=RunRepository(session),
|
||||
states=PipelineStateRepository(session),
|
||||
feed=feed,
|
||||
)
|
||||
|
||||
|
||||
async def _request(db, *, sandbox_root="/workspace", session_id="sess-1", **kwargs) -> TurnRequest:
|
||||
async with db.session_factory() as session:
|
||||
task = await TaskRepository(session).get(db.task_id)
|
||||
employee = await EmployeeRepository(session).get(db.employee_id)
|
||||
return TurnRequest(
|
||||
organization_id=db.organization_id,
|
||||
task=task,
|
||||
run_id=db.run_id,
|
||||
employee=employee,
|
||||
model_id=db.model_id,
|
||||
sandbox_root=sandbox_root,
|
||||
session_external_id=session_id,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The loop
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_write_then_run_then_report_ends_the_turn(_db, session_factory):
|
||||
"""The whole agent loop: one file, one command, one report."""
|
||||
scripted = ScriptedModel(
|
||||
[
|
||||
ScriptedReply(
|
||||
content='[[WRITE_FILE path="app/login.py"]]print("login")[[/WRITE_FILE]]'
|
||||
),
|
||||
ScriptedReply(content='[[RUN cmd="pytest -q" timeout="30"/]]'),
|
||||
ScriptedReply(content="[[WORK_REPORT]]صفحهٔ ورود نوشده و تستها سبز شد.[[/WORK_REPORT]]"),
|
||||
]
|
||||
)
|
||||
sandbox = FakeSandbox(
|
||||
commands={"pytest": [CommandResult(exit_code=0, stdout="3 passed")]},
|
||||
)
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
request = await _request(_db, **DEVELOPER)
|
||||
result = await runner.run_turn(request)
|
||||
await session.commit()
|
||||
|
||||
assert result.ok is True
|
||||
assert result.model_calls == 3
|
||||
assert result.tool_calls == 2
|
||||
assert result.tags[0].name.value == "WORK_REPORT"
|
||||
# The file really was written inside the sandbox, rooted at the worktree.
|
||||
assert sandbox.writes == [{"path": "/workspace/app/login.py", "content": 'print("login")'}]
|
||||
assert [call["command"] for call in sandbox.executed] == ["pytest -q"]
|
||||
assert scripted.calls == 3
|
||||
|
||||
|
||||
async def test_tool_results_feed_back_into_the_next_prompt(_db, session_factory):
|
||||
"""The model must see what its own command printed."""
|
||||
scripted = ScriptedModel(
|
||||
[
|
||||
ScriptedReply(content='[[RUN cmd="pytest -q"/]]'),
|
||||
ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"),
|
||||
]
|
||||
)
|
||||
sandbox = FakeSandbox(
|
||||
commands={"pytest": [CommandResult(exit_code=1, stdout="1 failed", stderr="boom")]}
|
||||
)
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
result = await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
|
||||
assert result.ok is True
|
||||
last_prompt = scripted.seen[-1]
|
||||
feedback = "\n".join(m.content for m in last_prompt["messages"])
|
||||
assert "1 failed" in feedback and "boom" in feedback
|
||||
|
||||
|
||||
async def test_each_tool_call_writes_a_tool_calls_row(_db, session_factory):
|
||||
"""The approvals UI finally sees real executions, not intentions."""
|
||||
scripted = ScriptedModel(
|
||||
[
|
||||
ScriptedReply(content='[[READ_FILE path="app/x.py"/]]'),
|
||||
ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"),
|
||||
]
|
||||
)
|
||||
sandbox = FakeSandbox(files={"/workspace/app/x.py": "print(1)"})
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
calls = await ToolCallRepository(session).list_for_run(_db.run_id)
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0].tool_key == "sandbox.read_file"
|
||||
assert calls[0].status.value == "succeeded"
|
||||
assert calls[0].result_json["ok"] is True
|
||||
|
||||
|
||||
async def test_a_failed_tool_is_recorded_as_failed(_db, session_factory):
|
||||
scripted = ScriptedModel(
|
||||
[
|
||||
ScriptedReply(content='[[READ_FILE path="app/missing.py"/]]'),
|
||||
ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"),
|
||||
]
|
||||
)
|
||||
sandbox = FakeSandbox()
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
result = await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
calls = await ToolCallRepository(session).list_for_run(_db.run_id)
|
||||
|
||||
assert result.ok is True
|
||||
assert calls[0].status.value == "failed"
|
||||
|
||||
|
||||
async def test_the_turn_leaves_an_engine_turns_row(_db, session_factory):
|
||||
scripted = ScriptedModel([ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]")])
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=FakeSandbox(), grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
result = await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
turns = await EngineTurnRepository(session).list_for_task(_db.task_id)
|
||||
|
||||
assert result.turn_id
|
||||
assert len(turns) == 1
|
||||
assert turns[0].ok is True
|
||||
assert turns[0].role is RoleKey.DEVELOPER
|
||||
assert turns[0].mode is Mode.WORK
|
||||
assert turns[0].stage is Stage.PENDING_IMPLEMENTATION
|
||||
assert turns[0].prompt_chars > 0
|
||||
assert "[[WORK_REPORT]]" in turns[0].reply_excerpt
|
||||
assert "تمام" in turns[0].reply_excerpt
|
||||
assert turns[0].tags == {"WORK_REPORT": 1}
|
||||
assert turns[0].finished_at is not None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The sandbox boundary
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
["../../etc/passwd", "/etc/passwd", ".git/config", "app/../../secret"],
|
||||
)
|
||||
async def test_a_path_escape_is_refused(_db, session_factory, path):
|
||||
scripted = ScriptedModel(
|
||||
[
|
||||
ScriptedReply(content=f'[[READ_FILE path="{path}"/]]'),
|
||||
ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"),
|
||||
]
|
||||
)
|
||||
sandbox = FakeSandbox(files={"/etc/passwd": "root"})
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
result = await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
|
||||
# The refusal is fed back to the model, and nothing was read.
|
||||
assert result.ok is True
|
||||
assert sandbox.reads == []
|
||||
assert "مجاز نیست" in scripted.all_text()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"command", ["git push origin dev", "git remote add x y", "git config user.email a@b.c"]
|
||||
)
|
||||
async def test_the_model_cannot_run_git(_db, session_factory, command):
|
||||
scripted = ScriptedModel(
|
||||
[
|
||||
ScriptedReply(content=f'[[RUN cmd="{command}"/]]'),
|
||||
ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"),
|
||||
]
|
||||
)
|
||||
sandbox = FakeSandbox()
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
result = await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
|
||||
assert result.ok is True
|
||||
assert sandbox.executed == []
|
||||
assert "پلتفرم" in scripted.all_text()
|
||||
|
||||
|
||||
async def test_edit_file_applies_a_search_replace_block(_db, session_factory):
|
||||
scripted = ScriptedModel(
|
||||
[
|
||||
ScriptedReply(
|
||||
content=(
|
||||
"[[EDIT_FILE path=\"app/x.py\"]]<<<OLD\nold line\n===\n"
|
||||
"new line\nNEW>>>[[/EDIT_FILE]]"
|
||||
)
|
||||
),
|
||||
ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"),
|
||||
]
|
||||
)
|
||||
sandbox = FakeSandbox(files={"/workspace/app/x.py": "before\nold line\nafter"})
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
result = await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
|
||||
assert result.ok is True
|
||||
assert sandbox.files["/workspace/app/x.py"] == "before\nnew line\nafter"
|
||||
|
||||
|
||||
async def test_edit_file_fails_when_the_old_text_is_not_unique(_db, session_factory):
|
||||
scripted = ScriptedModel(
|
||||
[
|
||||
ScriptedReply(
|
||||
content=(
|
||||
"[[EDIT_FILE path=\"app/x.py\"]]<<<OLD\nline\n===\n"
|
||||
"other\nNEW>>>[[/EDIT_FILE]]"
|
||||
)
|
||||
),
|
||||
ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"),
|
||||
]
|
||||
)
|
||||
sandbox = FakeSandbox(files={"/workspace/app/x.py": "line\nline"})
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
result = await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
|
||||
assert result.ok is True
|
||||
assert sandbox.files["/workspace/app/x.py"] == "line\nline"
|
||||
assert "دقیقاً یک بار" in scripted.all_text()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bounds: the tool cap and the timeout
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_the_tool_call_cap_stops_a_runaway_turn(_db, session_factory, monkeypatch):
|
||||
monkeypatch.setattr(settings, "engine_max_tool_calls_per_turn", 2)
|
||||
scripted = ScriptedModel([ScriptedReply(content='[[LIST_DIR path="."/]]') for _ in range(5)])
|
||||
sandbox = FakeSandbox()
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
result = await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
|
||||
assert result.ok is False
|
||||
assert result.incomplete is True
|
||||
assert result.tool_calls == 2
|
||||
assert result.error_kind == "protocol"
|
||||
|
||||
|
||||
async def test_the_turn_timeout_stops_a_slow_loop(_db, session_factory, monkeypatch):
|
||||
monkeypatch.setattr(settings, "engine_turn_timeout_seconds", 0)
|
||||
scripted = ScriptedModel([ScriptedReply(content='[[LIST_DIR path="."/]]')] * 3)
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=FakeSandbox(), grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
result = await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
|
||||
assert result.ok is False
|
||||
assert result.error_kind == "timeout"
|
||||
|
||||
|
||||
async def test_a_cancelled_run_stops_between_steps(_db, session_factory):
|
||||
scripted = ScriptedModel(
|
||||
[
|
||||
ScriptedReply(content='[[LIST_DIR path="."/]]'),
|
||||
ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"),
|
||||
]
|
||||
)
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=FakeSandbox(), grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
request = await _request(_db, **DEVELOPER)
|
||||
# The hook cancels after the first model call has been made.
|
||||
calls = {"n": 0}
|
||||
|
||||
async def should_cancel() -> bool:
|
||||
calls["n"] += 1
|
||||
return calls["n"] > 1
|
||||
|
||||
request.should_cancel = should_cancel
|
||||
result = await runner.run_turn(request)
|
||||
await session.commit()
|
||||
|
||||
assert result.ok is False
|
||||
assert result.error_kind == "cancelled"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Errors: transient versus fatal
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_rate_limit_is_transient_and_reported(_db, session_factory):
|
||||
scripted = ScriptedModel([ScriptedReply(raises=ProviderUnavailableError("429 rate limited"))])
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=FakeSandbox(), grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
result = await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
|
||||
assert result.ok is False
|
||||
assert result.error_kind == "rate_limit"
|
||||
assert result.retryable is True
|
||||
|
||||
|
||||
async def test_an_ungranted_model_is_refused_before_any_call(_db, session_factory):
|
||||
scripted = ScriptedModel([ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]")])
|
||||
grants = FakeGrants(["some-other-model"])
|
||||
async with session_factory() as session:
|
||||
runner = _runner(session, model=scripted, sandbox=FakeSandbox(), grants=grants)
|
||||
result = await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
|
||||
assert result.ok is False
|
||||
assert result.error_kind == "auth"
|
||||
assert result.retryable is False
|
||||
# The model was never called: an ungranted employee spends nothing.
|
||||
assert scripted.calls == 0
|
||||
assert grants.checks == [_db.model_id]
|
||||
|
||||
|
||||
def test_error_classification():
|
||||
assert classify_error(ProviderUnavailableError("429 rate limit")) is ErrorKind.RATE_LIMIT
|
||||
assert classify_error(ProviderUnavailableError("auth failed")) is ErrorKind.AUTH
|
||||
assert classify_error(ProviderUnavailableError("connection refused")) is ErrorKind.NETWORK
|
||||
assert classify_error(TimeoutError()) is ErrorKind.TIMEOUT
|
||||
assert classify_error(ProviderUnavailableError("boom")) is ErrorKind.UNAVAILABLE
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Billing and secrets
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_every_model_call_writes_usage_events(_db, session_factory):
|
||||
"""agybot forgot the manager chat; the engine does not."""
|
||||
scripted = ScriptedModel(
|
||||
[
|
||||
ScriptedReply(content='[[LIST_DIR path="."/]]', input_tokens=11, output_tokens=7),
|
||||
ScriptedReply(
|
||||
content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]", input_tokens=13, output_tokens=9
|
||||
),
|
||||
]
|
||||
)
|
||||
from sqlalchemy import select
|
||||
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=FakeSandbox(), grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
rows = (
|
||||
(await session.execute(select(UsageEventRow).where(UsageEventRow.run_id == _db.run_id)))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
assert len(rows) == 4 # two calls, input and output each
|
||||
assert sum(r.quantity for r in rows if r.unit == "model_input_tokens") == 24
|
||||
assert sum(r.quantity for r in rows if r.unit == "model_output_tokens") == 16
|
||||
|
||||
|
||||
async def test_no_secret_from_settings_reaches_the_model(_db, session_factory, monkeypatch):
|
||||
"""The prompt is built by the engine; the settings stay on the server."""
|
||||
monkeypatch.setattr(settings, "jwt_secret", "super-secret-jwt-value")
|
||||
monkeypatch.setattr(settings, "database_url", "postgresql+asyncpg://u:p@h/db")
|
||||
scripted = ScriptedModel(
|
||||
[
|
||||
ScriptedReply(content='[[READ_FILE path="app/x.py"/]]'),
|
||||
ScriptedReply(content="[[WORK_REPORT]]تمام[[/WORK_REPORT]]"),
|
||||
]
|
||||
)
|
||||
sandbox = FakeSandbox(files={"/workspace/app/x.py": "print(1)"})
|
||||
async with session_factory() as session:
|
||||
runner = _runner(
|
||||
session, model=scripted, sandbox=sandbox, grants=FakeGrants([_db.model_id])
|
||||
)
|
||||
await runner.run_turn(await _request(_db, **DEVELOPER))
|
||||
await session.commit()
|
||||
|
||||
seen = scripted.all_text()
|
||||
assert "super-secret-jwt-value" not in seen
|
||||
assert "postgresql+asyncpg://u:p@h/db" not in seen
|
||||
# The model still got the rules and the role it plays.
|
||||
assert "قوانین پلتفرم" in seen
|
||||
assert "WORK_REPORT" in seen
|
||||
Reference in New Issue
Block a user