u78649634_default/smart-org
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
d532929189cc529d87ff6b483b02cb87a75690bd
smart-org/apps/api/tests/test_memory.py
T

648 lines
26 KiB
Python
Raw Blame History

"""Org memory: what an organization learns about itself.
The invariants under test:
* teaching the same thing twice teaches nothing (dedupe by hash, corrections by
key);
* memory is capped and evicts the oldest *unpinned lesson* first — never a rule;
* one task may write at most three memories;
* the context block respects its budget, leads with pinned rules, and filters by
role (only code-reading roles see the code map; only the manager sees
preferences);
* a reflection that fails never raises into the engine;
* nothing secret is ever stored, and no organization ever sees another's memory.
"""
from __future__ import annotations
from contextlib import asynccontextmanager
import pytest
from httpx import AsyncClient
from sqlalchemy import select
from app.application.container_org import build_memory, build_memory_service
from app.application.ports.memory import TaskOutcome
from app.core.config import settings
from app.core.errors import NotFoundError, ValidationError
from app.domain.base import new_id, utcnow
from app.domain.identity.models import PlatformRole, Role
from app.domain.memory.models import (
CONTENT_MAX_CHARS,
KEY_MAX_CHARS,
MAX_WRITES_PER_TASK,
MemoryKind,
MemorySource,
content_hash,
lesson_expiry,
normalise,
tokenise,
validate_content,
validate_key,
visible_kinds,
)
from app.infrastructure.db.models import (
AuditEventRow,
MembershipRow,
OrganizationRow,
TaskRow,
UsageEventRow,
UserRow,
)
from app.infrastructure.db.models_org import OrgMemoryRow
from tests.conftest import auth_header, signup
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
async def _task(session_factory, org_id: str, task_id: str) -> str:
"""A real task row: ``evidence_task_id`` is a real reference, not a string."""
async with session_factory() as session:
session.add(
TaskRow(id=task_id, organization_id=org_id, title=f"کار {task_id}")
)
await session.commit()
return task_id
async def _org(session_factory, slug: str, role: Role = Role.OWNER) -> tuple[str, str]:
"""An organization with one member, written directly to the database."""
async with session_factory() as session:
user = UserRow(
id=new_id(),
email=f"{slug}@example.com",
password_hash="x",
name=slug,
platform_role=PlatformRole.MANAGER.value,
)
session.add(user)
await session.flush()
org = OrganizationRow(id=new_id(), name=slug, slug=slug, owner_user_id=user.id)
session.add(org)
await session.flush()
session.add(
MembershipRow(organization_id=org.id, user_id=user.id, role=role.value)
)
await session.commit()
return org.id, user.id
class _ShapedProvider:
"""A provider that answers the reflection prompt in the template it asks for."""
def __init__(self, answer: str) -> None:
self.answer = answer
self.calls = 0
async def complete(self, **_kwargs):
from app.application.ports import ModelResponse
self.calls += 1
return ModelResponse(
content=self.answer,
model_internal_name="fake",
input_tokens=40,
output_tokens=12,
)
async def health(self):
from app.application.ports import ModelHealth
return ModelHealth(ok=True, detail="shaped fake")
class _BrokenProvider:
"""A provider that fails exactly like a network outage would."""
async def complete(self, **_kwargs):
raise RuntimeError("provider is unreachable: token=super-secret-token")
async def health(self):
from app.application.ports import ModelHealth
return ModelHealth(ok=False, detail="down")
@asynccontextmanager
async def _service(session_factory, provider):
"""A memory service on its own session, committed when the block exits."""
session = session_factory()
try:
service = build_memory_service(session, provider=provider)
yield service
await session.commit()
finally:
await session.close()
# ---------------------------------------------------------------------------
# Domain rules
# ---------------------------------------------------------------------------
def test_normalisation_and_dedupe_hash():
assert normalise(" the API runs \n on 8000 ") == "the API runs on 8000"
assert content_hash("The API runs on 8000") == content_hash(" the api runs on 8000 ")
assert content_hash("the API runs on 8000") != content_hash("the API runs on 8001")
def test_limits_are_enforced_in_the_domain():
assert len(validate_key("k" * KEY_MAX_CHARS)) == KEY_MAX_CHARS
with pytest.raises(ValidationError):
validate_key("k" * (KEY_MAX_CHARS + 1))
assert validate_content(" x ") == "x"
with pytest.raises(ValidationError):
validate_content("x" * (CONTENT_MAX_CHARS + 1))
def test_role_filtering_of_kinds():
# A reviewer reads the code map; support does not.
assert MemoryKind.CODE_MAP in visible_kinds("reviewer")
assert MemoryKind.CODE_MAP not in visible_kinds("support")
assert MemoryKind.CODE_MAP in visible_kinds("developer")
# Preferences belong to the manager alone.
assert MemoryKind.PREFERENCE in visible_kinds("manager")
assert MemoryKind.PREFERENCE not in visible_kinds("developer")
# Rules and lessons are everyone's.
for role in ("manager", "developer", "support", "researcher"):
assert visible_kinds(role) >= {MemoryKind.RULE, MemoryKind.LESSON}
def test_lessons_expire_unless_pinned_or_confirmed():
assert lesson_expiry(pinned=False, confirmed=False) is not None
assert lesson_expiry(pinned=True, confirmed=False) is None
assert lesson_expiry(pinned=False, confirmed=True) is None
def test_tokenise_drops_stop_words():
words = tokenise("the API and the manager را")
assert "api" in words and "manager" in words
assert "the" not in words and "را" not in words
# ---------------------------------------------------------------------------
# record: dedupe, correction, cap, task ceiling, redaction
# ---------------------------------------------------------------------------
async def test_recording_the_same_thing_twice_updates_one_memory(session_factory):
org_id, _ = await _org(session_factory, "dedupe")
async with _service(session_factory, _ShapedProvider("")) as service:
first = await service.record(
organization_id=org_id, kind="rule", content="همیشه تست بنویس",
source="user",
)
second = await service.record(
organization_id=org_id, kind="rule", content=" همیشه تست بنویس ",
source="user",
)
assert first == second
async with session_factory() as session:
rows = (
(await session.execute(select(OrgMemoryRow))).scalars().all()
)
assert len(rows) == 1
async def test_a_key_corrects_instead_of_duplicating(session_factory):
org_id, _ = await _org(session_factory, "keyed")
async with _service(session_factory, _ShapedProvider("")) as service:
await service.record(
organization_id=org_id, kind="fact", content="پورت ۸۰۰۰ است",
source="user", key="app_port",
)
memory_id = await service.record(
organization_id=org_id, kind="fact", content="پورت ۸۰۰۱ است",
source="user", key="app_port",
)
memory = await service.get(org_id, memory_id)
assert memory.content == "پورت ۸۰۰۱ است"
memories = await service.list(org_id)
assert len(memories) == 1
async def test_the_cap_evicts_the_oldest_unpinned_lesson_first(session_factory, monkeypatch):
monkeypatch.setattr(settings, "memory_org_char_cap", 200)
org_id, _ = await _org(session_factory, "capped")
await _task(session_factory, org_id, "t-pinned")
async with _service(session_factory, _ShapedProvider("")) as service:
# A pinned lesson survives; the rest are evicted oldest-first.
pinned = await service.record(
organization_id=org_id, kind="lesson", content="پ" * 40,
source="reflection", evidence_task_id="t-pinned",
)
await service.set_pinned(organization_id=org_id, memory_id=pinned, pinned=True)
for index in range(6):
await _task(session_factory, org_id, f"t-{index}")
await service.record(
organization_id=org_id, kind="lesson", content=f"درس {index}" + "ی" * 30,
source="reflection", evidence_task_id=f"t-{index}",
)
memories = await service.list(org_id)
# Budget spent, and whatever fell out is the oldest, never the pinned one.
assert (await service.stats(org_id))["chars"] <= 200
ids = {m.id for m in memories}
assert pinned in ids
# Only lessons are evictable, so a rule or a preference is never the
# victim — and the budget is honoured.
assert all(m.kind is MemoryKind.LESSON for m in memories)
async def test_one_task_may_write_at_most_three_memories(session_factory):
org_id, _ = await _org(session_factory, "three-writes")
await _task(session_factory, org_id, "task-1")
async with _service(session_factory, _ShapedProvider("")) as service:
written = [
await service.record(
organization_id=org_id, kind="lesson", content=f"درس شمارهٔ {index}",
source="reflection", evidence_task_id="task-1",
)
for index in range(MAX_WRITES_PER_TASK + 2)
]
assert written[:MAX_WRITES_PER_TASK]
# The 4th write is refused — ``None``, not an exception.
assert written[MAX_WRITES_PER_TASK] is None
assert written[MAX_WRITES_PER_TASK + 1] is None
rows = await service.list(org_id)
assert len(rows) == MAX_WRITES_PER_TASK
async def test_nothing_secret_is_ever_stored(session_factory):
org_id, _ = await _org(session_factory, "redacted")
async with _service(session_factory, _ShapedProvider("")) as service:
await service.record(
organization_id=org_id, kind="rule",
content="کلید API کل من ghp_ABCDEFGHIJKLMNOPQRSTUVWXyz123 است",
source="user",
)
memories = await service.list(org_id)
assert "ghp_ABCDEFGHIJKLMNOPQRSTUVWXyz123" not in memories[0].content
# ---------------------------------------------------------------------------
# build_context
# ---------------------------------------------------------------------------
async def test_context_leads_with_pinned_rules_and_respects_the_budget(session_factory):
org_id, _ = await _org(session_factory, "context")
async with _service(session_factory, _ShapedProvider("")) as service:
pinned_rule = await service.record(
organization_id=org_id, kind="rule", content="قاعدهٔ سنجاق‌شده", source="user"
)
await service.set_pinned(organization_id=org_id, memory_id=pinned_rule, pinned=True)
await service.record(
organization_id=org_id, kind="preference", content="ترجیح معمولی", source="user"
)
for index in range(20):
await service.record(
organization_id=org_id, kind="lesson", content=f"درس قدیمی {index}" + " ز" * 20,
source="user",
)
block = await service.build_context(
organization_id=org_id, role="manager", query="درس", budget_chars=300
)
assert block
assert len(block) <= 300
assert "قاعدهٔ سنجاق‌شده" in block.splitlines()[1]
assert block.index("قاعدهٔ سنجاق‌شده") < block.index("ترجیح معمولی")
# A budget so small that nothing fits returns nothing, not a broken block.
assert await service.build_context(
organization_id=org_id, role="manager", query="درس", budget_chars=4
) == ""
async def test_context_filters_kinds_by_role(session_factory):
org_id, _ = await _org(session_factory, "roles")
async with _service(session_factory, _ShapedProvider("")) as service:
await service.record(
organization_id=org_id, kind="code_map",
content="نقشهٔ کد: مدل‌ها در db/models.py", source="system",
)
await service.record(
organization_id=org_id, kind="preference", content="ترجیح: فارسی بنویس",
source="user",
)
developer = await service.build_context(
organization_id=org_id, role="developer", query="db", budget_chars=2000
)
manager = await service.build_context(
organization_id=org_id, role="manager", query="db", budget_chars=2000
)
support = await service.build_context(
organization_id=org_id, role="support", query="db", budget_chars=2000
)
assert "نقشهٔ کد" in developer and "ترجیح" not in developer
assert "ترجیح" in manager and "نقشهٔ کد" not in manager
assert support == ""
async def test_context_counts_a_hit(session_factory):
org_id, _ = await _org(session_factory, "hits")
async with _service(session_factory, _ShapedProvider("")) as service:
memory_id = await service.record(
organization_id=org_id, kind="rule", content="هر تست قبل از کد", source="user"
)
await service.build_context(
organization_id=org_id, role="developer", query="تست", budget_chars=500
)
memory = await service.get(org_id, memory_id)
assert memory.hits == 1
assert memory.last_used_at is not None
# ---------------------------------------------------------------------------
# Reflection
# ---------------------------------------------------------------------------
async def test_reflection_stores_lessons_and_prices_the_call(session_factory, seeded):
org_id, _ = await _org(session_factory, "reflect")
await _task(session_factory, org_id, "task-77")
provider = _ShapedProvider(
"درس ۱: تست‌ها را قبل از ادغام اجرا کن.\n"
"درس ۲: دروازهٔ lint دو بار شکست.\n"
"نقشهٔ کد: app/db/models.py منبع حقیقت است.\n"
"یک خط اضافی که نادیده گرفته می‌شود."
)
async with _service(session_factory, provider) as service:
await service.on_task_finished(
organization_id=org_id,
task=TaskOutcome(
task_id="task-77", title="ساخت صفحهٔ ورود", status="completed",
summary="تمام شد", changed_files=["a.py"], cycles=2, gate_failures=["lint"],
),
)
memories = await service.list(org_id)
kinds = [m.kind for m in memories]
assert MemoryKind.LESSON in kinds
assert MemoryKind.CODE_MAP in kinds
assert len(memories) == 3
assert all(m.source is MemorySource.REFLECTION for m in memories)
assert all(m.confidence == 0.5 for m in memories)
assert all(m.evidence_task_id == "task-77" for m in memories)
# The reflection is billed like any other model call.
async with session_factory() as session:
usage = (
(await session.execute(select(UsageEventRow))).scalars().all()
)
assert usage
assert all(row.metadata_json.get("memory_reflection") for row in usage)
async def test_a_failing_reflection_never_raises_and_records_the_error(
session_factory, redirect_error_log, seeded
):
org_id, _ = await _org(session_factory, "reflect-fail")
async with _service(session_factory, _BrokenProvider()) as service:
# No exception escapes, and the org keeps its memory untouched.
await service.on_task_finished(
organization_id=org_id,
task=TaskOutcome(task_id="task-err", title="t", status="failed"),
)
assert await service.list(org_id) == []
from app.infrastructure.db.models import ErrorEventRow
async with session_factory() as session:
rows = (await session.execute(select(ErrorEventRow))).scalars().all()
assert any(row.kind == "memory.reflection_failed" for row in rows)
# A provider's failure text is not trusted verbatim: either it is redacted,
# or it never reached the row at all.
for row in rows:
if "super-secret-token" in (row.message or ""):
assert "REDACTED" in row.message
async def test_reflection_without_a_model_is_recorded_not_raised(
session_factory, redirect_error_log
):
"""No seeded catalogue → ``pick_for_mode`` returns None → something is recorded."""
org_id, _ = await _org(session_factory, "no-model")
async with _service(session_factory, _ShapedProvider("")) as service:
await service.on_task_finished(
organization_id=org_id,
task=TaskOutcome(task_id="task-x", title="t", status="completed"),
)
assert await service.list(org_id) == []
from app.infrastructure.db.models import ErrorEventRow
async with session_factory() as session:
rows = (await session.execute(select(ErrorEventRow))).scalars().all()
assert any(row.kind == "memory.reflection_no_model" for row in rows)
async def test_expired_lessons_are_removed_by_the_expiry_job(session_factory):
org_id, _ = await _org(session_factory, "expiry")
async with _service(session_factory, _ShapedProvider("")) as service:
expired = await service.record(
organization_id=org_id, kind="lesson", content="درس منقضی", source="reflection"
)
pinned = await service.record(
organization_id=org_id, kind="lesson", content="درس سنجاق‌شده", source="reflection"
)
await service.set_pinned(organization_id=org_id, memory_id=pinned, pinned=True)
async with session_factory() as session:
row = await session.get(OrgMemoryRow, expired)
row.expires_at = utcnow().replace(year=2020) # long past
await session.commit()
assert await service.expire_memories() == 1
remaining = {m.id for m in await service.list(org_id)}
assert expired not in remaining
assert pinned in remaining
# ---------------------------------------------------------------------------
# Tenant isolation
# ---------------------------------------------------------------------------
async def test_one_organization_never_sees_anothers_memory(session_factory):
org_a, _ = await _org(session_factory, "tenant-a")
org_b, _ = await _org(session_factory, "tenant-b")
async with _service(session_factory, _ShapedProvider("")) as service:
memory_id = await service.record(
organization_id=org_a, kind="rule", content="قاعدهٔ سازمان الف", source="user"
)
assert memory_id not in {m.id for m in await service.list(org_b)}
with pytest.raises(NotFoundError):
await service.get(org_b, memory_id)
block = await service.build_context(
organization_id=org_b, role="manager", query="قاعده", budget_chars=500
)
assert block == ""
# ---------------------------------------------------------------------------
# HTTP surface
# ---------------------------------------------------------------------------
async def _owner_with_org(client: AsyncClient, slug: str) -> dict:
body = await signup(client, f"{slug}-owner@example.com", name="Owner")
org = await client.post(
"/api/organizations", json={"name": slug}, headers=auth_header(body["access_token"])
)
assert org.status_code == 201, org.text
return {"token": body["access_token"], "org_id": org.json()["id"]}
async def _member(client: AsyncClient, owner_token: str, org_id: str, email: str, role: str):
invite = await client.post(
f"/api/organizations/{org_id}/invites",
json={"email": email, "role": role},
headers=auth_header(owner_token),
)
assert invite.status_code == 201, invite.text
account = await signup(client, email, name=role)
accepted = await client.post(
"/api/invites/accept",
json={"token": invite.json()["invite_token"]},
headers=auth_header(account["access_token"]),
)
assert accepted.status_code == 200, accepted.text
return account["access_token"]
async def test_memory_http_permissions(client: AsyncClient):
org = await _owner_with_org(client, "memory-org")
viewer = await _member(client, org["token"], org["org_id"], "viewer@example.com", "viewer")
member = await _member(client, org["token"], org["org_id"], "member@example.com", "member")
admin = await _member(client, org["token"], org["org_id"], "admin@example.com", "admin")
# A viewer reads but cannot teach.
empty = await client.get(
f"/api/organizations/{org['org_id']}/memory", headers=auth_header(viewer)
)
assert empty.status_code == 200, empty.text
assert empty.json()["items"] == []
refused = await client.post(
f"/api/organizations/{org['org_id']}/memory",
json={"kind": "rule", "content": "قاعدهٔ ممنوعه"},
headers=auth_header(viewer),
)
assert refused.status_code == 403, refused.text
# A member teaches a rule; a lesson is refused (teaching is human-only).
taught = await client.post(
f"/api/organizations/{org['org_id']}/memory",
json={"kind": "rule", "content": "قاعدهٔ تازه", "key": "style"},
headers=auth_header(member),
)
assert taught.status_code == 201, taught.text
memory_id = taught.json()["id"]
assert taught.json()["kind_label"] == "قاعده"
lesson = await client.post(
f"/api/organizations/{org['org_id']}/memory",
json={"kind": "lesson", "content": "درس دست‌نویس"},
headers=auth_header(member),
)
assert lesson.status_code == 403, lesson.text
# Editing is admin+.
edited = await client.patch(
f"/api/organizations/{org['org_id']}/memory/{memory_id}",
json={"content": "قاعدهٔ ویرایش‌شده"},
headers=auth_header(admin),
)
assert edited.status_code == 200, edited.text
assert edited.json()["content"] == "قاعدهٔ ویرایش‌شده"
member_edit = await client.patch(
f"/api/organizations/{org['org_id']}/memory/{memory_id}",
json={"content": "تلاش عضو"},
headers=auth_header(member),
)
assert member_edit.status_code == 403, member_edit.text
# Pin, then confirm (only a lesson may be confirmed).
pinned = await client.post(
f"/api/organizations/{org['org_id']}/memory/{memory_id}/pin",
json={"pinned": True},
headers=auth_header(admin),
)
assert pinned.json()["pinned"] is True
not_a_lesson = await client.post(
f"/api/organizations/{org['org_id']}/memory/{memory_id}/confirm",
headers=auth_header(admin),
)
assert not_a_lesson.status_code == 422, not_a_lesson.text
# Remembering without a session is not a thing.
anonymous = await client.get(f"/api/organizations/{org['org_id']}/memory")
assert anonymous.status_code == 401
async def test_memory_audit_and_isolation(client: AsyncClient, session_factory):
org = await _owner_with_org(client, "audited-memory")
other = await _owner_with_org(client, "other-memory")
admin = await _member(client, org["token"], org["org_id"], "a2@example.com", "admin")
created = await client.post(
f"/api/organizations/{org['org_id']}/memory",
json={"kind": "rule", "content": "قاعدهٔ حسابرسی"},
headers=auth_header(admin),
)
assert created.status_code == 201, created.text
memory_id = created.json()["id"]
await client.patch(
f"/api/organizations/{org['org_id']}/memory/{memory_id}",
json={"content": "قاعدهٔ حسابرسی، ویرایش‌شده"},
headers=auth_header(admin),
)
await client.post(
f"/api/organizations/{org['org_id']}/memory/{memory_id}/pin",
json={"pinned": True},
headers=auth_header(admin),
)
await client.delete(
f"/api/organizations/{org['org_id']}/memory/{memory_id}", headers=auth_header(admin)
)
async with session_factory() as session:
actions = [
row.action
for row in (
await session.execute(
select(AuditEventRow).where(
AuditEventRow.organization_id == org["org_id"]
)
)
).scalars()
]
assert {"memory.create", "memory.update", "memory.pin", "memory.delete"} <= set(actions)
# Another organization's memory is invisible here, never another tenant's
# data: the soft-deleted row exists, and the list is still empty.
stranger = await client.get(
f"/api/organizations/{other['org_id']}/memory",
headers=auth_header(other["token"]),
)
assert stranger.status_code == 200, stranger.text
assert stranger.json()["items"] == []
async def test_the_real_port_is_wired_in_the_container(session_factory):
from app.application.ports.memory import MemoryPort, NullMemory
org_id, _ = await _org(session_factory, "wired")
async with session_factory() as session:
memory = build_memory(session)
assert isinstance(memory, MemoryPort)
assert not isinstance(memory, NullMemory)
recorded = await memory.record(
organization_id=org_id, kind="rule", content="قاعدهٔ سیم‌کشی‌شده", source="user"
)
assert recorded
block = await memory.build_context(
organization_id=org_id, role="manager", query="سیم", budget_chars=200
)
assert "قاعدهٔ سیم‌کشی‌شده" in block
Reference in New Issue View Git Blame Copy Permalink