Files
smart-org/apps/api/app/application/container_delivery.py
T
mjalmoudy94 a1b8a28dda feat(delivery): live CI/CD pipeline + demo runtime (v2 phase A3-3)
A push to `dev` now becomes a live demo, and a failed build never takes the
demo down.

- Four tables (pipeline_runs, pipeline_steps, releases, app_runtimes) and their
  migration; alembic upgrade/downgrade/check all clean.
- DeliveryService implements DeliveryHooks: on_app_created provisions the demo
  port + host + first build (idempotently); on_branch_pushed queues a run for
  `dev` and supersedes older *queued* runs, leaving a running one alone.
- The 8-step pipeline. Blue/green is real: the replacement starts on a staging
  port, is healthchecked, and only then does the route move — then the previous
  container is stopped. A failure removes the replacement, releases its port and
  leaves the old container serving.
- The demo port ledger is the DATABASE, not the router adapters' in-memory maps:
  the API and the delivery worker are separate processes.
- Runs are claimed with a conditional UPDATE, so two workers never build the
  same run.
- Live logs are batched (1 event / 500 ms, 200 lines) and redacted as a whole
  tail — per-chunk redaction misses a secret split across pipe reads, and the
  *stored* tail leaked until this was fixed.
- delivery worker: drain the queue, then poll remotes for pushes made outside
  the engine.
- 7 routes; reads need membership, actions need projects.manage and are audited.

Tests: 415 API (+21, 1 docker-marked skipped); ruff, mypy, alembic check clean.

Report: docs/plan/reports/agent-3-phase-3.md
2026-10-14 00:44:02 +03:30

190 lines
7.3 KiB
Python

"""Wiring for the delivery workstream (repo workspaces, delivery hooks, deploy tester).
The real adapters are selected by settings; the fakes remain the defaults so a
test that does not opt in never touches a git binary, a container or a network.
``build_repo_workspace`` is the interesting one: the git adapter needs to know
*where* a project's repository lives and *how* to authenticate to it, and that
knowledge lives in three places — ``project_repositories`` +
``project_git_credentials`` for the platform forge, ``project_external_git`` for a
user-supplied remote, and ``projects`` for which of the two applies. The resolver
below is the only code that joins them, and it hands the plaintext secret
straight to the adapter for the duration of a single call.
"""
from __future__ import annotations
import logging
from sqlalchemy.ext.asyncio import AsyncSession
from app.application.ports.delivery_hooks import DeliveryHooks
from app.application.ports.deploy_target import DeployTargetTester, FakeDeployTargetTester
from app.application.ports.repo_workspace import FakeRepoWorkspace, RepoWorkspacePort
from app.application.services.delivery_service import DeliveryService
from app.core.config import settings
from app.core.crypto import decrypt
from app.infrastructure.adapters.git_workspace import (
LocalGitWorkspaceAdapter,
ProjectGitSource,
)
from app.infrastructure.adapters.registry import get_domain_router, get_sandbox_provider
from app.infrastructure.db.models import ProjectRow
from app.infrastructure.repositories.app_profile import AppProfileRepository
from app.infrastructure.repositories.delivery import (
AppRuntimeRepository,
PipelineRunRepository,
PipelineStepRepository,
ReleaseRepository,
)
from app.infrastructure.repositories.identity import OrganizationRepository
from app.infrastructure.repositories.projects import (
PreviewDomainRepository,
ProjectExternalGitRepository,
ProjectGitCredentialRepository,
ProjectGitRepoRepository,
ProjectRepository,
)
logger = logging.getLogger("agy.container.delivery")
_fake_workspace = FakeRepoWorkspace()
_fake_deploy_tester = FakeDeployTargetTester()
def build_repo_workspace(session: AsyncSession) -> RepoWorkspacePort:
"""The platform's git adapter, or the in-memory fake when ``GIT_WORKSPACE=fake``."""
if settings.git_workspace.strip().lower() != "local":
return _fake_workspace
projects = ProjectRepository(session)
repos = ProjectGitRepoRepository(session)
credentials = ProjectGitCredentialRepository(session)
external = ProjectExternalGitRepository(session)
async def resolve(project_id: str) -> ProjectGitSource | None:
project = await projects.get(project_id)
if project is None:
return None
row = await session.get(ProjectRow, project_id)
if row is None:
return None
git_source = (row.git_source or "platform").lower()
if git_source == "external":
url = (row.external_repo_url or "").strip()
if not url:
return None
record = await external.get_for_project(project_id)
return ProjectGitSource(
project_id=project_id,
organization_id=project.organization_id,
clone_url=url,
default_branch=project.default_branch,
auth_type=(record.auth_type if record else "none") or "none",
username=record.username if record else None,
secret=_decrypt_or_none(
record.secret_encrypted if record else None, project_id
),
known_hosts=record.known_hosts if record else None,
is_platform=False,
)
repo = await repos.get_for_project(project_id)
if repo is None or not repo.clone_url:
return None
credential = await credentials.get_for_project(project_id)
secret = _decrypt_or_none(
credential.encrypted_password if credential else None, project_id
)
return ProjectGitSource(
project_id=project_id,
organization_id=project.organization_id,
clone_url=repo.clone_url,
default_branch=repo.default_branch or project.default_branch,
auth_type="token" if secret else "none",
username=credential.username if credential else None,
secret=secret,
is_platform=True,
)
return LocalGitWorkspaceAdapter(resolver=resolve)
def _decrypt_or_none(ciphertext: str | None, project_id: str) -> str | None:
"""Decrypt a stored secret, tolerating a rotated ``ENCRYPTION_KEY``.
A rotated key must surface as "no credential" (git then fails with an auth
error the user can act on) rather than as an exception that breaks an
unrelated request. The failure is logged, never raised with the ciphertext.
"""
if not ciphertext:
return None
try:
return decrypt(ciphertext)
except Exception: # noqa: BLE001 - see docstring
logger.warning("git credential for project %s could not be decrypted", project_id)
return None
def build_delivery_hooks(session: AsyncSession) -> DeliveryHooks:
"""The real CI/CD service — it *is* the hook implementation.
The org wizard calls `on_app_created` after its own transaction commits, and
the engine calls `on_branch_pushed` after a successful merge; both get the
same object the delivery router and worker use, so there is exactly one
implementation of "a push becomes a build".
"""
return build_delivery_service(session)
def build_delivery_service(session: AsyncSession) -> DeliveryService:
"""CI/CD: pipeline runs, the demo runtime and the deploy hooks."""
return DeliveryService(
runs=PipelineRunRepository(session),
steps=PipelineStepRepository(session),
releases=ReleaseRepository(session),
runtimes=AppRuntimeRepository(session),
projects=ProjectRepository(session),
profiles=AppProfileRepository(session),
organizations=OrganizationRepository(session),
ports=PreviewDomainRepository(session),
workspace=build_repo_workspace(session),
domains=get_domain_router(),
containers=_container_runtime(),
)
def _container_runtime():
"""The adapter that can start build and demo containers.
Falls back to a runtime that refuses everything when the configured sandbox
provider cannot run containers (the mock, or a remote provider): a pipeline
run then fails cleanly with a plain message instead of pretending to deploy.
"""
provider = get_sandbox_provider()
if hasattr(provider, "run_step") and hasattr(provider, "remove_container"):
return provider
return _UnavailableContainerRuntime()
class _UnavailableContainerRuntime:
"""A container runtime that cannot run anything, and says so."""
async def run_step(self, **_kwargs): # noqa: ANN003
from app.application.ports import CommandResult
return CommandResult(
exit_code=127,
stdout="",
stderr="SANDBOX_PROVIDER cannot run containers; set SANDBOX_PROVIDER=docker.",
)
async def remove_container(self, container_id: str) -> bool: # noqa: ARG002
return False
def build_deploy_target_tester(session: AsyncSession) -> DeployTargetTester:
return _fake_deploy_tester