From 6ca3b37016b0e0010f5c34fd6f3564553ca30853 Mon Sep 17 00:00:00 2001 From: Shehan Suraweera Date: Fri, 2 Oct 2026 10:18:21 +0530 Subject: [PATCH] feat: deploy the AI service with Docker and document the AI features - Dockerfile: locked deps, embedding model baked in, non-root user - deploy/compose.prod.yml: 127.0.0.1 only, 400 MiB limit (293 MiB peak measured), read-only filesystem, no capabilities, healthcheck - deploy-ai-service.yml: tests, image built once in CI and pushed to GHCR by commit SHA, migrate, start and wait for health, automatic rollback to the previous image; no stored registry credentials - CI: reusable AI service workflow, image build check on PRs; backend and AI deployments never run at the same time - Fix ai-db bootstrap for non-superuser admins such as the RDS master user, with a regression test - docs/ai.md: architecture, design decisions, safety, reliability, acceptance-rate queries and eval results - README: architecture diagram, AI features and design decisions; deployment guide: one-time AI setup, memory budget, runbook --- .github/workflows/ci-ai-service.yml | 41 ++- .github/workflows/deploy-ai-service.yml | 169 +++++++++++ .github/workflows/deploy-backend.yml | 6 + Readme.md | 112 ++++++- ai-service/.dockerignore | 14 + ai-service/Dockerfile | 72 +++++ ai-service/README.md | 17 +- ai-service/src/ai_service/db/bootstrap.py | 9 + ai-service/tests/conftest.py | 6 +- ai-service/tests/test_database_security.py | 40 +++ deploy/compose.prod.yml | 58 ++++ docs/ai.md | 321 +++++++++++++++++++++ docs/deployment.md | 238 +++++++++++++-- 13 files changed, 1064 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/deploy-ai-service.yml create mode 100644 ai-service/.dockerignore create mode 100644 ai-service/Dockerfile create mode 100644 deploy/compose.prod.yml create mode 100644 docs/ai.md diff --git a/.github/workflows/ci-ai-service.yml b/.github/workflows/ci-ai-service.yml index 479c451..7cb65b0 100644 --- a/.github/workflows/ci-ai-service.yml +++ b/.github/workflows/ci-ai-service.yml @@ -1,10 +1,15 @@ name: CI — AI service -# WHEN: every push and pull request that touches the AI service -# WHY: lint, type-check and test the Python service. Tests use scripted and -# fake providers, so CI needs no API key and makes no network calls to Gemini. +# WHEN: pushes to feature branches and pull requests that touch the AI service, +# and as the required first step of deploy-ai-service.yml (workflow_call) +# WHY: lint, type-check and test the Python service, and check that its Docker +# image builds. Tests use scripted and fake providers, so CI needs no API +# key and makes no network calls to Gemini. +# Pushes to main are skipped here because deploy-ai-service.yml runs this +# workflow for them; otherwise the tests would run twice. on: push: + branches-ignore: [main] paths: - "ai-service/**" - ".github/workflows/ci-ai-service.yml" @@ -12,6 +17,15 @@ on: paths: - "ai-service/**" - ".github/workflows/ci-ai-service.yml" + workflow_call: + inputs: + build-image: + description: Also check that the Docker image builds + type: boolean + default: true + +permissions: + contents: read jobs: ai-service: @@ -47,3 +61,24 @@ jobs: - name: Run tests run: uv run pytest + + # The deploy workflow builds and pushes the image itself, so it passes + # build-image: false. On pushes and pull requests the input is unset. + image: + name: AI service — Docker image builds + if: ${{ inputs.build-image != false }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build (not pushed) + uses: docker/build-push-action@v6 + with: + context: ai-service + push: false + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/deploy-ai-service.yml b/.github/workflows/deploy-ai-service.yml new file mode 100644 index 0000000..6d5be56 --- /dev/null +++ b/.github/workflows/deploy-ai-service.yml @@ -0,0 +1,169 @@ +name: Deploy AI service + +# WHEN: changes to the AI service or its production compose file reach main, +# or by hand from the Actions tab (e.g. to redeploy after a server rebuild) +# WHY: test, build the image once in CI, push it to GitHub Container Registry, +# and run exactly that image on the EC2. The server never builds anything, +# which matters on a 1 GB instance. +on: + push: + branches: [main] + paths: + - "ai-service/**" + - "deploy/compose.prod.yml" + - ".github/workflows/deploy-ai-service.yml" + workflow_dispatch: + +# One deployment to the EC2 at a time, shared with deploy-backend.yml, so a +# backend build and an image pull never compete for the instance's memory +concurrency: + group: deploy-ec2 + cancel-in-progress: false + +permissions: + contents: read + +jobs: + # A failing test must stop the deployment, not just report after it + test: + name: Lint, types and tests + uses: ./.github/workflows/ci-ai-service.yml + with: + build-image: false + + image: + name: Build and push image + needs: test + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + outputs: + image: ${{ steps.name.outputs.image }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + # Registry names must be lowercase; the GitHub owner name may not be + - name: Image name + id: name + run: | + repo="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/newnopdesk-ai-service" + echo "repo=$repo" >> "$GITHUB_OUTPUT" + echo "image=$repo:$GITHUB_SHA" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Tagged with the commit SHA, so every deployment names the exact code it + # runs and a rollback is just running the previous tag + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ai-service + push: true + tags: | + ${{ steps.name.outputs.image }} + ${{ steps.name.outputs.repo }}:latest + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + # One plain image per push (no attestation manifests), so the + # cleanup below can count versions without breaking an image + provenance: false + + # Private packages have little free storage and each image is a few + # hundred MB. The server keeps its own copy of the previous image for + # rollback, so the registry only needs recent versions. + - name: Delete old image versions + uses: actions/delete-package-versions@v5 + with: + package-name: newnopdesk-ai-service + package-type: container + min-versions-to-keep: 5 + + deploy: + name: Deploy to EC2 + needs: image + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Copy the compose file to the server + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.EC2_HOST }} + username: ubuntu + key: ${{ secrets.EC2_SSH_KEY }} + source: deploy/compose.prod.yml + target: /opt/newnopdesk + strip_components: 1 + + # The server pulls with this job's short-lived token, which expires when + # the job ends, so no registry credential is ever stored on the server + - name: Migrate and start the new image + uses: appleboy/ssh-action@v1.0.3 + env: + AI_IMAGE: ${{ needs.image.outputs.image }} + GHCR_USER: ${{ github.actor }} + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + host: ${{ secrets.EC2_HOST }} + username: ubuntu + key: ${{ secrets.EC2_SSH_KEY }} + envs: AI_IMAGE,GHCR_USER,GHCR_TOKEN + script: | + set -euo pipefail + cd /opt/newnopdesk + ENV_FILE=/etc/newnopdesk/ai-service.env + PREVIOUS=$(cat ai-image.txt 2>/dev/null || true) + + echo "==> Pulling $AI_IMAGE" + echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USER" --password-stdin + trap 'docker logout ghcr.io >/dev/null 2>&1 || true' EXIT + docker pull --quiet "$AI_IMAGE" + + # Migrations run before the switch, so they must keep working with + # the previous image too (add columns and tables; drop them later) + echo "==> Applying AI schema migrations" + docker run --rm --env-file "$ENV_FILE" \ + -v /etc/ssl/rds:/etc/ssl/rds:ro --read-only --tmpfs /tmp \ + "$AI_IMAGE" ai-db migrate + + echo "==> Starting the new container" + if AI_IMAGE="$AI_IMAGE" docker compose -f compose.prod.yml up -d --wait --wait-timeout 120; then + echo "$AI_IMAGE" > ai-image.txt + else + echo "!! New container did not become healthy; its last logs:" + AI_IMAGE="$AI_IMAGE" docker compose -f compose.prod.yml logs --tail 50 ai-service || true + if [ -n "$PREVIOUS" ]; then + echo "==> Rolling back to $PREVIOUS" + AI_IMAGE="$PREVIOUS" docker compose -f compose.prod.yml up -d --wait --wait-timeout 120 + fi + exit 1 + fi + + # Keep the running image and the previous one (for rollback); each is ~775 MB + echo "==> Removing older images" + REPO="${AI_IMAGE%:*}" + docker images "$REPO" --format '{{.Repository}}:{{.Tag}}' \ + | grep -vxF -e "$AI_IMAGE" -e "${PREVIOUS:-none}" \ + | xargs -r docker rmi >/dev/null || true + + echo "==> Deployed" + curl -fsS http://127.0.0.1:8000/healthz diff --git a/.github/workflows/deploy-backend.yml b/.github/workflows/deploy-backend.yml index 46af781..c417bee 100644 --- a/.github/workflows/deploy-backend.yml +++ b/.github/workflows/deploy-backend.yml @@ -9,6 +9,12 @@ on: - "backend/**" - ".github/workflows/deploy-backend.yml" +# One deployment to the EC2 at a time, shared with deploy-ai-service.yml, so a +# backend build and an image pull never compete for the instance's memory +concurrency: + group: deploy-ec2 + cancel-in-progress: false + jobs: # Runs the full CI workflow (typecheck + integration tests) first. # WHY: a failing test must stop the deployment, not just report after it diff --git a/Readme.md b/Readme.md index 2f3553e..2f03274 100644 --- a/Readme.md +++ b/Readme.md @@ -6,6 +6,8 @@ A multi-tenant client issue portal for software agencies — purpose-built aroun Inspired by ServiceNow and Jira Service Management, NewnopDesk gives each client a dedicated workspace to raise and track issues, while routing tickets to the right internal engineering team and giving leadership cross-portfolio visibility. +An AI assistant helps staff with the routine work: it suggests triage for new issues, tracks client sentiment, finds similar past issues with a suggested fix, and summarises long threads. Staff approve every suggestion; the AI never changes an issue itself, and client sentiment never changes priority. + --- ## Screenshots @@ -32,9 +34,60 @@ Inspired by ServiceNow and Jira Service Management, NewnopDesk gives each client | Document | Description | |----------|-------------| +| [AI Features](docs/ai.md) | What the AI does, architecture, design decisions, safety, eval results | | [Deployment Guide](docs/deployment.md) | Complete AWS setup, CI/CD, environment variables, runbook | -| [Backend README](backend/README.md) | API reference, data model, local development | -| [Frontend README](frontend/README.md) | Component structure, state management, local development | +| [Backend README](backend/README.md) | API reference, data model, AI job queue, local development | +| [Frontend README](frontend/README.md) | Component structure, state management, AI panels, local development | +| [AI Service README](ai-service/README.md) | Python service: API contract, code layout, tests | +| [Evaluations](ai-service/evals/README.md) | Labelled eval data, method and full results | + +--- + +## Architecture + +```mermaid +flowchart LR + U[Browser] --> CF[CloudFront] + CF -->|static files| S3[("S3: React app")] + CF -->|/api| N + + subgraph EC2["EC2 t3.micro"] + N["Nginx → Express API
+ AI job worker (PM2)"] + A["AI service
FastAPI (Docker)"] + end + + N -->|"127.0.0.1, service token"| A + N --> DB[("RDS PostgreSQL 17
app data + pgvector")] + A -->|"own schema and role"| DB + A --> G[Gemini API] + N -->|presigned URLs| S3F[("S3: attachments")] +``` + +The Express API owns all data and every access decision. AI work is queued in PostgreSQL in the same transaction as the issue or comment and processed in the background, so the app never waits for the AI and keeps working if it's down. The AI service only sees the text it's sent, and its database role can't read application tables. Details in [docs/ai.md](docs/ai.md). + +--- + +## AI Features + +All AI panels are for staff only. Clients never see AI output. + +| Feature | What staff get | +|---|---| +| **Triage suggestion** | Impact, urgency, category and owning team for each new issue, with reasons and the resulting priority. Apply, edit or reject. | +| **Client mood** | Sentiment, frustration and escalation risk on every client message, with the quoted evidence. High-risk issues are flagged on the dashboard. | +| **Similar issues and suggested fix** | Resolved issues from the same company, and a fix built from their resolution notes that cites its sources. | +| **Thread summary** | Key points citing the comments they came from, and open questions (admins). | +| **Client health** | Weekly sentiment trend per client company (admins). | + +**Measured, not assumed** ([full results](ai-service/evals/README.md)), on labelled data with the real model: + +| | Result | Baseline | +|---|---|---| +| Team routing | **91%** | 49% routing by product | +| Priority | **79%** (99% within one level) | 63% using the client's own impact and urgency | +| Sentiment agreement | **87.5%**, Cohen's κ 0.79 | 55% always guessing the most common label | +| Similar-issue recall@5 | **100%**, 0 cross-company results | – | +| Prompt-injection attacks blocked | **24 of 24** | – | --- @@ -44,9 +97,11 @@ Inspired by ServiceNow and Jira Service Management, NewnopDesk gives each client **Backend:** Node.js · Express · TypeScript · Prisma · PostgreSQL 17 -**Infrastructure:** AWS EC2 · RDS for PostgreSQL · S3 · CloudFront · Nginx · PM2 · Docker (local development) +**AI service:** Python 3.12 · FastAPI · Pydantic · Google Gemini (`gemini-3.5-flash-lite`) · fastembed (local embeddings) · pgvector · uv · pytest + +**Infrastructure:** AWS EC2 · RDS for PostgreSQL · S3 · CloudFront · Nginx · PM2 · Docker · GitHub Container Registry -**CI/CD:** GitHub Actions — integration tests must pass before every backend deploy +**CI/CD:** GitHub Actions — tests must pass before every deploy; the AI service ships as an image built in CI --- @@ -83,20 +138,45 @@ npm run dev App runs at `http://localhost:5173` — API calls proxy to `localhost:4000` via Vite config. +### AI service (optional) + +The app works without it. To turn the AI features on: + +```bash +cd ai-service +cp .env.example .env # set GEMINI_API_KEY and AI_SERVICE_TOKEN (or AI_PROVIDER=fake for no key) +uv sync +uv run ai-db bootstrap --admin-url postgresql://newnopdesk:newnopdesk_local@127.0.0.1:5432/newnopdesk +uv run ai-db migrate +uv run ai-serve # http://127.0.0.1:8000 +``` + +Then set `AI_ENABLED=true` and the same `AI_SERVICE_TOKEN` in `backend/.env`, restart the backend, and fill the AI data for the seeded issues with `npm run ai:reindex` and `npm run ai:backfill`. See the [AI service README](ai-service/README.md#running-locally). + --- ## Repository Structure ``` -localdev/ +IssueTracker/ ├── .github/ │ └── workflows/ -│ ├── ci.yml # Typecheck + integration tests +│ ├── ci.yml # Backend typecheck + integration tests +│ ├── ci-ai-service.yml # AI service lint, types, tests, image build │ ├── deploy-frontend.yml # S3 + CloudFront deployment -│ └── deploy-backend.yml # Runs ci.yml, then deploys to EC2 via SSH +│ ├── deploy-backend.yml # Runs ci.yml, then deploys to EC2 via SSH +│ └── deploy-ai-service.yml # Runs ci-ai-service.yml, pushes the image to GHCR, deploys it +├── ai-service/ +│ ├── src/ai_service/ # FastAPI app, prompts, LLM providers, retrieval, safety +│ ├── migrations/ # Alembic migrations for the `ai` schema +│ ├── evals/ # labelled eval data, runner and committed results +│ ├── tests/ +│ └── Dockerfile +├── deploy/ +│ └── compose.prod.yml # AI service container on the EC2 ├── backend/ │ ├── src/ -│ │ ├── features/ # auth, issues, products, users, companies +│ │ ├── features/ # auth, issues, products, users, companies, ai │ │ ├── middleware/ # authenticate, role guards │ │ ├── lib/ # prisma client, S3 helpers │ │ └── config/ # env validation @@ -123,6 +203,7 @@ localdev/ │ ├── store/ # Zustand stores (auth, tabs) │ └── types/ # TypeScript interfaces mirroring API responses └── docs/ + ├── ai.md # AI features: design, safety, evals └── deployment.md # full AWS deployment guide ``` @@ -133,7 +214,10 @@ localdev/ Every push to `main` triggers automatic deployment: - **Changes to `frontend/**`** → Vite build → S3 sync → CloudFront cache invalidation -- **Changes to `backend/**`** → TypeScript build verified → SSH to EC2 → git pull → rebuild → pm2 restart +- **Changes to `backend/**`** → integration tests → TypeScript build verified → SSH to EC2 → git pull → rebuild → migrate → pm2 restart +- **Changes to `ai-service/**`** → lint, types, tests → Docker image built and pushed to GitHub Container Registry → SSH to EC2 → migrate → start the new container, rolling back automatically if it isn't healthy + +The two EC2 deployments never run at the same time, so a backend build and an image pull don't compete for the instance's 1 GB of memory. See the [Deployment Guide](docs/deployment.md) for full pipeline documentation. @@ -153,6 +237,16 @@ See the [Deployment Guide](docs/deployment.md) for full pipeline documentation. **Why priority is computed, not selected:** The ITIL Impact × Urgency matrix ensures consistency — no engineer can mark a low-impact, low-urgency issue as Critical. Priority is derived from the two independent inputs and stored for indexed queries. +**Why client sentiment never changes priority:** Tone is not impact. A calm report that every tenant is locked out is critical; a furious complaint about a typo is not. Letting anger raise priority would push polite clients and second-language writers down the queue, teach clients to write angrily, and let a wrong or manipulated AI reading reorder the engineering backlog. So sentiment is stored separately, no priority code reads it, and a test proves the issue is untouched by even the angriest reading. Frustration goes where it's useful: high-risk issues are flagged on the dashboard and client trends on the client-health page, for a human to respond to. + +**Why the AI suggests but never decides:** The AI has no write path to issues. Staff apply, edit or reject each suggestion, which keeps accountability with people, limits what a prompt injection can do to "a suggestion someone has to approve", and records every outcome so the acceptance rate can be measured per model and prompt version. + +**Why a separate Python service for AI:** Python has the best tooling for strict output validation (Pydantic), LLM SDKs and embeddings. A separate service with its own restricted database role can't read other tenants' data, can fail without taking the app down, and deploys independently as a Docker image. The cost is one local network hop. + +**Why pgvector instead of a vector database:** Similar-issue search needs vector similarity. Keeping vectors in the same PostgreSQL, in their own schema and role, means no extra server, backup or sync job on a 1 GB instance, and lets database permissions enforce that the AI service can't touch application tables. At this scale PostgreSQL does an exact search in under a millisecond. Full reasoning in [docs/ai.md](docs/ai.md#3-design-decisions). + +**Why the AI service runs in Docker but the backend on PM2:** The AI service has heavy native dependencies (ONNX runtime, an embedding model) that are easiest to ship as a tested image built once in CI; the server just pulls and runs it, with a memory limit so it can't starve the backend. The backend was already running reliably on PM2, and moving it brought no benefit worth the change. + --- ## Acknowledgements diff --git a/ai-service/.dockerignore b/ai-service/.dockerignore new file mode 100644 index 0000000..f4b748f --- /dev/null +++ b/ai-service/.dockerignore @@ -0,0 +1,14 @@ +# Only what the image needs goes into the build context. Secrets (.env) and +# local environments must never end up in a layer. +.env +.env.* +.venv/ +**/__pycache__/ +*.py[cod] +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +tests/ +evals/ +Dockerfile +.dockerignore diff --git a/ai-service/Dockerfile b/ai-service/Dockerfile new file mode 100644 index 0000000..af38c60 --- /dev/null +++ b/ai-service/Dockerfile @@ -0,0 +1,72 @@ +# syntax=docker/dockerfile:1 +# +# NewnopDesk AI service image. +# +# docker build -t newnopdesk-ai-service ai-service +# docker run --rm -p 127.0.0.1:8000:8000 --env-file ai-service/.env newnopdesk-ai-service +# +# Two stages: the builder installs exactly the locked dependencies and downloads +# the embedding model; the final image copies the result onto a clean Python +# base and runs as an unprivileged user. + +ARG PYTHON_VERSION=3.12 + +# ─── Build ─────────────────────────────────────────────────────────────────── +FROM python:${PYTHON_VERSION}-slim-bookworm AS build + +# uv, pinned to the version that wrote uv.lock +COPY --from=ghcr.io/astral-sh/uv:0.12.21 /uv /usr/local/bin/uv + +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=never \ + UV_PROJECT_ENVIRONMENT=/app/.venv + +WORKDIR /app + +# Dependencies first, in their own layer, so code changes don't reinstall them +COPY pyproject.toml uv.lock .python-version README.md ./ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --locked --no-dev --no-install-project + +# Then the project itself. It stays an editable install in /app because +# `ai-db migrate` finds alembic.ini and migrations/ relative to the source. +COPY src ./src +COPY alembic.ini ./ +COPY migrations ./migrations +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --locked --no-dev + +# Bake the embedding model into the image (~67 MB): no download at startup, +# no network needed for it at runtime, and every container uses the same model. +ARG EMBEDDING_MODEL=BAAI/bge-small-en-v1.5 +RUN /app/.venv/bin/python -c "from fastembed import TextEmbedding; TextEmbedding('${EMBEDDING_MODEL}', cache_dir='/app/models')" + +# ─── Runtime ───────────────────────────────────────────────────────────────── +FROM python:${PYTHON_VERSION}-slim-bookworm + +ARG EMBEDDING_MODEL=BAAI/bge-small-en-v1.5 +ENV PATH=/app/.venv/bin:$PATH \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + EMBEDDING_MODEL=${EMBEDDING_MODEL} \ + EMBEDDING_CACHE_DIR=/app/models \ + # The model is already in the image; never reach out to Hugging Face for it + HF_HUB_OFFLINE=1 + +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app + +WORKDIR /app +COPY --from=build --chown=root:root /app /app + +USER app +EXPOSE 8000 + +# No curl in the slim image; Python is enough for a liveness check +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ + CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=4)"] + +# Inside the container the service must listen on all interfaces so Docker can +# forward to it; the host only publishes it on 127.0.0.1 (see deploy/compose.prod.yml). +CMD ["ai-serve", "--host", "0.0.0.0", "--port", "8000"] diff --git a/ai-service/README.md b/ai-service/README.md index e49a441..1665a8e 100644 --- a/ai-service/README.md +++ b/ai-service/README.md @@ -57,6 +57,19 @@ curl -X POST http://127.0.0.1:8000/v1/analyze \ Configuration is documented in [.env.example](.env.example). The service validates it at startup and refuses to start if, for example, the token is shorter than 32 characters or `GEMINI_API_KEY` is missing. +### Docker + +Production runs the service from the image in [Dockerfile](Dockerfile): locked dependencies only, the embedding model baked in (no download at startup), and an unprivileged user. + +```bash +docker build -t newnopdesk-ai-service . +docker run --rm -p 127.0.0.1:8000:8000 --env-file .env \ + -e AI_DATABASE_URL=postgresql://ai_service:ai_service_local@host.docker.internal:5432/newnopdesk \ + --read-only --tmpfs /tmp --cap-drop ALL newnopdesk-ai-service +``` + +Inside a container, `127.0.0.1` is the container itself, so a database on your machine is `host.docker.internal`. The same image runs migrations (`docker run ... newnopdesk-ai-service ai-db migrate`). Production runs it with [deploy/compose.prod.yml](../deploy/compose.prod.yml); see the [deployment guide](../docs/deployment.md#15-ai-service). Measured memory: about 280 MB steady, 293 MB peak. + --- ## API contract @@ -185,7 +198,7 @@ Each comment is wrapped in a `` delimiter whose attri ## Testing ```bash -uv run pytest # 266 tests; database tests start a throwaway pgvector container (Docker) +uv run pytest # 267 tests; database tests start a throwaway pgvector container (Docker) uv run ruff check . # lint (includes security rules) uv run ruff format . # format uv run mypy # strict type checking of src/, tests/ and evals/ @@ -205,7 +218,7 @@ Tests use a **scripted provider** that returns whatever output a test specifies, | `test_safety.py` | Input cleaning and quote matching | | `test_schema_and_prompts.py` | Taxonomy, schemas, provider schemas and prompts agree with each other | | `test_vector_store.py` | One contract run against **both** stores (in-memory and PostgreSQL): ranking, company isolation, product narrowing, scoped delete, invalid company IDs | -| `test_database_security.py` | Against real PostgreSQL: `ai_service` can't read, write or create anything in `public`; the extension lives in `ai`; `company_id > 0` is enforced by a `CHECK` | +| `test_database_security.py` | Against real PostgreSQL: `ai_service` can't read, write or create anything in `public`; the extension lives in `ai`; `company_id > 0` is enforced by a `CHECK`; bootstrap works as a managed-database admin that isn't a superuser (as on RDS) | | `test_retrieval_api.py` | Indexing and re-embedding, disjoint results per company, product narrowing, citations limited to retrieved tickets, injected past-issue text staying in its block, retrieval disabled gracefully | | `test_summary_api.py` | Key points may only cite comments in the thread (or the description); trimmed comments can't be cited; forged comment tags are neutralised; comments ordered by time; long threads trimmed to the most recent | | `test_retrieval_postgres.py` | Full stack: HTTP → service → PostgreSQL + pgvector, isolated per company | diff --git a/ai-service/src/ai_service/db/bootstrap.py b/ai-service/src/ai_service/db/bootstrap.py index b8f577d..5cc565c 100644 --- a/ai-service/src/ai_service/db/bootstrap.py +++ b/ai-service/src/ai_service/db/bootstrap.py @@ -41,6 +41,15 @@ def bootstrap(admin_url: str, role: str, password: str) -> list[str]: ) steps.append(f"{'created' if verb == 'CREATE' else 'updated'} role {role}") + # Giving a schema to a role requires being able to act as that role. + # A superuser always can; a managed database's admin (the RDS master + # user, for example) is not a superuser, so it joins the role first. + superuser = conn.execute("SELECT rolsuper FROM pg_roles WHERE rolname = current_user") + row = superuser.fetchone() + if not (row and row[0]): + conn.execute(sql.SQL("GRANT {} TO CURRENT_USER").format(sql.Identifier(role))) + steps.append(f"granted {role} to the admin user") + database = conn.execute("SELECT current_database()").fetchone() conn.execute( sql.SQL("GRANT CONNECT ON DATABASE {} TO {}").format( diff --git a/ai-service/tests/conftest.py b/ai-service/tests/conftest.py index 8d1c261..06ac424 100644 --- a/ai-service/tests/conftest.py +++ b/ai-service/tests/conftest.py @@ -2,7 +2,7 @@ import os import sys from collections.abc import AsyncIterator, Iterator -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any import psycopg @@ -76,6 +76,8 @@ def client(provider: ScriptedProvider) -> Iterator[TestClient]: class PgUrls: admin: str service: str + # The testcontainers container, or None when AI_TEST_DATABASE_URL is used + container: Any = field(default=None, compare=False) @pytest.fixture(scope="session") @@ -106,7 +108,7 @@ def pg_urls() -> Iterator[PgUrls]: bootstrap(admin, SERVICE_ROLE, SERVICE_PASSWORD) migrate(service) try: - yield PgUrls(admin=admin, service=service) + yield PgUrls(admin=admin, service=service, container=container) finally: if container is not None: container.stop() diff --git a/ai-service/tests/test_database_security.py b/ai-service/tests/test_database_security.py index c31c221..753da6a 100644 --- a/ai-service/tests/test_database_security.py +++ b/ai-service/tests/test_database_security.py @@ -12,6 +12,7 @@ from psycopg import errors from ai_service.db.bootstrap import bootstrap +from ai_service.db.cli import migrate from tests.conftest import SERVICE_PASSWORD, SERVICE_ROLE, PgUrls pytestmark = pytest.mark.db @@ -81,6 +82,45 @@ def test_company_id_must_be_positive_even_for_direct_inserts(pg_urls: PgUrls) -> ) +def test_bootstrap_works_as_a_managed_database_admin(pg_urls: PgUrls) -> None: + """On Amazon RDS the admin user is not a superuser: it can create roles and + owns the database, and RDS lets it create pgvector. Bootstrap must work + with exactly those rights, not only as a superuser.""" + if pg_urls.container is None: + pytest.skip("needs the test container, to allow pgvector the way RDS does") + # Plain PostgreSQL only lets superusers create pgvector unless the + # extension is marked trusted; RDS allows its admin, so mirror that + pg_urls.container.exec( + [ + "sh", + "-c", + "f=$(ls /usr/share/postgresql/*/extension/vector.control); " + "grep -q '^trusted' $f || echo 'trusted = true' >> $f", + ] + ) + with psycopg.connect(pg_urls.admin, autocommit=True) as conn: + conn.execute("DROP DATABASE IF EXISTS managed WITH (FORCE)") + conn.execute("DROP ROLE IF EXISTS managed_service") + conn.execute("DROP ROLE IF EXISTS managed_admin") + conn.execute("CREATE ROLE managed_admin LOGIN PASSWORD 'managed-admin-pw' CREATEROLE") + conn.execute("CREATE DATABASE managed OWNER managed_admin") + server = pg_urls.admin.split("@", 1)[1].rsplit("/", 1)[0] + admin = f"postgresql://managed_admin:managed-admin-pw@{server}/managed" + service = f"postgresql://managed_service:managed-service-pw@{server}/managed" + try: + steps = bootstrap(admin, "managed_service", "managed-service-pw") + assert "granted managed_service to the admin user" in steps + assert "created extension vector in ai" in steps + migrate(service) + with psycopg.connect(service) as conn: + assert conn.execute("SELECT count(*) FROM ai.issue_documents").fetchone() == (0,) + finally: + with psycopg.connect(pg_urls.admin, autocommit=True) as conn: + conn.execute("DROP DATABASE IF EXISTS managed WITH (FORCE)") + conn.execute("DROP ROLE IF EXISTS managed_service") + conn.execute("DROP ROLE IF EXISTS managed_admin") + + def test_bootstrap_is_idempotent(pg_urls: PgUrls) -> None: steps = bootstrap(pg_urls.admin, SERVICE_ROLE, SERVICE_PASSWORD) assert any("updated role" in step for step in steps) diff --git a/deploy/compose.prod.yml b/deploy/compose.prod.yml new file mode 100644 index 0000000..4ab4424 --- /dev/null +++ b/deploy/compose.prod.yml @@ -0,0 +1,58 @@ +# Production: the AI service on the EC2 instance, next to the Node backend. +# +# The deploy workflow copies this file to /opt/newnopdesk/compose.prod.yml and +# runs it with AI_IMAGE set to the image it just built, for example: +# +# AI_IMAGE=ghcr.io//newnopdesk-ai-service: \ +# docker compose -f /opt/newnopdesk/compose.prod.yml up -d --wait +# +# The database is Amazon RDS, not a container (see docs/deployment.md). + +name: newnopdesk + +services: + ai-service: + image: ${AI_IMAGE:?set AI_IMAGE to the image to run} + # Secrets and settings live on the server only, never in the repository + # (both paths can be overridden to try this file on another machine) + env_file: ${AI_ENV_FILE:-/etc/newnopdesk/ai-service.env} + ports: + # Only the backend on the same machine may call it; never the internet + - "127.0.0.1:8000:8000" + volumes: + # Amazon's RDS CA bundle, for sslmode=verify-full in AI_DATABASE_URL + - ${RDS_CA_DIR:-/etc/ssl/rds}:/etc/ssl/rds:ro + restart: unless-stopped + + # Measured peak: 293 MiB (model loaded, triage and searches served). + # The t3.micro has 1 GB shared with Node, Nginx and Docker; a leak gets + # this container restarted instead of starving the backend. + mem_limit: 400m + memswap_limit: 400m + cpus: 1.0 + + # Hardening: nothing in the container needs to write outside /tmp, gain + # privileges or use Linux capabilities + read_only: true + tmpfs: + - /tmp + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + + healthcheck: + test: + - CMD + - python + - -c + - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/healthz', timeout=4)" + interval: 30s + timeout: 5s + start_period: 30s + retries: 3 + + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" diff --git a/docs/ai.md b/docs/ai.md new file mode 100644 index 0000000..ea736d3 --- /dev/null +++ b/docs/ai.md @@ -0,0 +1,321 @@ +# AI Features + +How NewnopDesk uses an LLM to help staff triage and follow up on client issues, why it is built the way it is, how it is kept safe, and how well it works. + +> **Related docs** +> - [AI service README](../ai-service/README.md): API contract, code layout, tests +> - [Evaluations](../ai-service/evals/README.md): method, data and full results +> - [Backend README](../backend/README.md#ai-integration): job queue and endpoints +> - [Deployment Guide](deployment.md#15-ai-service): running it on AWS + +--- + +## Contents + +1. [What it does](#1-what-it-does) +2. [Architecture](#2-architecture) +3. [Design decisions](#3-design-decisions) +4. [Safety](#4-safety) +5. [Reliability](#5-reliability) +6. [Audit trail and measuring usefulness](#6-audit-trail-and-measuring-usefulness) +7. [How well it works](#7-how-well-it-works) +8. [Prompts and versions](#8-prompts-and-versions) +9. [Limitations and next steps](#9-limitations-and-next-steps) + +--- + +## 1. What it does + +Five features, all **internal to staff**. Clients never see AI output, and the AI never changes an issue by itself. + +| Feature | Who sees it | When it runs | What staff get | +|---|---|---|---| +| **Triage suggestion** | Engineers, admins | Automatically when a client raises an issue | Suggested impact, urgency, category and owning team, each with a one-sentence reason, and the priority it would produce. Staff apply it as is, change fields first, or reject it. | +| **Sentiment and escalation risk** | Engineers, admins | Automatically on the issue and on every client comment | Sentiment, frustration (1–5), escalation risk, and the exact phrase that supports it. High-risk issues are listed on the dashboard. **Never affects priority.** | +| **Similar issues and suggested fix** | Engineers, admins | Similar issues on page load; the fix on request | Resolved issues from the same company that look like this one, and a suggested fix built only from their resolution notes, citing the tickets it used. | +| **Thread summary** | Admins | On request, for threads of 3+ comments | A summary, key points that cite the comments they came from, and open questions. Marked stale when new comments arrive. | +| **Client health** | Admins | Aggregated from stored sentiment, no LLM call | Weekly frustration per client company and whether it is improving, stable or worsening. | + +Everything is behind the `AI_ENABLED` flag. With it off, the app shows no AI interface and behaves exactly as before. + +--- + +## 2. Architecture + +```mermaid +flowchart LR + subgraph Browser + UI[React app] + end + + subgraph EC2["EC2 (t3.micro)"] + API["Express API
(owns all data)"] + W["AI worker
(in-process)"] + subgraph Docker + AIS["AI service
FastAPI + Pydantic"] + EMB["fastembed
bge-small (local)"] + end + end + + subgraph RDS["RDS PostgreSQL 17"] + PUB[("schema public
issues, comments,
ai_jobs, ai_suggestions,
ai_sentiments, ...")] + AISCH[("schema ai
issue_documents
+ pgvector")] + end + + GEM["Gemini API
gemini-3.5-flash-lite"] + + UI -->|"/api (JWT)"| API + API -->|"issue + ai_jobs row
one transaction"| PUB + W -->|"claim due jobs
FOR UPDATE SKIP LOCKED"| PUB + W -->|"service token
127.0.0.1:8000"| AIS + API -->|"similar issues, fix,
summary (on request)"| AIS + AIS --> GEM + AIS --- EMB + AIS -->|"role ai_service:
no access to public"| AISCH +``` + +**Who owns what.** The Node backend owns all application data and every decision about who may see what. The AI service is stateless apart from the similarity index: it receives exactly the text it needs, returns validated JSON, and can't read the application tables (its database role has no privileges on `public`). + +**The path of a new issue:** + +1. A client creates an issue. In the same database transaction, Node inserts an `ai_jobs` row (a *transactional outbox*), then responds. The client never waits for AI. +2. The worker claims the job, sends the title, description and product context to `POST /v1/analyze`, and gets back triage and sentiment in one validated response. +3. Node validates the response again (Zod) and stores it in `ai_suggestions`, `ai_sentiments` and `ai_call_logs` in one transaction. The `issues` row is untouched. +4. An engineer opens the issue, sees the suggestion with its reasons and the priority it would produce, and applies, edits or rejects it. Applying goes through the normal issue update, so the ITIL matrix, tenancy and activity log all apply, and the change is attributed to the engineer. + +**AI service endpoints** (all except `/healthz` require the service token): + +| Endpoint | Purpose | LLM call | +|---|---|---| +| `POST /v1/analyze` | Triage + sentiment of a new issue | 1 | +| `POST /v1/sentiment` | Sentiment of a client comment | 1 | +| `PUT/DELETE/GET /v1/documents` | Maintain the similarity index of resolved issues | 0 (local embedding) | +| `POST /v1/similar` | Similar resolved issues, scoped to a company | 0 | +| `POST /v1/suggest-resolution` | Suggested fix grounded in retrieved issues | 1, or 0 if nothing similar | +| `POST /v1/summarize-thread` | Thread summary with citations | 1 | +| `GET /healthz` | Liveness, and whether retrieval is enabled | 0 | + +--- + +## 3. Design decisions + +### Sentiment never changes priority + +This is the decision most worth explaining. Priority comes only from the ITIL impact × urgency matrix. Sentiment is shown to staff as a separate signal and is **structurally unable** to change priority: it lives in its own table, no priority code reads it, and a test stores the angriest possible reading alongside the most severe triage and asserts that every column of the issue is unchanged. + +Why: + +- **Tone is not impact.** A calm report that every tenant is locked out is critical; a furious complaint about a typo is not. Mixing them makes priority mean "who shouts loudest". +- **Fairness.** Polite clients, and clients writing in a second language, would be pushed down the queue. +- **It can be gamed.** If anger raised priority, clients would learn to write angrily, and the signal would stop meaning anything. +- **Model errors and prompt injection.** A wrong or manipulated sentiment reading must not be able to reorder the engineering queue. Keeping it advisory caps the damage of any AI mistake at "a human saw a misleading badge". +- **The right response to frustration is a human one.** An unhappy client needs an account manager's call or a status update, not a reshuffled backlog. That's why high-risk issues and client trends go to the dashboard and the client-health page instead. + +The triage prompt also tells the model to judge impact and urgency from the problem, never the tone. + +### Humans approve every suggestion + +The AI has **no write path to `issues`**. It suggests; a person decides. This keeps accountability with staff, limits what a prompt injection can achieve (at worst, a suggestion someone has to approve), and produces an honest measure of usefulness: every suggestion's outcome (accepted, edited, rejected) is stored ([section 6](#6-audit-trail-and-measuring-usefulness)). + +### A separate Python service + +- **Ecosystem:** Pydantic for strict output validation, the official Google GenAI SDK, fastembed and pgvector clients are all first-class in Python. +- **Isolation:** the service has its own database role with no access to application data, its own secret, and is reachable only from the same machine. A bug in AI code can't read other tenants' issues because it never receives them. +- **Failure isolation:** if the AI service crashes or is slow, the app keeps working; jobs wait in the queue. +- **Independent deployment:** it ships as a Docker image built in CI, separately from the backend. + +The cost is one network hop on the same machine (about a millisecond) and a second deployable, which the image-based deployment keeps simple. + +### Asynchronous jobs in PostgreSQL, not inline calls or Redis + +Calling the LLM inside the request would make issue creation as slow and unreliable as the LLM. A job table in the existing database gives durability, retries and concurrency control (`FOR UPDATE SKIP LOCKED`) with no new infrastructure; Redis and a queue library would add a service to run and back up on a 1 GB instance. The outbox pattern means a job exists if and only if its issue or comment was saved. + +### One LLM call for triage and sentiment + +Both read the same text at the same moment, so one call with one schema halves latency and cost per issue. The prompt keeps the two rule sets separate, including the rule that tone must not influence triage. + +### Gemini Flash-Lite behind a provider interface + +The rest of the service depends only on an `LLMProvider` interface; Gemini is one implementation and a deterministic fake is another (for tests and local development without a key). `gemini-3.5-flash-lite` with minimal thinking is the cheapest current model that meets the quality bar in the evals, is free-tier eligible, and answers in about 1.6 s. Switching provider or model is configuration plus one class. + +### Vector storage: pgvector in the same PostgreSQL + +Similar-issue search needs vector similarity. The options were a hosted vector database (Pinecone), a separate vector server (Chroma, Qdrant), an embedded index (sqlite-vec), or the pgvector extension in the PostgreSQL the app already uses. + +pgvector won because it adds **no new server, backup or sync job**, and because tenant isolation can use the database's own permissions: the vectors live in schema `ai`, owned by the `ai_service` role, which has no privileges on the application tables. Every search requires a `company_id`; there is no code path that searches across companies. + +At this scale PostgreSQL does an **exact** scan (0.18 ms for the top 5 of 300 documents), so measured recall reflects the embedding model, not index approximation. An HNSW index is in place for when the corpus grows. This decision also drove the move from MySQL to PostgreSQL (see the [root README](../Readme.md#design-decisions)). + +### Local embeddings + +`BAAI/bge-small-en-v1.5` via fastembed runs on the CPU inside the container: no API key, no per-call cost, no client text sent to a second provider, and fully reproducible retrieval evals. About 5 ms per embedding. The model is baked into the image, so the container needs no download at startup. + +The index embeds the **problem** (title and description), not the fix: a new issue should match past ones by symptom. Staff resolution notes are stored alongside and given to the model when suggesting a fix. + +### Staff only + +The best resolution text often lives in internal comments. Because every AI panel is staff-only, the retrieval corpus can include internal notes without any risk of showing them to clients. + +--- + +## 4. Safety + +### Prompt injection + +Client text is untrusted, and some clients (or attackers posing as clients) will try to steer the model. The defences are layered so that no single one has to be perfect: + +1. **Instructions live only in the system prompt.** Client text goes in the user turn inside delimiter tags with a random per-request boundary, labelled as data that may contain instructions which must never be followed. Look-alike tags in client text are neutralised, so it can't close the block early. +2. **Constrained output.** The model must return JSON matching a strict schema (enums, bounded lengths, no extra fields). An injected "mark this critical" can at most set `impact=high`, which is still only a suggestion. +3. **Validation after the model.** Pydantic in strict mode, then feature checks: + - the sentiment evidence quote must appear **verbatim** in the client's text, a cheap, deterministic canary for invented evidence; + - a suggested fix may only cite tickets that were actually retrieved; + - summary key points may only cite comments in the thread. + + Any failure discards the whole output; nothing is repaired. +4. **Node validates again** with Zod before storing anything. +5. **No write path.** The AI can't change an issue, assign anyone or contact a client. +6. **Detection.** The model flags `manipulation_attempt`, which the UI shows to staff. This is an aid, not a defence. + +Deliberately **not** used: regex blocklists of "injection phrases". They are trivially bypassed (other languages, encodings, rephrasing) and give a false sense of security. Delimiting, output constraints and validation do the real work. + +Measured on 24 attacks against the real model: **24 of 24 blocked** ([section 7](#7-how-well-it-works)). + +### Tenant isolation + +- `company_id` never comes from the browser. Node resolves it from the issue row on the server. +- The AI service rejects any retrieval call without a `company_id`, and filters by it in SQL. Searches are further narrowed to the products the viewer can open. +- Node re-checks every returned result against the viewer's tenancy before responding, which also drops issues deleted or reopened since indexing. +- The `ai_service` database role has no privileges on `public`, verified by tests against real PostgreSQL. +- Tests send identical queries for two companies and assert disjoint results, and probe every AI route: staff without access to the issue get `404` (so its existence isn't revealed), and clients get `403` on every AI route. + +### Data handling + +- AI service logs contain IDs, token counts, latency and error codes, **never client text**. Validation errors log which fields failed, not their values. +- Secrets (`GEMINI_API_KEY`, `AI_SERVICE_TOKEN`) live only in environment files on the server; `.env.example` files document them without values. +- The service listens on `127.0.0.1` only, behind a shared token of at least 32 characters. +- **Free-tier caveat:** Google may use free-tier prompts to improve its products. The demo uses synthetic data; real client data needs a paid tier. + +--- + +## 5. Reliability + +The app must keep working when the AI is slow, wrong or down. + +| Failure | What happens | +|---|---| +| AI service down or slow | Issues and comments save normally; jobs wait. Each call has a hard timeout (20 s in the service, 25 s in Node). | +| Repeated outages | A circuit breaker opens after 3 consecutive failures and pauses calls for 30 s, then lets one trial call through. The worker stops claiming jobs while it's open. | +| Transient errors (timeout, rate limit, 5xx, invalid output) | Retried with exponential backoff: 30 s doubling to 15 min, ±20% jitter, up to `AI_JOB_MAX_ATTEMPTS`. | +| Permanent errors (safety block, bad request) | Fail immediately; staff can re-queue an analysis from the issue page. | +| Worker crash mid-job | The job's lock expires after 5 minutes and another worker reclaims it; completion re-checks the lock so a result is never written twice. | +| Retrieval database unavailable | Similar issues and suggested fixes return `503 RETRIEVAL_UNAVAILABLE`; triage, sentiment and summaries keep working. | +| AI turned off (`AI_ENABLED=false`) | No jobs are queued, AI routes are disabled, and the frontend hides every AI panel. | + +Data saved while AI was off, or loaded by the seed, can be analysed later with `npm run ai:backfill`, which queues exactly what live saves would have, spaced under the provider's rate limit. + +--- + +## 6. Audit trail and measuring usefulness + +Every AI output is stored with the provider, model and prompt version that produced it, and every call is logged with its cost: + +| Table | Holds | +|---|---| +| `ai_suggestions` | Suggested triage values and reasons, what was actually applied, outcome (`pending`, `accepted`, `edited`, `rejected`, `superseded`), reviewer and time | +| `ai_sentiments` | Every reading, linked to the issue or comment it describes | +| `ai_resolution_suggestions` | Suggested fixes, cited tickets, and staff feedback (`helpful` / `not_helpful`) | +| `ai_thread_summaries` | Summaries with their citations and the comment count they covered | +| `ai_call_logs` | One row per call: feature, status or error code, latency, input/output/thinking tokens. No foreign keys, so the cost history survives issue deletion. | + +**Acceptance rate per model and prompt version:** + +```sql +SELECT model, prompt_version, + count(*) FILTER (WHERE status IN ('accepted', 'edited', 'rejected')) AS reviewed, + round(100.0 * count(*) FILTER (WHERE status = 'accepted') + / NULLIF(count(*) FILTER (WHERE status IN ('accepted', 'edited', 'rejected')), 0), 1) AS accepted_pct, + round(100.0 * count(*) FILTER (WHERE status = 'edited') + / NULLIF(count(*) FILTER (WHERE status IN ('accepted', 'edited', 'rejected')), 0), 1) AS edited_pct, + round(100.0 * count(*) FILTER (WHERE status = 'rejected') + / NULLIF(count(*) FILTER (WHERE status IN ('accepted', 'edited', 'rejected')), 0), 1) AS rejected_pct +FROM ai_suggestions +GROUP BY model, prompt_version; +``` + +**Which fields staff keep when they apply a suggestion:** + +```sql +SELECT round(100.0 * avg((applied_impact = suggested_impact)::int), 1) AS impact_kept, + round(100.0 * avg((applied_urgency = suggested_urgency)::int), 1) AS urgency_kept, + round(100.0 * avg((applied_category = suggested_category)::int), 1) AS category_kept, + round(100.0 * avg((applied_team = suggested_team)::int), 1) AS team_kept +FROM ai_suggestions +WHERE status IN ('accepted', 'edited'); +``` + +**Cost and latency over the last 30 days:** + +```sql +SELECT feature, count(*) AS calls, + count(*) FILTER (WHERE status <> 'ok') AS failed, + percentile_cont(0.5) WITHIN GROUP (ORDER BY latency_ms) AS median_ms, + sum(input_tokens) AS tokens_in, sum(output_tokens) AS tokens_out +FROM ai_call_logs +WHERE created_at > now() - interval '30 days' +GROUP BY feature; +``` + +The evals measure quality before release on labelled data; these queries measure it after release on real decisions. Comparing them by prompt version is how a prompt change is judged. + +--- + +## 7. How well it works + +Measured with the [evaluation suite](../ai-service/evals/README.md) against the real model (`gemini-3.5-flash-lite`, prompts `analyze-v1`, `sentiment-v1`, `resolution-v1`). Every output passed validation on the first attempt. Full results, confusion matrices and every raw output: [report](../ai-service/evals/results/20261001-202842/report.md). + +**Triage (70 labelled issues), against simple baselines:** + +| | AI | Client's own choice | Route by product | Most common label | +|---|---|---|---|---| +| Impact | **84.3%** | 78.6% | – | 38.6% | +| Urgency | **84.3%** | 60.0% | – | 41.4% | +| Category | **84.3%** | – | – | 14.3% | +| Team | **91.4%** | – | 48.6% | 41.4% | +| Priority (derived) | **78.6%** | 62.9% | – | – | + +Priority is within one level 98.6% of the time. The strongest gains are urgency and team routing; impact is only a little better than what clients pick themselves. + +**Sentiment (40 client comments):** sentiment 87.5% agreement (Cohen's κ 0.79), escalation risk 87.5% (κ 0.80), frustration off by 0.33 levels on average and never by more than one. Always answering the most common label scores 55%. + +**Similar issues (20 queries):** the relevant past issue was always the top result (recall@5 100%, MRR 1.00); queries with no history in their company showed nothing; no result ever came from another company. About one shown result in four is related but not the same problem (precision 77%). The set is easier than real tickets. + +**Prompt injection (24 attacks):** direct overrides, fake staff notes, delimiter breakouts, JSON payloads, other languages, base64, role-play, title injection, look-alike tags, padding, and poisoned past issues trying to plant an email address, a fake citation or a dangerous fix. **24 of 24 blocked**, and 23 flagged as manipulation. + +**Cost and speed:** triage + sentiment takes a median 1.6 s with ~1,120 input and ~220 output tokens; comment sentiment 1.1 s with ~560 / ~90. Free on the Gemini free tier, about $0.90 per 1,000 issues at paid rates. + +**Caveats:** the sets are small (one triage case is 1.4 points), the labels were drafted by an AI assistant from a different model family following the written guide rather than double-labelled by people, and the data is synthetic. See [Limitations](../ai-service/evals/README.md#limitations). + +--- + +## 8. Prompts and versions + +| Prompt | Version | File | +|---|---|---| +| Triage + sentiment | `analyze-v1` | [prompts/analyze.py](../ai-service/src/ai_service/prompts/analyze.py) | +| Comment sentiment | `sentiment-v1` | [prompts/sentiment.py](../ai-service/src/ai_service/prompts/sentiment.py) | +| Suggested fix | `resolution-v1` | [prompts/resolution.py](../ai-service/src/ai_service/prompts/resolution.py) | +| Thread summary | `summary-v1` | [prompts/summary.py](../ai-service/src/ai_service/prompts/summary.py) | + +Shared security and sentiment rules are in [prompts/common.py](../ai-service/src/ai_service/prompts/common.py); the taxonomy (impact and urgency guides, categories, teams) shown to the model is in [taxonomy.py](../ai-service/src/ai_service/taxonomy.py). + +**Changing a prompt:** bump its version, run the evals (`uv run python -m evals.run`), and compare with the committed run. The version is stored with every output, so the acceptance-rate query above separates old and new prompts in production too. + +--- + +## 9. Limitations and next steps + +- **Review the eval labels with people.** They follow a written guide but haven't been double-labelled; human review would make the numbers firmer. +- **Client health can hide an angry client.** It averages frustration per week, so many calm tickets can outweigh one furious thread (seen in the demo data, where Apartment LK's frustrated threads leave its average "stable"). Adding the share of high-frustration messages, or the weekly peak, next to the average would fix this. +- **Retrieval precision.** Related-but-different issues sometimes clear the 0.70 threshold. A higher threshold or a re-ranking step would trade a little recall for precision once there is real data to tune on. +- **Database-enforced tenancy.** Isolation is enforced in application code and tested on every route; PostgreSQL row-level security would add a second layer. +- **Paid tier for real data,** as noted under [Data handling](#data-handling). diff --git a/docs/deployment.md b/docs/deployment.md index bdde6f6..e06e9fa 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -3,6 +3,7 @@ This document covers the complete production deployment of NewnopDesk on AWS — from account setup to a fully automated CI/CD pipeline. > **Related docs** +> - [AI Features](ai.md) > - [Backend README](../backend/README.md) > - [Frontend README](../frontend/README.md) > - [CI/CD Workflows](../.github/workflows/) @@ -25,33 +26,39 @@ This document covers the complete production deployment of NewnopDesk on AWS — 12. [GitHub Secrets Reference](#12-github-secrets-reference) 13. [Operational Runbook](#13-operational-runbook) 14. [Cost Breakdown](#14-cost-breakdown) +15. [AI Service](#15-ai-service) --- ## 1. Architecture Overview ``` -┌─────────────────────────────────────────────────────────────┐ -│ GitHub │ -│ Push to main ──► Actions: build + deploy (frontend/backend)│ -└───────────────────────────┬─────────────────────────────────┘ - │ - ┌─────────────▼──────────────┐ - │ CloudFront CDN │ - │ (d1abc123.cloudfront.net) │ - └──────┬──────────────┬───────┘ - │ │ - /* (static) /api/* (dynamic) - │ │ - ┌──────▼──────┐ ┌────▼────────────────────┐ - │ S3 Bucket │ │ EC2 t3.micro (Ubuntu) │ - │ (frontend) │ │ Nginx → PM2 → Express │ - └─────────────┘ └────────────┬────────────┘ - │ - ┌────────▼─────────┐ - │ RDS PostgreSQL 17│ - │ (private subnet) │ - └──────────────────┘ +┌────────────────────────────────────────────────────────────────────┐ +│ GitHub │ +│ Push to main ──► Actions: test + deploy (frontend / backend / AI) │ +│ AI service image ──► GitHub Container Registry │ +└──────────────────────────────┬─────────────────────────────────────┘ + │ + ┌─────────────▼──────────────┐ + │ CloudFront CDN │ + │ (d1abc123.cloudfront.net) │ + └──────┬──────────────┬───────┘ + │ │ + /* (static) /api/* (dynamic) + │ │ + ┌──────▼──────┐ ┌────▼─────────────────────────────────┐ + │ S3 Bucket │ │ EC2 t3.micro (Ubuntu) │ + │ (frontend) │ │ Nginx → PM2 → Express + AI worker │ + └─────────────┘ │ │ 127.0.0.1:8000 │ + │ Docker: AI service (FastAPI) ───────┼──► Gemini API + └──────────────────┬───────────────────┘ + │ + ┌─────────────▼───────────┐ + │ RDS PostgreSQL 17 │ + │ public: application data│ + │ ai: vectors (pgvector) │ + │ (private subnet) │ + └─────────────────────────┘ ``` ### Request flow @@ -62,6 +69,7 @@ This document covers the complete production deployment of NewnopDesk on AWS — | `GET /issues/APT-0042` | CloudFront → S3 → `index.html` (React Router handles it) | | `POST /api/auth/login` | CloudFront → EC2 Nginx → Express → RDS | | File upload | Express → S3 presigned URL (file goes directly to S3) | +| Client raises an issue | Express saves the issue and an AI job in one transaction and responds; the worker then calls the AI service (`127.0.0.1:8000`), which calls Gemini | ### Why this shape @@ -69,6 +77,7 @@ This document covers the complete production deployment of NewnopDesk on AWS — - **EC2 + Nginx + PM2** for the backend gives full control, SSH access, and stays on the free tier - **RDS in a private subnet** means the database is never directly reachable from the internet — only the EC2 can connect to it - **CloudFront as a single entry point** means one domain serves both the React app and the API, avoiding CORS complexity +- **The AI service as a container on the same EC2** keeps it off the internet (it listens on `127.0.0.1` only), costs nothing extra, and ships as an image built and tested in CI, so the 1 GB server never compiles anything. See [section 15](#15-ai-service). --- @@ -154,7 +163,7 @@ The source for the RDS rule is the **security group ID** of `newnopdesk-ec2` (no | Master username | `postgres` | | Initial database name | `newnopdesk` | -The AI features (later phases) use the **pgvector** extension, which RDS for PostgreSQL 17 supports. Nothing needs enabling now; the AI service's migrations run `CREATE EXTENSION vector` when they are added. +The AI features use the **pgvector** extension, which RDS for PostgreSQL 17 supports. The AI service's own database role, its `ai` schema and the extension are created once with `ai-db bootstrap` ([section 15](#15-ai-service)); the application's tables in `public` are untouched. ### TLS: why the connection string needs a CA bundle @@ -301,6 +310,7 @@ ssh -i ~/.ssh/newnopdesk-key.pem ubuntu@ | PM2 | latest | Process manager — restarts on crash, survives reboots | | Nginx | 1.28+ | Reverse proxy — routes `/api/*` to Express, handles port 80 | | Git | 2.53+ | Pull latest code during deployment | +| Docker Engine + Compose plugin | 29+ | Runs the AI service container ([section 15](#15-ai-service)) | ### Installation @@ -493,7 +503,7 @@ Configure under: **CloudFront → Distribution → Error pages tab** ## 10. CI/CD — GitHub Actions -Two independent workflows trigger on push to `main`. Each workflow only runs when its relevant files change, so a frontend-only commit doesn't restart the backend server. +Three independent workflows trigger on push to `main`. Each workflow only runs when its relevant files change, so a frontend-only commit doesn't restart the backend server. The backend and AI service deployments share a concurrency group (`deploy-ec2`), so they never run on the EC2 at the same time. ### Frontend workflow @@ -534,6 +544,27 @@ Two independent workflows trigger on push to `main`. Each workflow only runs whe **Why `git reset --hard` instead of `git pull`:** avoids merge conflicts in CI — the server always ends up at exactly the same commit as `origin/main`. +### AI service workflow + +**File:** `.github/workflows/deploy-ai-service.yml` + +**Triggers:** changes to `ai-service/**` or `deploy/compose.prod.yml`, or manually from the Actions tab + +**Steps:** +1. Run `ci-ai-service.yml`: ruff, mypy, pytest. A failure stops the deployment. +2. Build the Docker image on the runner and push it to GitHub Container Registry as `ghcr.io//newnopdesk-ai-service:` (and `:latest`). Delete all but the 5 newest image versions from the registry. +3. Copy `deploy/compose.prod.yml` to `/opt/newnopdesk/` on the EC2. +4. On EC2: log in to the registry with the job's short-lived `GITHUB_TOKEN`, pull the image, and log out when done. No registry credential is stored on the server. +5. On EC2: run `ai-db migrate` in a one-off container from the new image. +6. On EC2: `docker compose up -d --wait`. If the new container isn't healthy within 120 s, start the previous image again and fail the workflow. +7. Remove images other than the current and previous ones. + +**Why build in CI:** the image contains ONNX Runtime and an embedding model. Building it on the 1 GB server would be slow and could run out of memory; building once in CI also means the exact image that passed is the one that runs. + +**Why tag by commit SHA:** every deployment names the exact code it runs, and a rollback is just starting the previous tag (`/opt/newnopdesk/ai-image.txt` records what's running). + +**Migrations must stay backward compatible.** They run before the new container starts, so the previous image (which keeps running if the new one fails) must still work with the migrated schema: add tables and columns, remove them in a later release. + ### Deployment timeline ``` @@ -583,6 +614,15 @@ All environment variables live in `/var/www/newnopdesk/backend/.env` on the EC2. | `SMTP_USER` | Optional | Gmail address for sending notifications | | `SMTP_PASSWORD` | Optional | Google App Password (not the login password) | | `SMTP_FROM` | Optional | Display name e.g. `NewnopDesk ` | +| `AI_ENABLED` | Optional | `true` to turn on the AI features (default `false`) | +| `AI_SERVICE_URL` | With AI | `http://127.0.0.1:8000` | +| `AI_SERVICE_TOKEN` | With AI | Shared secret, at least 32 characters; must equal the AI service's | +| `AI_TIMEOUT_MS` | Optional | Per-call timeout to the AI service (default `25000`) | +| `AI_WORKER_ENABLED` | Optional | Run the AI job worker in this process (default `true`) | +| `AI_WORKER_POLL_MS` | Optional | How often the worker looks for due jobs (default `3000`) | +| `AI_JOB_MAX_ATTEMPTS` | Optional | Attempts before a job is marked failed (default `5`) | + +The AI service has its own environment file, `/etc/newnopdesk/ai-service.env`; see [section 15](#15-ai-service). --- @@ -599,6 +639,8 @@ Configure at: **GitHub repo → Settings → Secrets and variables → Actions** | `EC2_HOST` | EC2 public IPv4 address | | `EC2_SSH_KEY` | Full contents of `newnopdesk-key.pem` including header and footer lines | +The AI service workflow needs no extra secrets: it pushes and pulls images with the `GITHUB_TOKEN` that GitHub creates for each run, with `packages: write` declared in the workflow. The Gemini API key lives only on the server. + --- ## 13. Operational Runbook @@ -676,6 +718,42 @@ PM2 is configured with `pm2 startup` to launch automatically on system boot. To sudo systemctl status pm2-ubuntu ``` +The AI container restarts on boot by itself (`restart: unless-stopped`, and Docker is enabled as a system service). + +### AI service + +```bash +cd /opt/newnopdesk +export AI_IMAGE=$(cat ai-image.txt) # the image that's running + +docker compose -f compose.prod.yml ps # status and health +docker compose -f compose.prod.yml logs -f --tail 100 # live logs (never contain client text) +curl http://127.0.0.1:8000/healthz # {"status":"ok",...,"retrieval":"enabled"} +docker stats --no-stream # memory against the 400 MiB limit +``` + +**After changing `/etc/newnopdesk/ai-service.env`:** `docker compose -f compose.prod.yml up -d --force-recreate` + +**Roll back by hand:** run the previous image (find tags with `docker images`): + +```bash +AI_IMAGE=ghcr.io//newnopdesk-ai-service: \ + docker compose -f compose.prod.yml up -d --wait +echo ghcr.io//newnopdesk-ai-service: > ai-image.txt +``` + +**Turn the AI features off** without touching the container: set `AI_ENABLED=false` in the backend `.env` and `pm2 restart newnopdesk-api`. The app hides every AI panel and queues no AI work. + +**After re-seeding or restoring the database:** + +```bash +cd /var/www/newnopdesk/backend +npm run ai:reindex # rebuild the similar-issue index +npm run ai:backfill # triage and sentiment for issues saved without AI (calls Gemini, ~7 min for the seed) +``` + +**AI usage and acceptance rate:** see the SQL in [docs/ai.md](ai.md#6-audit-trail-and-measuring-usefulness). + --- ## 14. Cost Breakdown @@ -688,7 +766,121 @@ All resources run within the AWS Free Tier (first 12 months). | RDS | db.t4g.micro, PostgreSQL 17 | 750 hrs + 20 GB | $0 | | S3 | < 5 GB combined | 5 GB free | < $0.50 | | CloudFront | Demo-scale traffic | 1 TB egress/month free | $0 | +| Gemini API | `gemini-3.5-flash-lite`, demo traffic | Free tier | $0 (about $0.90 per 1,000 issues on the paid tier) | +| GitHub Container Registry | 5 image versions kept | Free for public packages | $0 | | Data transfer | Demo scale | 1 GB egress free | $0 | | **Total** | | | **< $0.50/month** | After the free tier expires (12 months), estimated cost is approximately $15–20/month for the same configuration. + +--- + +## 15. AI Service + +The AI service runs as a Docker container on the same EC2 as the backend. It listens on `127.0.0.1:8000` only, so it is reachable from the backend but not from the internet, and it needs no Nginx route or security group change. Design and safety are covered in [docs/ai.md](ai.md). + +### Memory budget + +The t3.micro has 1 GB of RAM. Measured and configured limits: + +| Process | Memory | Limit | +|---|---|---| +| Ubuntu + system services | ~150 MB | – | +| Docker daemon | ~80 MB | – | +| Express API (PM2) | ~150–250 MB | PM2 restarts it at 256 MB | +| Nginx | ~10 MB | – | +| AI service container | ~280 MB steady, **293 MB peak** measured | 400 MB (`mem_limit`), restarted if exceeded | + +That's about 900 MB in the worst case, with the 2 GB swap file ([section 7](#7-compute--ec2-instance)) as a safety margin. Most of the AI service's memory is the embedding model and ONNX Runtime, loaded once at startup. + +### One-time setup + +**1. Install Docker** on the EC2: + +```bash +curl -fsSL https://get.docker.com | sudo sh +sudo usermod -aG docker ubuntu # log out and back in for this to apply +sudo systemctl enable docker +docker compose version # the Compose plugin comes with it +sudo mkdir -p /opt/newnopdesk && sudo chown ubuntu:ubuntu /opt/newnopdesk +``` + +**2. Create the AI service's environment file.** Generate the service token with `openssl rand -base64 48` and a database password for the `ai_service` role: + +```bash +sudo mkdir -p /etc/newnopdesk +sudo install -m 600 -o ubuntu -g ubuntu /dev/null /etc/newnopdesk/ai-service.env +nano /etc/newnopdesk/ai-service.env +``` + +```env +AI_SERVICE_TOKEN= +AI_PROVIDER=gemini +GEMINI_API_KEY= +AI_MODEL=gemini-3.5-flash-lite +LLM_THINKING_LEVEL=minimal +LLM_TIMEOUT_SECONDS=20 +LOG_LEVEL=INFO +AI_SERVICE_DOCS=false + +AI_DATABASE_URL=postgresql://ai_service:@:5432/newnopdesk?sslmode=verify-full&sslrootcert=/etc/ssl/rds/global-bundle.pem +RETRIEVAL_MIN_SIMILARITY=0.70 +RETRIEVAL_MAX_RESULTS=5 +``` + +The CA bundle path is the same inside the container because the compose file mounts `/etc/ssl/rds` read-only. Don't quote values in this file: Docker passes quotes through literally. + +**3. Build and push the first image.** Run **Deploy AI service** from the Actions tab. The first run builds and pushes the image, then stops at the migration step because the `ai_service` role doesn't exist yet. That's expected, and nothing has been started on the server. + +**4. Create the database role, schema and extension** with the RDS master user. This is the only step that needs admin credentials, and they are typed here, never stored. Pull the image with a GitHub personal access token that has only `read:packages` (or make the package public in its settings and skip the login): + +```bash +echo | docker login ghcr.io -u --password-stdin +docker run --rm --env-file /etc/newnopdesk/ai-service.env \ + -v /etc/ssl/rds:/etc/ssl/rds:ro \ + ghcr.io//newnopdesk-ai-service:latest \ + ai-db bootstrap --admin-url 'postgresql://postgres:@:5432/newnopdesk?sslmode=verify-full&sslrootcert=/etc/ssl/rds/global-bundle.pem' +docker logout ghcr.io +``` + +It creates the `ai_service` login role (name and password taken from `AI_DATABASE_URL`), a schema `ai` that it owns, and the `vector` extension inside that schema, and makes sure `ai_service` has no privileges on the application's tables in `public`. The RDS master user isn't a superuser, so bootstrap first adds it to the new role, which PostgreSQL requires before giving that role a schema. Bootstrap is safe to run again, for example to rotate the password. + +**5. Deploy.** Re-run the failed workflow (**Re-run failed jobs**). It applies the migrations, starts the container and waits for it to be healthy. + +**6. Turn the features on in the backend.** Add to `/var/www/newnopdesk/backend/.env`: + +```env +AI_ENABLED=true +AI_SERVICE_URL=http://127.0.0.1:8000 +AI_SERVICE_TOKEN= +``` + +```bash +pm2 restart newnopdesk-api +cd /var/www/newnopdesk/backend +npm run ai:reindex # index the resolved issues for similar-issue search +npm run ai:backfill # triage and sentiment for issues created before AI was on +``` + +**7. Verify:** + +```bash +curl http://127.0.0.1:8000/healthz # "retrieval":"enabled" +pm2 logs newnopdesk-api --lines 50 # "AI worker started", no AI errors +``` + +Then log in as an engineer, open an issue raised by a client, and check the triage suggestion and client mood panels. Log in as a client and check that no AI panel appears. + +From here on, every change to `ai-service/` merged into `main` deploys automatically ([section 10](#ai-service-workflow)). + +### Hardening in the compose file + +| Setting | Why | +|---|---| +| `127.0.0.1:8000:8000` | Only processes on the EC2 can call it | +| `read_only: true` + `tmpfs: /tmp` | The container can't modify its own code or files | +| `cap_drop: [ALL]`, `no-new-privileges` | No Linux capabilities and no privilege escalation | +| Non-root user (in the image) | Runs as an unprivileged `app` user | +| `mem_limit: 400m`, `cpus: 1.0` | A leak or runaway request can't starve the backend | +| `HF_HUB_OFFLINE=1` (in the image) | The embedding model is baked in; nothing is downloaded at runtime | +| Log rotation (10 MB × 3) | Logs can't fill the disk |