security_fix.patch

diff --git a/app.py b/app.py
index aa59d09..64855b3 100644
--- a/app.py
+++ b/app.py
@@ -1,13 +1,20 @@
 """
 Comp2Close — Flask server
-Serves comp2close.html (3 sheets: Cross-Asset Comps, IC Memo Drafter,
-DD Synthesizer) with ANTHROPIC_API_KEY and OPENAI_API_KEY injected as JS
-variables. Keys never appear in source code — read from environment / .env
-at runtime.
+Serves comp2close.html and proxies all AI API calls server-side through
+/api/claude and /api/openai. ANTHROPIC_API_KEY and OPENAI_API_KEY are read
+from the environment / .env at request time and NEVER sent to the browser —
+the client only receives booleans indicating which provider is configured.
+
+(Earlier versions of this file injected the raw keys into the page's HTML as
+JS variables, which meant anyone viewing page source on a deployed URL could
+read and reuse them. Fixed: keys now stay server-side; the browser calls this
+server's /api/claude and /api/openai routes, which attach the real key.)
 """
 
 import os
-from flask import Flask
+import re
+from urllib.parse import urlparse
+from flask import Flask, request, jsonify
 
 try:
     from dotenv import load_dotenv
@@ -15,30 +22,182 @@ try:
 except ImportError:
     pass
 
+import requests
+
 app = Flask(__name__)
 
+# Known publisher name -> expected domain(s), for the "does source_name match
+# source_url's domain" sanity check. Best-effort and not exhaustive — an
+# unrecognized publisher name is not treated as an error, just unverified.
+KNOWN_PUBLISHER_DOMAINS = {
+    "cbre": ["cbre.com"], "jll": ["jll.com", "jll.co.uk"],
+    "savills": ["savills.com", "savills.co.uk"], "knight frank": ["knightfrank.com", "knightfrank.co.uk"],
+    "cushman & wakefield": ["cushmanwakefield.com"], "cushman and wakefield": ["cushmanwakefield.com"],
+    "colliers": ["colliers.com"], "avison young": ["avisonyoung.com"],
+    "christie & co": ["christie.com"], "christie and co": ["christie.com"],
+    "msci": ["msci.com"], "green street": ["greenstreet.com"], "rca": ["rcanalytics.com"],
+    "preqin": ["preqin.com"], "pitchbook": ["pitchbook.com"], "s&p": ["spglobal.com"],
+    "s&p global": ["spglobal.com"], "deloitte": ["deloitte.com"], "bain": ["bain.com"],
+    "bain & company": ["bain.com"], "ey": ["ey.com"], "debtwire": ["debtwire.com"],
+    "cliffwater": ["cliffwater.com"],
+}
+
+
+def _domain(url: str) -> str:
+    try:
+        netloc = urlparse(url).netloc.lower()
+        return re.sub(r"^www\.", "", netloc)
+    except Exception:
+        return ""
+
+
+@app.route("/api/validate-source", methods=["POST"])
+def validate_source():
+    """
+    Server-side URL check for Sheet 1 comps — checks (a) whether source_url
+    actually resolves, and (b) whether its domain plausibly matches the
+    claimed source_name. Runs server-side specifically to avoid the CORS
+    failures a browser would hit calling arbitrary third-party domains
+    directly from client-side JS.
+
+    This is a lightweight sanity check, not a guarantee of currency/accuracy —
+    it catches dead links and obviously mismatched sources (e.g. a "CBRE"
+    citation pointing at a random blog), the same category of error as the
+    Crowne Plaza Ealing stale-listing case, applied to URL-level integrity
+    rather than content freshness.
+    """
+    data = request.get_json(silent=True) or {}
+    url = (data.get("url") or "").strip()
+    source_name = (data.get("source_name") or "").strip().lower()
+
+    if not url:
+        return jsonify({"resolves": None, "reason": "no_url", "domain_match": None}), 200
+
+    try:
+        resp = requests.head(url, timeout=6, allow_redirects=True,
+                              headers={"User-Agent": "Comp2Close-SourceValidator/1.0"})
+        if resp.status_code >= 400:
+            # Some servers reject HEAD but allow GET — retry once before giving up
+            resp = requests.get(url, timeout=8, allow_redirects=True,
+                                 headers={"User-Agent": "Comp2Close-SourceValidator/1.0"}, stream=True)
+        resolves = resp.status_code < 400
+        final_url = resp.url
+        status_code = resp.status_code
+    except requests.RequestException as exc:
+        return jsonify({
+            "resolves": False, "reason": str(exc)[:120], "status_code": None,
+            "final_url": None, "domain": _domain(url), "domain_match": None,
+        }), 200
+
+    domain = _domain(final_url)
+    domain_match = None
+    for name, domains in KNOWN_PUBLISHER_DOMAINS.items():
+        if name in source_name:
+            domain_match = any(d in domain for d in domains)
+            break
+
+    return jsonify({
+        "resolves": resolves, "status_code": status_code,
+        "final_url": final_url, "domain": domain, "domain_match": domain_match,
+    }), 200
+
+
+@app.route("/api/claude", methods=["POST"])
+def proxy_claude():
+    """
+    Server-side proxy for Claude calls from comp2close.html.
+
+    Previously the browser called api.anthropic.com directly with the raw
+    ANTHROPIC_API_KEY injected into the page's HTML/JS — meaning anyone who
+    viewed page source on the deployed URL could read and reuse the key.
+    This route keeps the key server-side: the browser sends {system,
+    messages, tools}, this forwards it to Anthropic with the key attached
+    here, and returns the response. Same request/response shape as before,
+    so the client's existing tool-loop logic (handling stop_reason ==
+    'pause_turn' for multi-round web_search) needed no other changes.
+    """
+    anthropic_key = os.environ.get("ANTHROPIC_API_KEY", "")
+    if not anthropic_key:
+        return jsonify({"error": {"message": "ANTHROPIC_API_KEY not set on server"}}), 500
+
+    payload = request.get_json(silent=True) or {}
+    body = {
+        "model": "claude-sonnet-4-6",
+        "max_tokens": 20000,
+        "system": payload.get("system", ""),
+        "messages": payload.get("messages", []),
+    }
+    if payload.get("tools"):
+        body["tools"] = payload["tools"]
+
+    try:
+        resp = requests.post(
+            "https://api.anthropic.com/v1/messages",
+            json=body,
+            headers={
+                "Content-Type": "application/json",
+                "x-api-key": anthropic_key,
+                "anthropic-version": "2023-06-01",
+            },
+            timeout=120,
+        )
+        return jsonify(resp.json()), resp.status_code
+    except requests.RequestException as exc:
+        return jsonify({"error": {"message": str(exc)}}), 502
+
+
+@app.route("/api/openai", methods=["POST"])
+def proxy_openai():
+    """Server-side proxy for the OpenAI fallback path, same rationale as /api/claude."""
+    openai_key = os.environ.get("OPENAI_API_KEY", "")
+    if not openai_key:
+        return jsonify({"error": {"message": "OPENAI_API_KEY not set on server"}}), 500
+
+    payload = request.get_json(silent=True) or {}
+    body = {
+        "model": "gpt-4.1",
+        "instructions": payload.get("system", ""),
+        "input": payload.get("input", ""),
+        "max_output_tokens": 30000,
+    }
+    if payload.get("useTools"):
+        body["tools"] = [{"type": "web_search"}]
+
+    try:
+        resp = requests.post(
+            "https://api.openai.com/v1/responses",
+            json=body,
+            headers={"Content-Type": "application/json", "Authorization": f"Bearer {openai_key}"},
+            timeout=120,
+        )
+        return jsonify(resp.json()), resp.status_code
+    except requests.RequestException as exc:
+        return jsonify({"error": {"message": str(exc)}}), 502
+
 
 @app.route("/debug")
 def debug():
-    a = os.environ.get("ANTHROPIC_API_KEY", "")
-    o = os.environ.get("OPENAI_API_KEY", "")
-    return (
-        f"Anthropic: {'SET (' + a[:12] + '...)' if a else 'MISSING'}<br>"
-        f"OpenAI: {'SET (' + o[:8] + '...)' if o else 'MISSING'}"
-    )
+    a = bool(os.environ.get("ANTHROPIC_API_KEY", ""))
+    o = bool(os.environ.get("OPENAI_API_KEY", ""))
+    # Boolean only — no key fragments. A truncated key (even 8-12 chars) is
+    # still material that shouldn't be echoed on a public debug endpoint.
+    return f"Anthropic: {'SET' if a else 'MISSING'}<br>OpenAI: {'SET' if o else 'MISSING'}"
 
 
 @app.route("/")
 def index():
-    anthropic_key = os.environ.get("ANTHROPIC_API_KEY", "")
-    openai_key = os.environ.get("OPENAI_API_KEY", "")
+    anthropic_available = bool(os.environ.get("ANTHROPIC_API_KEY", ""))
+    openai_available = bool(os.environ.get("OPENAI_API_KEY", ""))
     with open("comp2close.html", "r", encoding="utf-8") as f:
         html = f.read()
 
+    # Only booleans go to the client now — no raw keys. Actual API calls route
+    # through /api/claude and /api/openai, which read the real keys from the
+    # server's own environment and never expose them to the browser.
     injection = (
         f"<script>"
-        f'const ANTHROPIC_API_KEY = "{anthropic_key}";'
-        f'const OPENAI_API_KEY = "{openai_key}";'
+        f"const ANTHROPIC_AVAILABLE = {str(anthropic_available).lower()};"
+        f"const OPENAI_AVAILABLE = {str(openai_available).lower()};"
         f"</script>"
     )
     html = html.replace("<script>", injection + "\n<script>", 1)
diff --git a/comp2close.html b/comp2close.html
index ac1c654..c46f367 100644
--- a/comp2close.html
+++ b/comp2close.html
@@ -6,6 +6,7 @@
 <title>Comp2Close — Tikehau-style research desk</title>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/xlsx/0.18.5/xlsx.full.min.js"></script>
 <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/4.4.0/chart.umd.min.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.min.js"></script>
 <link rel="preconnect" href="https://fonts.googleapis.com" />
 <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
 <link href="https://fonts.googleapis.com/css2?family=Source+Serif+4:opsz,wght@8..60,400;8..60,500;8..60,600&family=Inter:wght@400;500;600;700&family=IBM+Plex+Mono:wght@400;500&display=swap" rel="stylesheet" />
@@ -266,6 +267,11 @@
 
   .source-chip { display: inline-flex; align-items: center; gap: 5px; background: var(--ink); color: var(--canvas); border-radius: 4px; padding: 4px 10px; font-size: 11px; font-weight: 500; text-decoration: none; white-space: nowrap; }
   .source-chip:hover { background: var(--ink-2); }
+  .src-check { display: inline-flex; align-items: center; justify-content: center; width: 16px; height: 16px; margin-left: 6px; border-radius: 50%; font-size: 10px; font-weight: 700; cursor: help; vertical-align: middle; }
+  .src-check-ok { background: #e6f4ea; color: #1e7e34; }
+  .src-check-bad { background: #fdecea; color: #c0392b; }
+  .src-check-warn { background: #fff8e1; color: #a06800; }
+  .src-check-unknown { background: var(--surface-2, #eee); color: var(--ink-3); }
 
   .empty { text-align: center; padding: 5rem 2rem; background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius-lg); }
   .empty h2 { font-family: 'Source Serif 4', Georgia, serif; font-size: 24px; font-weight: 500; color: var(--ink-2); margin-bottom: 6px; }
@@ -440,7 +446,7 @@
         <label>Rent roll / income summary <span class="opt">key figures: total rent, WAULT, occupancy, top tenants</span></label>
         <textarea id="memoRentRoll" rows="4" placeholder="e.g. Total passing rent €4.2m p.a., WAULT 6.3 years, 94% occupied. Top tenants: TenantCo (32% of income, lease to 2031), ..."></textarea>
         <div class="upload-row">
-          <label class="upload-btn">📎 Upload .txt/.md<input type="file" accept=".txt,.md" style="display:none" onchange="loadFileInto(this,'memoRentRoll')"></label>
+          <label class="upload-btn">📎 Upload .txt/.md/.pdf<input type="file" accept=".txt,.md,.pdf" style="display:none" onchange="loadFileInto(this,'memoRentRoll')"></label>
         </div>
       </div>
       <div class="field full">
@@ -490,21 +496,21 @@
         <label>Leases <span class="opt">paste text, or upload .txt/.md</span></label>
         <textarea id="ddLeases" rows="6" placeholder="Paste lease clauses, tenant schedule, break options, rent review terms..."></textarea>
         <div class="upload-row">
-          <label class="upload-btn">📎 Upload .txt/.md<input type="file" accept=".txt,.md" style="display:none" onchange="loadFileInto(this,'ddLeases')"></label>
+          <label class="upload-btn">📎 Upload .txt/.md/.pdf<input type="file" accept=".txt,.md,.pdf" style="display:none" onchange="loadFileInto(this,'ddLeases')"></label>
         </div>
       </div>
       <div class="field full">
         <label>Environmental &amp; technical reports <span class="opt">paste text, or upload .txt/.md</span></label>
         <textarea id="ddEnv" rows="6" placeholder="Paste environmental survey findings, building condition survey, contamination reports..."></textarea>
         <div class="upload-row">
-          <label class="upload-btn">📎 Upload .txt/.md<input type="file" accept=".txt,.md" style="display:none" onchange="loadFileInto(this,'ddEnv')"></label>
+          <label class="upload-btn">📎 Upload .txt/.md/.pdf<input type="file" accept=".txt,.md,.pdf" style="display:none" onchange="loadFileInto(this,'ddEnv')"></label>
         </div>
       </div>
       <div class="field full">
         <label>Title &amp; legal documents <span class="opt">paste text, or upload .txt/.md</span></label>
         <textarea id="ddTitle" rows="6" placeholder="Paste title register extracts, easements, restrictive covenants, litigation disclosures..."></textarea>
         <div class="upload-row">
-          <label class="upload-btn">📎 Upload .txt/.md<input type="file" accept=".txt,.md" style="display:none" onchange="loadFileInto(this,'ddTitle')"></label>
+          <label class="upload-btn">📎 Upload .txt/.md/.pdf<input type="file" accept=".txt,.md,.pdf" style="display:none" onchange="loadFileInto(this,'ddTitle')"></label>
         </div>
       </div>
     </div>
@@ -528,8 +534,14 @@
 // SHARED INFRASTRUCTURE — tabs, API calls (used by all 3 sheets)
 // ═══════════════════════════════════════════════════════
 
-const API_KEY = typeof ANTHROPIC_API_KEY !== 'undefined' ? ANTHROPIC_API_KEY : '';
-const OPENAI_KEY = typeof OPENAI_API_KEY !== 'undefined' ? OPENAI_API_KEY : '';
+// API keys are NEVER sent to the browser — calls route through this server's
+// /api/claude and /api/openai, which attach the real keys server-side. The
+// page only knows whether each provider is *configured*, via booleans
+// injected by app.py's index() route (ANTHROPIC_AVAILABLE / OPENAI_AVAILABLE
+// globals, declared earlier in <head>) — read into differently-named local
+// consts here to avoid redeclaring the same identifier.
+const hasAnthropic = typeof ANTHROPIC_AVAILABLE !== 'undefined' ? ANTHROPIC_AVAILABLE : false;
+const hasOpenAI = typeof OPENAI_AVAILABLE !== 'undefined' ? OPENAI_AVAILABLE : false;
 
 function switchSheet(name) {
   document.querySelectorAll('.sheet').forEach(s => s.classList.remove('active'));
@@ -538,12 +550,59 @@ function switchSheet(name) {
   document.querySelector(`.tab-btn[data-sheet="${name}"]`).classList.add('active');
 }
 
+// pdf.js worker — must be set once before first use
+if (typeof pdfjsLib !== 'undefined') {
+  pdfjsLib.GlobalWorkerOptions.workerSrc = 'https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.worker.min.js';
+}
+
+/**
+ * Extracts plain text from a PDF File object using pdf.js, page by page.
+ * Scanned/image-only PDFs will return little or no text — this is text
+ * extraction, not OCR, matching the same limitation as the dd_agent.py CLI.
+ */
+async function extractPdfText(file) {
+  const buf = await file.arrayBuffer();
+  const pdf = await pdfjsLib.getDocument({ data: buf }).promise;
+  const pages = [];
+  for (let i = 1; i <= pdf.numPages; i++) {
+    const page = await pdf.getPage(i);
+    const content = await page.getTextContent();
+    const text = content.items.map(it => it.str).join(' ').trim();
+    if (text) pages.push(`--- page ${i} ---\n${text}`);
+  }
+  return pages.join('\n\n');
+}
+
 function loadFileInto(input, textareaId) {
   const file = input.files[0];
   if (!file) return;
-  const reader = new FileReader();
-  reader.onload = (e) => { document.getElementById(textareaId).value = e.target.result; };
-  reader.readAsText(file);
+  const textarea = document.getElementById(textareaId);
+  const isPdf = file.name.toLowerCase().endsWith('.pdf') || file.type === 'application/pdf';
+
+  if (!isPdf) {
+    const reader = new FileReader();
+    reader.onload = (e) => { textarea.value = e.target.result; };
+    reader.readAsText(file);
+    return;
+  }
+
+  if (typeof pdfjsLib === 'undefined') {
+    textarea.value = '⚠ PDF library failed to load — try pasting the text directly, or refresh the page.';
+    return;
+  }
+
+  const placeholder = textarea.value;
+  textarea.value = `⏳ Extracting text from ${file.name}…`;
+  textarea.disabled = true;
+  extractPdfText(file)
+    .then(text => {
+      textarea.value = text || `⚠ No extractable text found in ${file.name} (likely a scanned/image PDF — text extraction only, no OCR).`;
+    })
+    .catch(err => {
+      textarea.value = placeholder;
+      alert(`Could not read ${file.name}: ${err.message}`);
+    })
+    .finally(() => { textarea.disabled = false; });
 }
 
 /**
@@ -556,16 +615,11 @@ async function callClaude(messages, system, useTools, onProgress) {
   const tools = useTools ? [{ type: 'web_search_20250305', name: 'web_search', max_uses: 10 }] : [];
   let msgs = [...messages];
   for (let i = 0; i < 12; i++) {
-    const body = { model: 'claude-sonnet-4-6', max_tokens: 20000, system, messages: msgs };
+    const body = { system, messages: msgs };
     if (tools.length) body.tools = tools;
-    const res = await fetch('https://api.anthropic.com/v1/messages', {
+    const res = await fetch('/api/claude', {
       method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-        'x-api-key': API_KEY,
-        'anthropic-version': '2023-06-01',
-        'anthropic-dangerous-direct-browser-access': 'true'
-      },
+      headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify(body)
     });
     const data = await res.json();
@@ -581,13 +635,11 @@ async function callClaude(messages, system, useTools, onProgress) {
 }
 
 async function callOpenAI(userMsg, system, useTools) {
-  if (!OPENAI_KEY) throw new Error('Anthropic failed and no OPENAI_API_KEY is set for fallback');
-  const body = { model: 'gpt-4.1', instructions: system, input: userMsg, max_output_tokens: 30000 };
-  if (useTools) body.tools = [{ type: 'web_search' }];
-  const res = await fetch('https://api.openai.com/v1/responses', {
+  if (!hasOpenAI) throw new Error('Anthropic failed and no OPENAI_API_KEY is set on the server for fallback');
+  const res = await fetch('/api/openai', {
     method: 'POST',
-    headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${OPENAI_KEY}` },
-    body: JSON.stringify(body)
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ system, input: userMsg, useTools })
   });
   const data = await res.json();
   if (!res.ok) throw new Error(`OpenAI ${res.status}: ${data.error?.message || JSON.stringify(data)}`);
@@ -602,7 +654,7 @@ async function callAI(messages, system, userMsg, useTools = true, onProgress = n
   let err0;
   try { return await callClaude(messages, system, useTools, onProgress); }
   catch (err) { err0 = err.message; console.warn('Anthropic failed:', err0); }
-  if (!OPENAI_KEY) throw new Error(`Anthropic error: ${err0}\n\nNo OpenAI fallback key available.`);
+  if (!hasOpenAI) throw new Error(`Anthropic error: ${err0}\n\nNo OpenAI fallback configured on the server.`);
   if (onProgress) onProgress('Anthropic unavailable — retrying with OpenAI…');
   try { return await callOpenAI(userMsg, system, useTools); }
   catch (err) { throw new Error(`Both providers failed.\nAnthropic: ${err0}\nOpenAI: ${err.message}`); }
@@ -842,7 +894,9 @@ function renderTable(data) {
     tr.appendChild(tdTrend);
 
     const tdSrc = document.createElement('td');
-    if (d.source_url) tdSrc.innerHTML = `<a class="source-chip" href="${d.source_url}" target="_blank">↗ ${d.source_name||'Source'}</a>`;
+    if (d.source_url) {
+      tdSrc.innerHTML = `<a class="source-chip" href="${d.source_url}" target="_blank">↗ ${d.source_name||'Source'}</a><span class="src-check" data-url="${escapeHtml(d.source_url)}" data-name="${escapeHtml(d.source_name||'')}" title="Checking link…">⋯</span>`;
+    }
     else if (d.source_name) tdSrc.innerHTML = `<span style="font-size:11px;color:var(--ink-3)">${d.source_name}</span>`;
     else tdSrc.innerHTML = '<span style="color:var(--ink-3)">—</span>';
     tr.appendChild(tdSrc);
@@ -858,6 +912,73 @@ function renderTable(data) {
   wrap.appendChild(table);
   document.getElementById('resultsArea').innerHTML = '';
   document.getElementById('resultsArea').appendChild(wrap);
+  validateVisibleSources();
+}
+
+// ─────────────────────────────────────────────
+// SOURCE-CITATION VALIDATION (Sheet 1)
+// Checks each source_url actually resolves, and whether its domain plausibly
+// matches the claimed source_name — via a server-side endpoint (avoids the
+// CORS failures a direct browser fetch to arbitrary third-party domains would
+// hit). Results are cached by URL so re-sorting/re-rendering doesn't re-check
+// links already validated in this session.
+// ─────────────────────────────────────────────
+
+const sourceValidationCache = new Map();
+
+async function validateVisibleSources() {
+  const badges = Array.from(document.querySelectorAll('.src-check'));
+  const CONCURRENCY = 4;
+  let idx = 0;
+
+  async function worker() {
+    while (idx < badges.length) {
+      const badge = badges[idx++];
+      const url = badge.dataset.url;
+      const name = badge.dataset.name;
+      if (!url) continue;
+      try {
+        const result = sourceValidationCache.has(url)
+          ? sourceValidationCache.get(url)
+          : await (async () => {
+              const resp = await fetch('/api/validate-source', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ url, source_name: name }),
+              });
+              const r = await resp.json();
+              sourceValidationCache.set(url, r);
+              return r;
+            })();
+        applyValidationBadge(badge, result);
+      } catch (err) {
+        badge.textContent = '?';
+        badge.title = 'Validation check failed — network error';
+        badge.className = 'src-check src-check-unknown';
+      }
+    }
+  }
+  await Promise.all(Array.from({ length: CONCURRENCY }, worker));
+}
+
+function applyValidationBadge(badge, result) {
+  if (result.resolves === false) {
+    badge.textContent = '✕';
+    badge.title = `Link does not resolve${result.reason ? ' — ' + result.reason : ''} (status: ${result.status_code ?? 'n/a'})`;
+    badge.className = 'src-check src-check-bad';
+    return;
+  }
+  if (result.domain_match === false) {
+    badge.textContent = '⚠';
+    badge.title = `Link resolves, but domain (${result.domain}) doesn't match the claimed source — verify manually`;
+    badge.className = 'src-check src-check-warn';
+    return;
+  }
+  badge.textContent = '✓';
+  badge.title = result.domain_match
+    ? `Link resolves and domain matches claimed source (${result.domain})`
+    : `Link resolves (${result.domain}) — publisher not in the known-domain list, so domain match unverified`;
+  badge.className = 'src-check src-check-ok';
 }
 
 function renderSummary(data) {