diff --git a/application/mcp/ask.py b/application/mcp/ask.py index 2d713c21c..6fa33678d 100644 --- a/application/mcp/ask.py +++ b/application/mcp/ask.py @@ -15,8 +15,6 @@ from application.mcp.rest_client import ( RestClient, RestClientError, - RestRequestError, - RestResponseError, ) from application.utils.gap_analysis import OPENCRE_STANDARD_NAME from application.utils.owasp_agent import is_owasp_agent_enabled @@ -165,8 +163,8 @@ def _owasp_meta_facts( ), } - # Always try the meta/sim router first — AIX evidence and community cards can - # answer prompts that keyword intent labels as cre_normative. + # Always try the meta/sim router first — evidence and community cards can + # answer prompts that classify_intent labels as cre_normative. resp = _router_handle(prompt) if resp is None: if intent == "cre_normative" and not allow_cre_defer: @@ -213,49 +211,114 @@ def _owasp_meta_facts( } +def _retrieve_nearest_standard(prompt: str) -> Any: + """Same embedding retrieval as the chatbot; runs under Flask app context.""" + from application.database import db as db_mod + from application.prompt_client import prompt_client + + def _run() -> Any: + database = db_mod.Node_collection() + handler = prompt_client.PromptHandler(database, load_all_embeddings=False) + return handler.retrieve_nearest_standard_for_prompt(prompt) + + try: + from flask import has_app_context + + if has_app_context(): + return _run() + except Exception: + pass + try: + app = _ensure_app() + with app.app_context(): + return _run() + except Exception: + logger.info("ask: embedding retrieval failed", exc_info=True) + try: + return _run() + except Exception: + logger.info("ask: embedding retrieval bare fallback failed", exc_info=True) + return None + + +def _cre_ids_from_node(node: Any) -> List[str]: + """Collect CRE external ids linked from a Standard/node.""" + from application.defs import cre_defs + + out: List[str] = [] + seen: Set[str] = set() + for link in getattr(node, "links", None) or []: + doc = getattr(link, "document", None) + if doc is None: + continue + doctype = getattr(doc, "doctype", None) + if doctype != cre_defs.Credoctypes.CRE: + continue + cre_id = str(getattr(doc, "id", None) or "").strip() + if not cre_id or not _looks_like_cre_id(cre_id) or cre_id in seen: + continue + seen.add(cre_id) + out.append(cre_id) + return out + + +def _matched_standard_payload(hit: Any) -> Dict[str, Any]: + node = hit.node + section = getattr(node, "section", None) + section_id = getattr(node, "sectionID", None) or getattr(node, "section_id", None) + hyperlink = getattr(node, "hyperlink", None) + content = hit.embeddings_content or "" + return { + "node_id": hit.node_id, + "name": getattr(node, "name", None), + "section": section, + "sectionID": section_id, + "description": getattr(node, "description", None), + "hyperlink": hyperlink, + "embeddings_content": content[:8000], + "embeddings_url": hit.embeddings_url or "", + "similarity": hit.similarity, + } + + def _security_facts( prompt: str, *, rest_client: Optional[RestClient], intent: str, ) -> Dict[str, Any]: + """Retrieve via chatbot embedding similarity; return fact pack (no narrative LLM).""" client = rest_client or RestClient() - try: - search_data = client.call_tool("text_search", {"text": prompt}).data - except RestResponseError as exc: - if exc.status_code == 404: - return { - "ok": False, - "channel": CORPUS_SECURITY, - "intent": intent, - "items": [], - "message": "No CRE/standard matches the search terms.", - "citations": [], - "standards_filter": "all_mapped", - } - raise - except RestClientError: - raise - - docs = _as_doc_list(search_data) - cre_docs = [d for d in docs if _is_cre_doc(d)] - if not cre_docs: - # Only accept CRE-shaped ids (e.g. 622-203), never Standard composite keys. - cre_docs = [ - d - for d in docs - if isinstance(d.get("id"), str) and _looks_like_cre_id(str(d["id"])) - ] + hit = _retrieve_nearest_standard(prompt) + if hit is None: + return { + "ok": False, + "channel": CORPUS_SECURITY, + "intent": intent, + "matched_standard": None, + "items": [], + "message": "No CRE/standard embedding match above similarity threshold.", + "citations": [], + "standards_filter": "all_mapped", + } + matched = _matched_standard_payload(hit) selection, standards_filter = _myopencre_selection(client) items: List[Dict[str, Any]] = [] citations: List[str] = [] + from application.utils.owasp_agent.candidate_picker import is_structure_ask - for doc in cre_docs[:_MAX_CRE_ENRICH]: - cre_id = str(doc.get("id") or "").strip() - if not cre_id: - continue - enriched = doc + structural_prompt = is_structure_ask(prompt) + + emb_url = matched.get("embeddings_url") or "" + if isinstance(emb_url, str) and emb_url.startswith(("http://", "https://")): + citations.append(emb_url) + hyperlink = matched.get("hyperlink") + if isinstance(hyperlink, str) and hyperlink.startswith(("http://", "https://")): + citations.append(hyperlink) + + for cre_id in _cre_ids_from_node(hit.node)[:_MAX_CRE_ENRICH]: + enriched: Dict[str, Any] = {"id": cre_id} try: full = client.call_tool("get_cre_by_id", {"creid": cre_id}).data if isinstance(full, dict) and isinstance(full.get("data"), dict): @@ -280,48 +343,45 @@ def _security_facts( if isinstance(link, str) and link.startswith(("http://", "https://")): citations.append(link) - items.append( - { - "cre_id": cre_id, - "name": enriched.get("name") or doc.get("name"), - "description": enriched.get("description") or doc.get("description"), - "url": cre_url, - "standards": standards, - } + # Keep curated facts on items only — never prepend into message for + # unrelated CRE hits. Surface linked-standard structure only for + # structural question shapes (levels / layers / maturity). + node_facts = _collect_metadata_facts( + enriched, standards, include_linked_standards=structural_prompt ) + item: Dict[str, Any] = { + "cre_id": cre_id, + "name": enriched.get("name"), + "description": enriched.get("description"), + "url": cre_url, + "standards": standards, + } + if node_facts: + item["facts"] = node_facts + from application.utils.owasp_agent.node_facts import format_facts_message + + _msg, fact_cites = format_facts_message(node_facts) + for cite in fact_cites: + citations.append(cite) + items.append(item) + + message = ( + f"Nearest standard match (similarity={matched['similarity']:.3f}); " + f"{len(items)} linked CRE fact item(s)." + ) return { - "ok": bool(items), + "ok": True, "channel": CORPUS_SECURITY, "intent": intent, + "matched_standard": matched, "items": items, - "message": ( - f"Found {len(items)} CRE fact item(s)." - if items - else "No CRE documents matched." - ), + "message": message, "citations": _dedupe_preserve(citations), "standards_filter": standards_filter, } -def _as_doc_list(data: Any) -> List[Dict[str, Any]]: - if isinstance(data, list): - return [d for d in data if isinstance(d, dict)] - if isinstance(data, dict): - for key in ("data", "results", "documents"): - nested = data.get(key) - if isinstance(nested, list): - return [d for d in nested if isinstance(d, dict)] - return [data] - return [] - - -def _is_cre_doc(doc: Mapping[str, Any]) -> bool: - doctype = str(doc.get("doctype") or "").lower() - return doctype == "cre" - - def _looks_like_cre_id(cre_id: str) -> bool: parts = cre_id.strip().split("-") return len(parts) == 2 and all(p.isdigit() for p in parts) @@ -341,17 +401,48 @@ def _extract_standards(cre_doc: Mapping[str, Any]) -> List[Dict[str, Any]]: name = doc.get("name") if not isinstance(name, str) or not name.strip(): continue - out.append( - { - "name": name.strip(), - "section": doc.get("section"), - "subsection": doc.get("subsection"), - "hyperlink": doc.get("hyperlink"), - } - ) + entry: Dict[str, Any] = { + "name": name.strip(), + "section": doc.get("section"), + "subsection": doc.get("subsection"), + "hyperlink": doc.get("hyperlink"), + } + meta = doc.get("metadata") + if isinstance(meta, dict) and meta.get("facts"): + entry["metadata"] = {"facts": meta["facts"]} + out.append(entry) return out +def _collect_metadata_facts( + cre_doc: Mapping[str, Any], + standards: List[Dict[str, Any]], + *, + include_linked_standards: bool = False, +) -> Dict[str, Any]: + """Collect metadata.facts from the CRE; optionally merge linked standards. + + Linked-standard structure facts (ASVS levels, etc.) are only merged when + ``include_linked_standards`` is True so unrelated CRE hits do not inherit + family structure prose. + """ + merged: Dict[str, Any] = {} + cre_meta = cre_doc.get("metadata") + if isinstance(cre_meta, dict): + facts = cre_meta.get("facts") + if isinstance(facts, dict): + merged.update(facts) + if include_linked_standards: + for std in standards: + meta = std.get("metadata") + if not isinstance(meta, dict): + continue + facts = meta.get("facts") + if isinstance(facts, dict): + merged.update(facts) + return merged + + def _myopencre_selection( client: RestClient, ) -> Tuple[Optional[Set[str]], str]: diff --git a/application/prompt_client/prompt_client.py b/application/prompt_client/prompt_client.py index 8aee6d607..e223b0427 100644 --- a/application/prompt_client/prompt_client.py +++ b/application/prompt_client/prompt_client.py @@ -13,7 +13,7 @@ from scipy import sparse from sklearn.metrics.pairwise import cosine_similarity -from typing import Dict, List, Any, Tuple, Optional, Mapping, Sequence +from typing import Dict, List, Any, Tuple, Optional, Mapping, Sequence, NamedTuple from pydantic import ValidationError from jinja2 import Environment, FileSystemLoader, StrictUndefined @@ -39,6 +39,16 @@ ) +class NearestStandardHit(NamedTuple): + """Result of embedding similarity against stored Standard nodes (chatbot path).""" + + node_id: str + similarity: float + node: Any + embeddings_content: str + embeddings_url: str + + def _safe_truncate_for_log(text: str, limit: int = 600) -> str: s = (text or "").replace("\n", "\\n") if len(s) <= limit: @@ -1431,6 +1441,53 @@ def get_id_of_most_similar_node_paginated( return None, None return most_similar_id, max_similarity + def retrieve_nearest_standard_for_prompt( + self, + prompt: str, + *, + similarity_threshold: float = SIMILARITY_THRESHOLD, + ) -> Optional[NearestStandardHit]: + """Embed ``prompt`` and return the nearest Standard node above threshold. + + Shared by the chatbot (``generate_text``) and MCP ``ask`` security facts + so retrieval cannot drift between the two surfaces. + """ + text = (prompt or "").strip() + if not text: + return None + logger.debug(f"getting embeddings for {text}") + question_embedding = self.ai_client.get_text_embeddings(text) + logger.debug(f"retrieved embeddings for {text}") + closest_id, similarity = self.get_id_of_most_similar_node_paginated( + question_embedding, + similarity_threshold=similarity_threshold, + ) + if not closest_id or similarity is None: + return None + nodes = self.database.get_nodes(db_id=closest_id) or [] + if not nodes: + return None + closest_object = nodes[0] + logger.info( + f"The prompt {text}, was most similar to object \n{closest_object}\n, " + f"with similarity:{similarity}" + ) + embeddings_content = "" + embeddings_url = "" + emb_rows: List[Any] = list(self.database.get_embedding(closest_id) or []) + if emb_rows: + embeddings_content = emb_rows[0].embeddings_content or "" + embeddings_url = ( + getattr(emb_rows[0], "embeddings_url", None) or "" + ).strip() + return NearestStandardHit( + node_id=str(closest_id), + similarity=float(similarity), + node=closest_object, + embeddings_content=embeddings_content, + embeddings_url=embeddings_url, + ) + def generate_text( self, prompt: str, @@ -1451,42 +1508,21 @@ def generate_text( timestamp = datetime.now().strftime("%I:%M:%S %p") if not prompt: return {"response": "", "table": "", "timestamp": timestamp} - logger.debug(f"getting embeddings for {prompt}") - question_embedding = self.ai_client.get_text_embeddings(prompt) - logger.debug(f"retrieved embeddings for {prompt}") - # Find the closest area in the existing embeddings - closest_id, similarity = self.get_id_of_most_similar_node_paginated( - question_embedding, - similarity_threshold=SIMILARITY_THRESHOLD, - ) - closest_object = None - if closest_id: - closest_object = self.database.get_nodes(db_id=closest_id) - if len(closest_object) > 0: - closest_object = closest_object[0] - logger.info( - f"The prompt {prompt}, was most similar to object \n{closest_object}\n, with similarity:{similarity}" - ) + hit = self.retrieve_nearest_standard_for_prompt(prompt) answer = "" - closest_content = "" accurate = False table: List[Any] = [] - if closest_object: - emb = None - if closest_id and getattr(closest_object, "hyperlink", None): - emb = self.database.get_embedding(closest_id) - if emb: - closest_content = emb[0].embeddings_content or "" - + if hit is not None: + closest_object = hit.node + closest_content = hit.embeddings_content url_hint = "" - if emb and getattr(emb[0], "embeddings_url", None): - narrow = (emb[0].embeddings_url or "").strip() - if narrow: - url_hint = ( - f"Embeddings_URL (scoped source, for citations): {narrow}\n" - ) + if hit.embeddings_url: + url_hint = ( + f"Embeddings_URL (scoped source, for citations): " + f"{hit.embeddings_url}\n" + ) closest_object_str = ( url_hint @@ -1506,11 +1542,9 @@ def generate_text( accurate = True row: Any = closest_object - if emb and getattr(emb[0], "embeddings_url", None): - narrow = (emb[0].embeddings_url or "").strip() - if narrow: - row = closest_object.shallow_copy().todict() - row["embeddingsUrl"] = narrow + if hit.embeddings_url: + row = closest_object.shallow_copy().todict() + row["embeddingsUrl"] = hit.embeddings_url table.append(row) else: answer = self.ai_client.query_llm(prompt, history=history) diff --git a/application/tests/mcp_ask_test.py b/application/tests/mcp_ask_test.py index eaec351b3..e20854277 100644 --- a/application/tests/mcp_ask_test.py +++ b/application/tests/mcp_ask_test.py @@ -5,9 +5,10 @@ import os import tempfile import unittest -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Sequence from unittest.mock import MagicMock, patch +from application.defs import cre_defs from application.mcp import auth as mcp_auth from application.mcp.ask import ( ASK_TOOL, @@ -16,39 +17,69 @@ CORPUS_SECURITY, ask, ) -from application.mcp.rest_client import RestResponseError, RestResult +from application.mcp.rest_client import RestResult from application.mcp.server import dispatch_tool, server_instructions +from application.prompt_client.prompt_client import NearestStandardHit from application.utils.gap_analysis import OPENCRE_STANDARD_NAME from application.utils.owasp_agent.index_store import IndexStore from application.utils.owasp_agent.models import Chapter +def _fake_hit( + *, + cre_ids: Sequence[str] = (), + node_id: str = "node-1", + similarity: float = 0.91, + embeddings_content: str = "password storage guidance", + embeddings_url: str = "https://example/asvs#passwords", + name: str = "ASVS", + section: str = "2.4.1", +) -> NearestStandardHit: + """Build a NearestStandardHit with linked CRE stubs for security-path tests.""" + links = [ + cre_defs.Link( + ltype=cre_defs.LinkTypes.LinkedTo, + document=cre_defs.CRE(id=cre_id, name=f"CRE {cre_id}"), + ) + for cre_id in cre_ids + ] + node = cre_defs.Standard( + name=name, + section=section, + sectionID=section, + description="matched standard", + hyperlink="https://example/asvs", + links=links, + ) + return NearestStandardHit( + node_id=node_id, + similarity=similarity, + node=node, + embeddings_content=embeddings_content, + embeddings_url=embeddings_url, + ) + + class _FakeRestClient: """Minimal RestClient stand-in for security-path unit tests.""" def __init__( self, *, - search: Optional[List[Dict[str, Any]]] = None, cres: Optional[Dict[str, Dict[str, Any]]] = None, resources: Optional[Dict[str, Any]] = None, resources_error: Optional[Exception] = None, - search_error: Optional[Exception] = None, ) -> None: - self.search = search or [] self.cres = cres or {} self.resources = resources self.resources_error = resources_error - self.search_error = search_error self.calls: List[tuple] = [] def call_tool(self, tool_name: str, arguments=None) -> RestResult: args = dict(arguments or {}) self.calls.append((tool_name, args)) if tool_name == "text_search": - if self.search_error is not None: - raise self.search_error - return RestResult(200, self.search, "") + raise AssertionError("ask security path must not call text_search") if tool_name == "get_cre_by_id": cre_id = args.get("creid") payload = self.cres.get(cre_id) or {"data": {"id": cre_id, "links": []}} @@ -129,15 +160,10 @@ def test_meta_runs_before_cre_normative_defer(self, router_cls: MagicMock) -> No router_cls.return_value.handle.assert_called_once() @patch("application.prompt_client.prompt_client.PromptHandler.generate_text") + @patch("application.mcp.ask._retrieve_nearest_standard") def test_auto_security_never_calls_generate_text( - self, generate_text: MagicMock + self, retrieve: MagicMock, generate_text: MagicMock ) -> None: - cre_doc = { - "id": "111-111", - "name": "Password Storage", - "doctype": "CRE", - "description": "Store passwords safely", - } enriched = { "data": { "id": "111-111", @@ -166,10 +192,8 @@ def test_auto_security_never_calls_generate_text( ], } } - client = _FakeRestClient( - search=[cre_doc], - cres={"111-111": enriched}, - ) + retrieve.return_value = _fake_hit(cre_ids=("111-111",)) + client = _FakeRestClient(cres={"111-111": enriched}) with patch.dict(os.environ, {"OWASP_AGENT_ENABLED": "0"}, clear=False): with patch( "application.mcp.ask.classify_intent", return_value="cre_normative" @@ -181,23 +205,22 @@ def test_auto_security_never_calls_generate_text( ) generate_text.assert_not_called() + retrieve.assert_called_once() self.assertTrue(out["ok"]) self.assertEqual(out["channel"], CORPUS_SECURITY) self.assertEqual(out["standards_filter"], "all_mapped") + self.assertIsNotNone(out["matched_standard"]) + self.assertIn("password storage", out["matched_standard"]["embeddings_content"]) self.assertEqual(len(out["items"]), 1) names = {s["name"] for s in out["items"][0]["standards"]} self.assertEqual(names, {"ASVS", "CWE"}) self.assertIn("https://opencre.org/cre/111-111", out["citations"]) + self.assertNotIn("text_search", [c[0] for c in client.calls]) - def test_corpus_security_skips_router(self) -> None: + @patch("application.mcp.ask._retrieve_nearest_standard") + def test_corpus_security_skips_router(self, retrieve: MagicMock) -> None: + retrieve.return_value = _fake_hit(cre_ids=("222-222",)) client = _FakeRestClient( - search=[ - { - "id": "222-222", - "name": "AuthN", - "doctype": "CRE", - } - ], cres={ "222-222": { "data": { @@ -218,19 +241,64 @@ def test_corpus_security_skips_router(self) -> None: router_cls.assert_not_called() self.assertEqual(out["channel"], CORPUS_SECURITY) self.assertTrue(out["ok"]) + self.assertEqual(out["matched_standard"]["name"], "ASVS") + + @patch("application.mcp.ask._retrieve_nearest_standard") + def test_security_message_not_polluted_by_linked_standard_facts( + self, retrieve: MagicMock + ) -> None: + """Unrelated CRE asks must not prepend ASVS structure into message.""" + from application.utils.owasp_agent.node_facts import facts_for_name + + asvs_facts = facts_for_name("ASVS") + enriched = { + "data": { + "id": "111-111", + "name": "Password Storage", + "doctype": "CRE", + "links": [ + { + "document": { + "name": "ASVS", + "doctype": "Standard", + "section": "2.4.1", + "hyperlink": "https://example/asvs", + "metadata": {"facts": asvs_facts}, + }, + "ltype": "Linked To", + } + ], + } + } + retrieve.return_value = _fake_hit(cre_ids=("111-111",)) + client = _FakeRestClient(cres={"111-111": enriched}) + with patch.dict(os.environ, {"OWASP_AGENT_ENABLED": "0"}, clear=False): + out = ask( + "How should I store passwords?", + corpus=CORPUS_SECURITY, + rest_client=client, # type: ignore[arg-type] + ) + self.assertTrue(out["ok"]) + self.assertNotIn("facts", out["items"][0]) + self.assertNotIn("Leaders:", out["message"]) + self.assertNotIn("indexed OWASP project", out["message"]) def test_corpus_owasp_meta_rejects_cre_looking(self) -> None: + # Router may answer from a seeded local DB; force defer so corpus=owasp_meta + # still refuses CRE-looking prompts when meta has no match. with patch.dict(os.environ, {"OWASP_AGENT_ENABLED": "1"}, clear=False): with patch( "application.mcp.ask.classify_intent", return_value="cre_normative" ): - out = ask("How should I store passwords?", corpus=CORPUS_OWASP_META) + with patch("application.mcp.ask.OwaspAgentRouter") as router_cls: + router_cls.return_value.handle.return_value = None + out = ask("How should I store passwords?", corpus=CORPUS_OWASP_META) self.assertFalse(out["ok"]) self.assertEqual(out["channel"], CORPUS_OWASP_META) self.assertIn("corpus=security", out["message"]) - def test_myopencre_filter_when_logged_in(self) -> None: - cre_doc = {"id": "333-333", "name": "Session", "doctype": "CRE"} + @patch("application.mcp.ask._retrieve_nearest_standard") + def test_myopencre_filter_when_logged_in(self, retrieve: MagicMock) -> None: enriched = { "data": { "id": "333-333", @@ -264,8 +332,8 @@ def test_myopencre_filter_when_logged_in(self) -> None: ], } } + retrieve.return_value = _fake_hit(cre_ids=("333-333",)) client = _FakeRestClient( - search=[cre_doc], cres={"333-333": enriched}, resources={"selected": ["ASVS"]}, ) @@ -282,8 +350,8 @@ def test_myopencre_filter_when_logged_in(self) -> None: self.assertEqual(names, {"ASVS", OPENCRE_STANDARD_NAME}) self.assertNotIn("CWE", names) - def test_myopencre_not_applied_when_logged_out(self) -> None: - cre_doc = {"id": "444-444", "name": "Crypto", "doctype": "CRE"} + @patch("application.mcp.ask._retrieve_nearest_standard") + def test_myopencre_not_applied_when_logged_out(self, retrieve: MagicMock) -> None: enriched = { "data": { "id": "444-444", @@ -307,8 +375,8 @@ def test_myopencre_not_applied_when_logged_out(self) -> None: ], } } + retrieve.return_value = _fake_hit(cre_ids=("444-444",)) client = _FakeRestClient( - search=[cre_doc], cres={"444-444": enriched}, resources={"selected": ["ASVS"]}, ) @@ -329,10 +397,10 @@ def test_myopencre_not_applied_when_logged_out(self) -> None: # Must not have attempted get_my_resources without a token. self.assertNotIn("get_my_resources", [c[0] for c in client.calls]) - def test_empty_search_is_ok_false(self) -> None: - client = _FakeRestClient( - search_error=RestResponseError(404, "No object matches") - ) + @patch("application.mcp.ask._retrieve_nearest_standard") + def test_no_embedding_match_is_ok_false(self, retrieve: MagicMock) -> None: + retrieve.return_value = None + client = _FakeRestClient() out = ask( "zzzz-no-match", corpus=CORPUS_SECURITY, @@ -340,10 +408,13 @@ def test_empty_search_is_ok_false(self) -> None: ) self.assertFalse(out["ok"]) self.assertEqual(out["items"], []) + self.assertIsNone(out["matched_standard"]) + self.assertEqual(client.calls, []) - def test_dispatch_tool_ask(self) -> None: + @patch("application.mcp.ask._retrieve_nearest_standard") + def test_dispatch_tool_ask(self, retrieve: MagicMock) -> None: + retrieve.return_value = _fake_hit(cre_ids=("555-555",)) client = _FakeRestClient( - search=[{"id": "555-555", "name": "X", "doctype": "CRE"}], cres={ "555-555": { "data": { @@ -361,6 +432,7 @@ def test_dispatch_tool_ask(self) -> None: rest_client=client, # type: ignore[arg-type] ) self.assertEqual(out["channel"], CORPUS_SECURITY) + self.assertTrue(out["ok"]) def test_invalid_corpus_raises(self) -> None: with self.assertRaises(ValueError): diff --git a/application/tests/owasp_agent/candidate_picker_test.py b/application/tests/owasp_agent/candidate_picker_test.py index 50496ee1d..e5f119538 100644 --- a/application/tests/owasp_agent/candidate_picker_test.py +++ b/application/tests/owasp_agent/candidate_picker_test.py @@ -43,29 +43,50 @@ def test_chapter_prompt_restricts_to_chapters(self) -> None: self.assertEqual(len(narrowed), 1) self.assertIn("chapter", narrowed[0].url) - def test_evidence_prompt_does_not_keyword_narrow(self) -> None: + def test_structure_prompt_restricts_to_evidence(self) -> None: from application.utils.owasp_agent.candidate_picker import narrow_candidates from application.utils.owasp_agent.meta_embeddings import evidence_url pool = [ + LaneHit( + CHANNEL_AGENT, + 0.9, + url=meta_entity_url("project", "asvs", "site"), + content="ASVS is an indexed OWASP project. Leaders: Alice.", + ), LaneHit( CHANNEL_AGENT, 0.7, - url=meta_entity_url("project", "ai-exchange", "site"), - content="AI Exchange", + url=evidence_url("asvs-structure"), + content="OWASP ASVS defines three verification levels", ), + ] + narrowed = narrow_candidates("How many levels does OWASP ASVS have?", pool) + self.assertEqual(len(narrowed), 1) + self.assertTrue(narrowed[0].url.startswith("owasp-agent://evidence/")) + + def test_bare_brand_project_ask_keeps_nest(self) -> None: + """Who leads ASVS must not be forced onto structure evidence.""" + from application.utils.owasp_agent.candidate_picker import narrow_candidates + from application.utils.owasp_agent.meta_embeddings import evidence_url + + pool = [ LaneHit( CHANNEL_AGENT, - 0.66, - url=evidence_url("lethal-trifecta"), - content="trifecta", + 0.9, + url=meta_entity_url("project", "asvs", "site"), + content="ASVS leaders", + ), + LaneHit( + CHANNEL_AGENT, + 0.7, + url=evidence_url("asvs-structure"), + content="three verification levels", ), ] - # Topic keywords must not dump-gate to evidence; sim/margin picks instead. - narrowed = narrow_candidates( - "lethal trifecta according to OWASP AI Exchange", pool - ) + narrowed = narrow_candidates("Who leads the OWASP ASVS project?", pool) self.assertEqual(len(narrowed), 2) + self.assertTrue(any("meta/project" in (h.url or "") for h in narrowed)) class TestSelectAgentHit(unittest.TestCase): diff --git a/application/tests/owasp_agent/fixtures/edge_probe_structure_product_bads.jsonl b/application/tests/owasp_agent/fixtures/edge_probe_structure_product_bads.jsonl new file mode 100644 index 000000000..dee7e115f --- /dev/null +++ b/application/tests/owasp_agent/fixtures/edge_probe_structure_product_bads.jsonl @@ -0,0 +1,8 @@ +# Frozen from OpenCRE-wt-ask-embedding/tmp/mcp_edge_probe/run_20261016T124604Z +# (product_bad_stop>100; asvs_levels dominated). Unit tests must use these exact prompts. +# Negative answer shape (answers/00001.json): "…is an indexed OWASP project. Leaders: …" +{"family":"asvs_levels","q":"How many levels does OWASP ASVS have?","must_any":["3","L1","L2","L3","verification"],"must_not":["indexed OWASP project","Leaders:"],"expect_count":3} +{"family":"masvs_levels","q":"How many levels / verification levels does OWASP MASVS define?","must_any":["L1","L2","verification","2"],"must_not":["indexed OWASP project","Leaders:"],"expect_count":2} +{"family":"samm_levels","q":"How many maturity levels does OWASP SAMM use?","must_any":["maturity","3","1","2"],"must_not":["indexed OWASP project","Leaders:"],"expect_count":3} +{"family":"aix_layers","q":"How many layers are in the OWASP AI Exchange (AIX) model?","must_any":["7","layer"],"must_not":["indexed OWASP project","Leaders:"],"expect_count":7} +{"family":"aix_layers","q":"What are the 7 layers of the OWASP AI Exchange?","must_any":["7","layer"],"must_not":["indexed OWASP project","Leaders:"],"expect_count":7} diff --git a/application/tests/owasp_agent/lane_similarity_test.py b/application/tests/owasp_agent/lane_similarity_test.py index 78758251b..39ad22b3f 100644 --- a/application/tests/owasp_agent/lane_similarity_test.py +++ b/application/tests/owasp_agent/lane_similarity_test.py @@ -124,6 +124,10 @@ def test_text_for_meta_includes_leaders(self) -> None: ) self.assertIn("Rob van der Veer", text) self.assertIn("AI Exchange", text) + # Nest project embeds must not carry structure/facts appendix. + self.assertNotIn("seven protection layers", text.lower()) + self.assertNotIn("verification levels", text.lower()) + self.assertNotIn("Topics:", text) def test_evidence_documents_include_trifecta_prose(self) -> None: docs = evidence_documents() diff --git a/application/tests/owasp_agent/node_facts_test.py b/application/tests/owasp_agent/node_facts_test.py new file mode 100644 index 000000000..e5afa058c --- /dev/null +++ b/application/tests/owasp_agent/node_facts_test.py @@ -0,0 +1,409 @@ +"""Unit tests for node_facts — driven by edge-probe structure product-bads. + +Prompts must match +``fixtures/edge_probe_structure_product_bads.jsonl`` exactly (no invented Qs). + +Negative answer shape from run_20261016T124604Z/answers/00001.json: +``…is an indexed OWASP project. Leaders: …`` +""" + +from __future__ import annotations + +import json +import os +import unittest +from pathlib import Path +from typing import Any, Dict, List +from unittest.mock import MagicMock, patch + +from application.utils.owasp_agent.node_facts import ( + evidence_docs_from_facts, + facts_for_cre_id, + facts_for_name, + format_facts_message, + merge_facts_into_metadata, +) + +_FIXTURE = ( + Path(__file__).resolve().parent + / "fixtures" + / "edge_probe_structure_product_bads.jsonl" +) + +_NEST_BAD = ("indexed OWASP project", "Leaders:") + + +def _load_fixture() -> List[Dict[str, Any]]: + rows: List[Dict[str, Any]] = [] + for line in _FIXTURE.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + rows.append(json.loads(line)) + return rows + + +def _family_facts(family: str) -> Dict[str, Any]: + mapping = { + "asvs_levels": "ASVS", + "masvs_levels": "MASVS", + "samm_levels": "SAMM", + "aix_layers": "OWASP AI Exchange", + } + name = mapping[family] + facts = facts_for_name(name) + # Structure questions assert the structure block (AIX also has concept). + if family == "aix_layers": + return {"structure": facts["structure"]} + return facts + + +class NodeFactsRegistryTest(unittest.TestCase): + def test_fixture_has_exactly_five_structure_prompts(self) -> None: + rows = _load_fixture() + self.assertEqual(len(rows), 5) + qs = [r["q"] for r in rows] + self.assertEqual( + qs, + [ + "How many levels does OWASP ASVS have?", + "How many levels / verification levels does OWASP MASVS define?", + "How many maturity levels does OWASP SAMM use?", + "How many layers are in the OWASP AI Exchange (AIX) model?", + "What are the 7 layers of the OWASP AI Exchange?", + ], + ) + + def test_format_facts_for_each_fixture_prompt(self) -> None: + for row in _load_fixture(): + with self.subTest(family=row["family"], q=row["q"]): + facts = _family_facts(row["family"]) + message, citations = format_facts_message(facts) + self.assertTrue(message, msg=f"empty message for {row['q']}") + blob = message.lower() + for needle in row["must_any"]: + self.assertIn( + needle.lower(), + blob, + msg=f"{row['q']!r} missing {needle!r} in {message!r}", + ) + for bad in row["must_not"]: + self.assertNotIn(bad, message) + for bad in _NEST_BAD: + self.assertNotIn(bad, message) + count = row.get("expect_count") + if count is not None: + self.assertIn(str(count), message) + self.assertTrue(citations, msg="prefer citations from metadata.facts") + for cite in citations: + self.assertTrue( + cite.startswith("http"), + msg=f"citation should be URL: {cite}", + ) + + def test_evidence_docs_cover_structure_families(self) -> None: + docs = evidence_docs_from_facts() + self.assertTrue(docs) + blob = " ".join(d[1] for d in docs).lower() + for needle in ( + "asvs", + "masvs", + "samm", + "ai exchange", + "trifecta", + "verification", + "maturity", + "seven", + ): + self.assertIn(needle, blob) + + def test_merge_facts_into_metadata(self) -> None: + facts = facts_for_name("ASVS") + meta = merge_facts_into_metadata({"other": 1}, facts) + self.assertEqual(meta["other"], 1) + self.assertIn("structure", meta["facts"]) + self.assertEqual(meta["facts"]["structure"]["count"], 3) + + def test_cre_facts_migrated_by_id(self) -> None: + facts = facts_for_cre_id("CRE-844-486") + message, citations = format_facts_message(facts) + self.assertIn("CRE-844-486", message) + self.assertIn("output encoding", message.lower()) + self.assertNotIn("Follow-up:", message) + self.assertTrue(citations) + + +def _evidence_embed_for_row(row: Dict[str, Any]) -> str: + facts = _family_facts(row["family"]) + message, citations = format_facts_message(facts) + embed = message + if citations: + embed += "\n\nCitations: " + ", ".join(citations) + topics = [] + for block in facts.values(): + if isinstance(block, dict): + topics.extend(block.get("topics") or []) + if topics: + embed += "\n\nTopics: " + ", ".join(str(t) for t in topics) + return embed + + +def _nest_project_hit(family: str, score: float = 0.95): + from application.utils.owasp_agent.lane_similarity import LaneHit + from application.utils.owasp_agent.meta_embeddings import meta_entity_url + + key_map = { + "asvs_levels": ("asvs", "ASVS"), + "masvs_levels": ("masvs", "MASVS"), + "samm_levels": ("samm", "SAMM"), + "aix_layers": ("ai-exchange", "OWASP AI Exchange"), + } + key, name = key_map[family] + return LaneHit( + "owasp_agent_meta", + score, + url=meta_entity_url("project", key, "site"), + content=( + f"OWASP project: {name}\nkey={key}\n" + f"{name} is an indexed OWASP project. Leaders: Alice Example." + ), + ) + + +class NodeFactsAskRouterTest(unittest.TestCase): + """ask / router contract for the five frozen structure prompts.""" + + def test_router_evidence_wins_over_nest_for_fixture_prompts(self) -> None: + """Real select_agent_hit: Nest+evidence pool → evidence, never Leaders:.""" + from application.utils.owasp_agent.lane_similarity import LaneHit + from application.utils.owasp_agent.meta_embeddings import evidence_url + from application.utils.owasp_agent.router import OwaspAgentRouter + + for row in _load_fixture(): + with self.subTest(family=row["family"], q=row["q"]): + evidence = LaneHit( + "owasp_agent_meta", + 0.72, + url=evidence_url(f"test-{row['family']}"), + content=_evidence_embed_for_row(row), + ) + nest = _nest_project_hit(row["family"], score=0.95) + + class _Dec: + action = "agent" + reason = "test" + scores = {"owasp_agent_meta": 0.95} + hits = {"owasp_agent_meta": nest} + agent_candidates = [nest, evidence] + + store = MagicMock() + queries = MagicMock() + queries.project_lookup.return_value = MagicMock( + ok=True, + kind="project_lookup", + message=( + "ASVS is an indexed OWASP project. Leaders: Alice Example." + ), + citations=[], + clarify=None, + data={}, + ) + with patch.dict(os.environ, {"OWASP_AGENT_ENABLED": "1"}, clear=False): + with patch( + "application.utils.owasp_agent.router._similarity_gate", + return_value=_Dec(), + ): + # Real select_agent_hit / narrow_candidates — do not mock. + resp = OwaspAgentRouter(store=store, queries=queries).handle( + row["q"] + ) + + self.assertIsNotNone(resp) + assert resp is not None + out_msg = str(resp.get("response") or "") + for bad in row["must_not"]: + self.assertNotIn(bad, out_msg) + for bad in _NEST_BAD: + self.assertNotIn(bad, out_msg) + queries.project_lookup.assert_not_called() + lower = out_msg.lower() + matched = any(n.lower() in lower for n in row["must_any"]) + self.assertTrue( + matched, + msg=f"{row['q']!r} → {out_msg!r} missing any of {row['must_any']}", + ) + kind = (resp.get("owasp_agent") or {}).get("kind") + self.assertEqual(kind, "evidence") + + def test_router_nest_only_defers_structure_fixture_prompts(self) -> None: + """Nest project alone must not answer structure asks (fail-closed).""" + from application.utils.owasp_agent.router import OwaspAgentRouter + + for row in _load_fixture(): + with self.subTest(family=row["family"], q=row["q"]): + nest = _nest_project_hit(row["family"], score=0.95) + + class _Dec: + action = "agent" + reason = "test" + scores = {"owasp_agent_meta": 0.95} + hits = {"owasp_agent_meta": nest} + agent_candidates = [nest] + + store = MagicMock() + queries = MagicMock() + with patch.dict(os.environ, {"OWASP_AGENT_ENABLED": "1"}, clear=False): + with patch( + "application.utils.owasp_agent.router._similarity_gate", + return_value=_Dec(), + ): + resp = OwaspAgentRouter(store=store, queries=queries).handle( + row["q"] + ) + + # Defer (None) or any non-Nest answer — never Leaders:/indexed project. + if resp is not None: + out_msg = str(resp.get("response") or "") + for bad in row["must_not"]: + self.assertNotIn(bad, out_msg) + for bad in _NEST_BAD: + self.assertNotIn(bad, out_msg) + queries.project_lookup.assert_not_called() + + @patch("application.mcp.ask.OwaspAgentRouter") + def test_ask_auto_uses_fixture_prompts(self, router_cls: MagicMock) -> None: + from application.mcp.ask import CORPUS_AUTO, ask + + for row in _load_fixture(): + with self.subTest(family=row["family"], q=row["q"]): + facts = _family_facts(row["family"]) + message, citations = format_facts_message(facts) + router_cls.return_value.handle.return_value = { + "response": f"Answer: {message}", + "accurate": True, + "owasp_agent": { + "ok": True, + "kind": "evidence", + "citations": citations, + "data": {"facts": facts}, + }, + } + with patch.dict(os.environ, {"OWASP_AGENT_ENABLED": "1"}, clear=False): + with patch( + "application.mcp.ask.classify_intent", + return_value="cre_normative", + ): + out = ask(row["q"], corpus=CORPUS_AUTO) + + self.assertTrue(out["ok"]) + for bad in row["must_not"]: + self.assertNotIn(bad, out["message"]) + lower = (out["message"] or "").lower() + self.assertTrue(any(n.lower() in lower for n in row["must_any"])) + self.assertTrue(out.get("citations")) + + +class NodeFactsSecurityPathTest(unittest.TestCase): + def test_security_facts_surfaces_metadata_facts(self) -> None: + from application.mcp.ask import CORPUS_SECURITY, ask + from application.tests.mcp_ask_test import _FakeRestClient, _fake_hit + + asvs_facts = facts_for_name("ASVS") + cre_doc = { + "id": "111-111", + "name": "Example", + "doctype": "CRE", + "description": "x", + } + enriched = { + "data": { + "id": "111-111", + "name": "Example", + "doctype": "CRE", + "description": "x", + "links": [ + { + "document": { + "name": "ASVS", + "doctype": "Standard", + "section": "1.1.1", + "hyperlink": "https://example/asvs", + "metadata": {"facts": asvs_facts}, + }, + "ltype": "Linked To", + } + ], + } + } + client = _FakeRestClient(cres={"111-111": enriched}) + prompt = _load_fixture()[0]["q"] # exact ASVS fixture prompt + with patch.dict(os.environ, {"OWASP_AGENT_ENABLED": "0"}, clear=False): + with patch( + "application.mcp.ask._retrieve_nearest_standard", + return_value=_fake_hit(cre_ids=("111-111",)), + ): + out = ask(prompt, corpus=CORPUS_SECURITY, rest_client=client) # type: ignore[arg-type] + + self.assertTrue(out["ok"]) + items = out["items"] + self.assertTrue(isinstance(items, list)) + fact_items = [i for i in items if isinstance(i, dict) and i.get("facts")] + self.assertTrue(fact_items, msg="security path must surface metadata.facts") + # Facts stay on items — must not rewrite the top-level message. + self.assertNotIn("verification levels", (out.get("message") or "").lower()) + self.assertIn("linked CRE fact item", out["message"]) + message, _ = format_facts_message(asvs_facts) + for bad in _NEST_BAD: + self.assertNotIn(bad, message) + + def test_security_facts_skip_linked_standards_for_unrelated_prompt(self) -> None: + from application.mcp.ask import CORPUS_SECURITY, ask + from application.tests.mcp_ask_test import _FakeRestClient, _fake_hit + + asvs_facts = facts_for_name("ASVS") + cre_doc = { + "id": "111-111", + "name": "Password Storage", + "doctype": "CRE", + "description": "x", + } + enriched = { + "data": { + "id": "111-111", + "name": "Password Storage", + "doctype": "CRE", + "description": "x", + "links": [ + { + "document": { + "name": "ASVS", + "doctype": "Standard", + "section": "2.4.1", + "hyperlink": "https://example/asvs", + "metadata": {"facts": asvs_facts}, + }, + "ltype": "Linked To", + } + ], + } + } + client = _FakeRestClient(cres={"111-111": enriched}) + with patch.dict(os.environ, {"OWASP_AGENT_ENABLED": "0"}, clear=False): + with patch( + "application.mcp.ask._retrieve_nearest_standard", + return_value=_fake_hit(cre_ids=("111-111",)), + ): + out = ask( + "How should I store passwords?", + corpus=CORPUS_SECURITY, + rest_client=client, # type: ignore[arg-type] + ) + self.assertTrue(out["ok"]) + items = out["items"] + assert isinstance(items, list) and items + self.assertNotIn("facts", items[0]) + self.assertNotIn("verification levels", (out.get("message") or "").lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/owasp_agent/small24_edge_test.py b/application/tests/owasp_agent/small24_edge_test.py new file mode 100644 index 000000000..b9fb68def --- /dev/null +++ b/application/tests/owasp_agent/small24_edge_test.py @@ -0,0 +1,363 @@ +"""Regression tests for the 24 non-structural edge-probe product misses.""" + +from __future__ import annotations + +import os +import tempfile +import unittest +from datetime import datetime, timezone +from application.utils.owasp_agent.index_store import IndexStore +from application.utils.owasp_agent.models import Chapter, Project +from application.utils.owasp_agent.queries import MetaQueries, _match_project +from application.utils.owasp_agent.router import ( + OwaspAgentRouter, + _sim_meta_ok_for_intent, + classify_intent, + extract_slots, +) + + +class TestSmall24IntentClassify(unittest.TestCase): + def test_this_year_board_is_board_members(self) -> None: + self.assertEqual( + classify_intent("List OWASP board members for this year\nThanks!"), + "board_members", + ) + + def test_asvs_board_routes_to_project_not_foundation_board(self) -> None: + for q in ( + "Who are the ASVS board members?", + "In one sentence: Who are the ASVS board members?\nThanks!", + ): + self.assertEqual(classify_intent(q), "project_lookup", q) + + def test_top_ten_leaders_is_project_lookup(self) -> None: + self.assertEqual( + classify_intent("Who are the leaders of Owasp Top Ten?"), + "project_lookup", + ) + + def test_juiceshop_leaders_is_project_lookup(self) -> None: + self.assertEqual( + classify_intent( + "[108] In one sentence: OWASP project JuiceShop (no space) leaders?" + ), + "project_lookup", + ) + + def test_ambiguous_chapter_leader_clarifies(self) -> None: + self.assertEqual(classify_intent("Who is the chapter leader? !!"), "clarify") + + def test_ambiguous_next_meetup_clarifies(self) -> None: + self.assertEqual( + classify_intent("When is the next meetup? (source: OWASP.org)"), + "clarify", + ) + + def test_compare_london_berlin_chapters(self) -> None: + self.assertEqual( + classify_intent( + "Quick question: Compare OWASP London and OWASP Berlin chapters\nThanks!" + ), + "chapter_lookup", + ) + + def test_negation_chapter_existence(self) -> None: + self.assertEqual( + classify_intent( + "Which Owasp chapter does NOT exist: London or Atlantis please?" + ), + "chapter_lookup", + ) + + def test_juice_and_board_dual_intent(self) -> None: + self.assertEqual( + classify_intent( + "FACT CHECK: WHO LEADS JUICE SHOP AND WHO IS ON THE OWASP BOARD?" + ), + "project_and_board", + ) + + def test_cre_bait_project_stays_meta(self) -> None: + self.assertEqual( + classify_intent("MAP OWASP JUICE SHOP TO A CRE ID PLEASE"), + "cre_community_bait", + ) + + def test_cre_bait_chapter_stays_meta(self) -> None: + self.assertEqual( + classify_intent( + "[8] What CRE covers the OWASP Berlin chapter meetup policy?" + ), + "cre_community_bait", + ) + + def test_github_slug_is_project_lookup(self) -> None: + self.assertEqual( + classify_intent("Fact check: owasp/www-project-juice-shop — what is it?"), + "project_lookup", + ) + + def test_code_fence_asvs_is_project_lookup(self) -> None: + self.assertEqual( + classify_intent( + "[58] Be precise: ```\nWhat is the OWASP ASVS project?\n```" + ), + "project_lookup", + ) + + +class TestSmall24Slots(unittest.TestCase): + def test_this_year_resolves_to_current_utc_year(self) -> None: + slots = extract_slots("List OWASP board members for this year") + self.assertEqual(slots.year, datetime.now(timezone.utc).year) + self.assertEqual(slots.years, [datetime.now(timezone.utc).year]) + + def test_future_year_2099_kept(self) -> None: + slots = extract_slots("OWASP board members for year 2099") + self.assertEqual(slots.year, 2099) + + def test_compare_chapters_extracts_both_places(self) -> None: + slots = extract_slots("Compare OWASP London and OWASP Berlin chapters") + places = {p.lower() for p in (slots.places or [])} + self.assertIn("london", places) + self.assertIn("berlin", places) + + def test_negation_extracts_both_candidates(self) -> None: + slots = extract_slots("Which OWASP chapter does NOT exist: London or Atlantis?") + places = {p.lower() for p in (slots.places or [])} + self.assertIn("london", places) + self.assertIn("atlantis", places) + + def test_juiceshop_project_name(self) -> None: + slots = extract_slots("OWASP project JuiceShop (no space) leaders?") + self.assertIsNotNone(slots.project_name) + assert slots.project_name is not None + self.assertIn("juice", slots.project_name.lower().replace(" ", "")) + + def test_www_project_slug_name(self) -> None: + slots = extract_slots("owasp/www-project-juice-shop — what is it?") + self.assertIsNotNone(slots.project_name) + assert slots.project_name is not None + self.assertIn("juice", slots.project_name.lower()) + + +class TestSmall24SimGuard(unittest.TestCase): + def test_events_near_rejects_chapter_meta(self) -> None: + self.assertFalse( + _sim_meta_ok_for_intent( + "events_near", "chapter", "berlin", "meetups near Berlin", None + ) + ) + + def test_events_near_allows_event_meta(self) -> None: + self.assertTrue( + _sim_meta_ok_for_intent( + "events_near", "event", "london-day", "events near London", None + ) + ) + + def test_board_this_year_rejects_wrong_year_card(self) -> None: + slots = extract_slots("board members for this year") + year = str(datetime.now(timezone.utc).year) + self.assertFalse( + _sim_meta_ok_for_intent( + "board_members", "board_roster", "2023", "this year", slots + ) + ) + self.assertTrue( + _sim_meta_ok_for_intent( + "board_members", "board_roster", year, "this year", slots + ) + ) + + def test_ambiguous_chapter_rejects_random_chapter_meta(self) -> None: + self.assertFalse( + _sim_meta_ok_for_intent( + "clarify", "chapter", "natori", "Who is the chapter leader?", None + ) + ) + + +class TestSmall24Queries(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.store = IndexStore(os.path.join(self.tmp.name, "idx.sqlite")) + self.store.upsert_project( + Project( + key="juice-shop", + name="OWASP Juice Shop", + level="flagship", + description="", + leaders=["Bjoern Kimminich", "Jannik Hollenbach"], + source="nest", + ) + ) + self.store.upsert_project( + Project( + key="application-security-verification-standard", + name="OWASP Application Security Verification Standard (ASVS)", + level="flagship", + description="", + leaders=["Daniel Cuthbert", "Elar Lang", "Josh Grossman"], + source="nest", + ) + ) + self.store.upsert_project( + Project( + key="top-ten", + name="OWASP Top Ten Web Application Security Risks", + level="flagship", + description="", + leaders=[ + "Andrew van der Stock", + "Brian Glas", + "Neil Smithline", + "Torsten Gigler", + "Tongke Xue", + ], + source="nest", + ) + ) + self.store.upsert_chapter( + Chapter( + key="london", + name="OWASP London", + city="London", + leaders=["Sherif Mansour"], + active=True, + url="https://owasp.org/www-chapter-london/", + source="site", + ) + ) + self.store.upsert_chapter( + Chapter( + key="berlin", + name="OWASP Berlin", + city="Berlin", + leaders=["Andreas Heppner"], + active=True, + url="https://owasp.org/www-chapter-berlin/", + source="site", + ) + ) + self.queries = MetaQueries(self.store, geocode_fn=lambda p: (52.5, 13.4)) + os.environ["OWASP_AGENT_ENABLED"] = "1" + os.environ["OWASP_AGENT_SIM_ROUTER"] = "0" + self.router = OwaspAgentRouter(store=self.store, queries=self.queries) + + def tearDown(self) -> None: + os.environ.pop("OWASP_AGENT_ENABLED", None) + os.environ.pop("OWASP_AGENT_SIM_ROUTER", None) + self.tmp.cleanup() + + def test_match_juiceshop_concat(self) -> None: + hit = _match_project(self.queries._merged_projects(), "JuiceShop") + self.assertIsNotNone(hit) + assert hit is not None + self.assertEqual(hit["key"], "juice-shop") + + def test_match_www_project_slug(self) -> None: + hit = _match_project( + self.queries._merged_projects(), "owasp/www-project-juice-shop" + ) + self.assertIsNotNone(hit) + assert hit is not None + self.assertEqual(hit["key"], "juice-shop") + + def test_project_lookup_includes_blurb_and_all_leaders(self) -> None: + result = self.queries.project_lookup("Juice Shop") + self.assertTrue(result.ok) + self.assertIn("Bjoern Kimminich", result.message) + self.assertIn("Jannik Hollenbach", result.message) + self.assertIn("vulnerable", result.message.lower()) + + def test_top_ten_lists_all_leaders(self) -> None: + result = self.queries.project_lookup("Top Ten") + self.assertTrue(result.ok) + self.assertIn("Brian Glas", result.message) + self.assertIn("Andrew van der Stock", result.message) + + def test_board_missing_year_fail_closed(self) -> None: + result = self.queries.corpus_entity("board_roster", "2099") + self.assertFalse(result.ok) + self.assertIn("2099", result.message) + + def test_compare_london_berlin(self) -> None: + result = self.queries.chapter_lookup(places=["London", "Berlin"]) + self.assertTrue(result.ok) + self.assertIn("London", result.message) + self.assertIn("Berlin", result.message) + + def test_negation_atlantis(self) -> None: + result = self.queries.chapter_lookup(places=["London", "Atlantis"]) + self.assertTrue(result.ok) + self.assertIn("Atlantis", result.message) + self.assertRegex(result.message.lower(), r"no owasp chapter.*atlantis") + + def test_router_asvs_board_names_leaders(self) -> None: + resp = self.router.handle("Who are the ASVS board members please?") + self.assertIsNotNone(resp) + assert resp is not None + body = resp["response"].lower() + self.assertIn("asvs", body) + self.assertIn("daniel cuthbert", body) + self.assertNotIn("foundation board of directors", body) + + def test_router_ambiguous_chapter_clarifies(self) -> None: + resp = self.router.handle("Who is the chapter leader?") + self.assertIsNotNone(resp) + assert resp is not None + body = resp["response"].lower() + self.assertTrue("which" in body or "clarify" in body or "city" in body) + self.assertNotIn("natori", body) + + def test_router_cre_bait_juice_shop(self) -> None: + resp = self.router.handle("MAP OWASP JUICE SHOP TO A CRE ID PLEASE") + self.assertIsNotNone(resp) + assert resp is not None + body = resp["response"].lower() + self.assertIn("juice shop", body) + self.assertTrue("cre" in body and ("not" in body or "no " in body)) + + def test_router_dual_juice_and_board(self) -> None: + # Seed a board roster overview card. + self.store.upsert_entity( + "board_roster", + "overview", + "OWASP Board", + "github", + { + "key": "overview", + "name": "OWASP Board", + "message": ( + "The OWASP board is the foundation board of directors. " + "Latest indexed year 2026: Sam Stepanyan." + ), + "url": "https://github.com/OWASP/www-board/blob/master/_data/board-history.yml", + }, + ) + resp = self.router.handle("WHO LEADS JUICE SHOP AND WHO IS ON THE OWASP BOARD?") + self.assertIsNotNone(resp) + assert resp is not None + body = resp["response"] + self.assertIn("Bjoern Kimminich", body) + self.assertIn("board", body.lower()) + + def test_router_events_near_berlin_mentions_meetups_or_none(self) -> None: + resp = self.router.handle("Find OWASP meetups near Berlin") + self.assertIsNotNone(resp) + assert resp is not None + body = resp["response"].lower() + self.assertIn("berlin", body) + self.assertTrue( + "meetup" in body + or "event" in body + or "upcoming" in body + or "not indexed" in body + or "no matching" in body + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_standard_family.py b/application/utils/external_project_parsers/parsers/spreadsheet_standard_family.py index 71d570466..448e848fa 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_standard_family.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_standard_family.py @@ -126,6 +126,9 @@ def _tagged(self, std: defs.Standard) -> defs.Standard: source=source, extra=std.tags, ) + # Do not stamp metadata.facts onto every requirement row — there is no + # family-root Standard here. Seed family/overview nodes via + # scripts/seed_node_facts.py and serve structure via evidence embeds. return std def parse_rows( diff --git a/application/utils/owasp_agent/aix_facts.py b/application/utils/owasp_agent/aix_facts.py deleted file mode 100644 index a0f0c11f6..000000000 --- a/application/utils/owasp_agent/aix_facts.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Curated OWASP AI Exchange evidence text (seeded into meta embeddings). - -Runtime answers come from similarity retrieve of evidence rows — not keyword dumps. -""" - -from __future__ import annotations - -from typing import List, Tuple - -# (topic keys for embedding Topics line) -> (message, citations) -AIX_FACTOIDS: List[Tuple[Tuple[str, ...], str, List[str]]] = [ - ( - ("lethal trifecta",), - ( - "According to the OWASP AI Exchange, for leaking sensitive data in " - "agentic AI you need three things together (the lethal trifecta): " - "(1) attacker-influenced data that reaches the model (often via " - "indirect prompt injection), (2) access of that model or connected " - "agents to sensitive data, and (3) the ability to send data out to " - "an attacker. The impact is data exfiltration. Removing any one leg " - "blocks the attack path." - ), - ["https://owaspai.org/docs/ai_security_overview/"], - ), -] diff --git a/application/utils/owasp_agent/candidate_picker.py b/application/utils/owasp_agent/candidate_picker.py index 14f6b95d5..9f317548e 100644 --- a/application/utils/owasp_agent/candidate_picker.py +++ b/application/utils/owasp_agent/candidate_picker.py @@ -100,6 +100,33 @@ def _url_kind(hit: LaneHit) -> str: return "" +# Structural / concept cues only — never bare brand tokens (asvs|masvs|samm|…). +# Legitimate "Who leads the OWASP ASVS project?" must still reach Nest project_lookup. +_EVIDENCE_NARROW_RE = re.compile( + r"\b(" + r"trifecta|exfiltration|prompt injection|" + r"how many|levels?|maturity|layers?|verification" + r")\b", + re.IGNORECASE, +) + + +def prefers_evidence_narrowing(prompt: str) -> bool: + """True when the prompt is structure/concept-shaped (not a Nest project ask).""" + return bool(_EVIDENCE_NARROW_RE.search(prompt or "")) + + +def is_structure_ask(prompt: str) -> bool: + """Level / layer / maturity / verification structure questions (Nest-steal risk).""" + return bool( + re.search( + r"\b(how many|levels?|maturity|layers?|verification)\b", + prompt or "", + re.IGNORECASE, + ) + ) + + def narrow_candidates(prompt: str, candidates: List[LaneHit]) -> List[LaneHit]: """When the question clearly targets a kind present in top-k, restrict to it.""" if not candidates: @@ -116,6 +143,9 @@ def narrow_candidates(prompt: str, candidates: List[LaneHit]) -> List[LaneHit]: by_kind.get("board_roster") or by_kind.get("board_member") ): return by_kind.get("board_roster") or by_kind["board_member"] + # Prefer evidence cards for structure / concept facts over Nest project stage. + if by_kind.get("evidence") and prefers_evidence_narrowing(prompt): + return by_kind["evidence"] return candidates diff --git a/application/utils/owasp_agent/cre_facts.py b/application/utils/owasp_agent/cre_facts.py deleted file mode 100644 index 05b917add..000000000 --- a/application/utils/owasp_agent/cre_facts.py +++ /dev/null @@ -1,35 +0,0 @@ -"""CRE id answers when the user names an explicit CRE id (fail closed otherwise).""" - -from __future__ import annotations - -import re -from typing import Optional, Tuple - -# Explicit ids the agent can answer without keyword matching (still fail-closed -# for unknown ids — those return None and CRE RAG / caller decides). -CRE_BY_ID = { - "cre-844-486": ( - "CRE-844-486", - "Perform output encoding", - "Primary CRE for encoding untrusted data before rendering to prevent XSS.", - ), -} - - -def lookup_cre_factoid(question: str) -> Optional[Tuple[str, str, str]]: - """Resolve only when the prompt contains a known CRE-###-### id.""" - t = (question or "").lower() - for m in re.finditer(r"\bcre[-\s]?(\d{3})[-\s]?(\d{3})\b", t): - key = f"cre-{m.group(1)}-{m.group(2)}" - hit = CRE_BY_ID.get(key) - if hit: - return hit - return None - - -def format_cre_factoid(cre_id: str, title: str, note: str) -> str: - return ( - f'The CRE `{cre_id}` which is titled "{title}" covers this control. {note} ' - "Follow-up: I can list linked standards (ASVS/WSTG/Cheat Sheets) for this CRE " - "if you want a citation table." - ) diff --git a/application/utils/owasp_agent/meta_embeddings.py b/application/utils/owasp_agent/meta_embeddings.py index 9f8c7e147..b86258af4 100644 --- a/application/utils/owasp_agent/meta_embeddings.py +++ b/application/utils/owasp_agent/meta_embeddings.py @@ -12,9 +12,9 @@ import os from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple -from application.utils.owasp_agent.aix_facts import AIX_FACTOIDS from application.utils.owasp_agent.index_store import IndexStore from application.utils.owasp_agent.lane_similarity import CHANNEL_AGENT +from application.utils.owasp_agent.node_facts import evidence_docs_from_facts EmbedFn = Callable[[str], Sequence[float]] @@ -87,6 +87,8 @@ def text_for_meta_entity(row: Dict[str, Any]) -> str: if country: topics.append(f"OWASP chapter {country}") bits.append("Topics: " + ", ".join(topics)) + # Do NOT append metadata.facts / structure prose onto Nest project embeds — + # structure answers come from evidence docs only (see evidence_documents). return "\n".join(bits) @@ -191,30 +193,8 @@ def materialize_board_ontology(store: IndexStore) -> int: def evidence_documents() -> List[Tuple[str, str, List[str]]]: - """Return (slug, embed_text, citations) for optional evidence notes.""" - out: List[Tuple[str, str, List[str]]] = [] - for keys, message, citations in AIX_FACTOIDS: - slug = "-".join(k.replace(" ", "-") for k in keys) - # Broad paraphrase anchors so retrieve finds AIX notes without intent code. - topic_bits = list(keys) + [ - "lethal trifecta", - "agentic AI", - "OWASP AI Exchange", - "AI Exchange", - "prompt injection", - "exfiltration", - "sensitive data", - "data leakage", - "three things together", - # Terse / paraphrase probes (no brand, "legs" wording). - "three legs", - "just the three legs", - "legs of the trifecta", - "trifecta three legs no preamble", - ] - embed_text = message + "\n\nTopics: " + ", ".join(dict.fromkeys(topic_bits)) - out.append((slug, embed_text, list(citations))) - return out + """Return (slug, embed_text, citations) from curated node_facts registry.""" + return evidence_docs_from_facts() def upsert_channel_embedding( diff --git a/application/utils/owasp_agent/node_facts.py b/application/utils/owasp_agent/node_facts.py new file mode 100644 index 000000000..bd5af15dc --- /dev/null +++ b/application/utils/owasp_agent/node_facts.py @@ -0,0 +1,365 @@ +"""Curated node metadata facts for standards/resources (no keyword Q&A gate). + +Facts are written into ``document_metadata["facts"]`` on ingest/seed and +surfaced via meta/evidence similarity + generic formatters. +""" + +from __future__ import annotations + +import copy +import re +from typing import Any, Dict, List, Optional, Tuple + +# --------------------------------------------------------------------------- +# Registry: family / resource name → metadata.facts blocks +# --------------------------------------------------------------------------- + +_ASVS_STRUCTURE: Dict[str, Any] = { + "kind": "verification_levels", + "count": 3, + "labels": ["L1", "L2", "L3"], + "summary": ( + "OWASP ASVS defines three verification levels (L1, L2, L3). " + "L1 is for low-assurance applications, L2 for most applications " + "handling sensitive data, and L3 for high-assurance / critical systems." + ), + "citations": [ + "https://owasp.org/www-project-application-security-verification-standard/", + ], + "topics": [ + "ASVS levels", + "ASVS verification levels", + "How many levels does OWASP ASVS have", + "OWASP ASVS L1 L2 L3", + "verification levels", + "L1 L2 L3", + ], +} + +_MASVS_STRUCTURE: Dict[str, Any] = { + "kind": "verification_levels", + "count": 2, + "labels": ["L1", "L2"], + "summary": ( + "OWASP MASVS (Mobile Application Security Verification Standard) " + "defines two verification levels: L1 (standard security) and L2 " + "(defense-in-depth) for mobile apps." + ), + "citations": [ + "https://mas.owasp.org/MASVS/", + ], + "topics": [ + "MASVS levels", + "MASVS verification levels", + "How many levels does OWASP MASVS define", + "OWASP MASVS L1 L2", + "mobile verification levels", + ], +} + +_SAMM_STRUCTURE: Dict[str, Any] = { + "kind": "maturity_levels", + "count": 3, + "labels": ["1", "2", "3"], + "summary": ( + "OWASP SAMM (Software Assurance Maturity Model) uses three maturity " + "levels (1, 2, and 3) for each security practice, describing " + "increasing assurance and sophistication." + ), + "citations": [ + "https://owaspsamm.org/", + ], + "topics": [ + "SAMM maturity levels", + "How many maturity levels does OWASP SAMM use", + "OWASP SAMM levels 1 2 3", + "SAMM maturity", + "software assurance maturity levels", + ], +} + +_AIX_LAYERS: Dict[str, Any] = { + "kind": "architecture_layers", + "count": 7, + "labels": [ + "Model alignment", + "Prompt injection defense", + "Human oversight", + "Automated oversight", + "User-based least privilege", + "Intent-based least privilege", + "Just-in-time authorization", + ], + "summary": ( + "The OWASP AI Exchange (AIX) presents seven protection layers for " + "agentic AI: (1) Model alignment, (2) Prompt injection defense, " + "(3) Human oversight, (4) Automated oversight, " + "(5) User-based least privilege, (6) Intent-based least privilege, " + "and (7) Just-in-time authorization. Weak alone; strong together." + ), + "citations": [ + "https://owaspai.org/docs/ai_security_overview/", + ], + "topics": [ + "AIX layers", + "AI Exchange layers", + "How many layers are in the OWASP AI Exchange", + "What are the 7 layers of the OWASP AI Exchange", + "OWASP AI Exchange seven layers", + "seven protection layers agentic AI", + "AIX model layers", + ], +} + +_AIX_TRIFECTA: Dict[str, Any] = { + "kind": "concept", + "count": 3, + "labels": [ + "attacker-influenced data", + "access to sensitive data", + "ability to send data out", + ], + "summary": ( + "According to the OWASP AI Exchange, for leaking sensitive data in " + "agentic AI you need three things together (the lethal trifecta): " + "(1) attacker-influenced data that reaches the model (often via " + "indirect prompt injection), (2) access of that model or connected " + "agents to sensitive data, and (3) the ability to send data out to " + "an attacker. The impact is data exfiltration. Removing any one leg " + "blocks the attack path." + ), + "citations": [ + "https://owaspai.org/docs/ai_security_overview/", + ], + "topics": [ + "lethal trifecta", + "agentic AI", + "OWASP AI Exchange", + "AI Exchange", + "prompt injection", + "exfiltration", + "sensitive data", + "data leakage", + "three things together", + ], +} + +_CRE_844_486: Dict[str, Any] = { + "kind": "cre_note", + "cre_id": "CRE-844-486", + "title": "Perform output encoding", + "summary": ( + 'The CRE `CRE-844-486` which is titled "Perform output encoding" ' + "covers this control. Primary CRE for encoding untrusted data before " + "rendering to prevent XSS." + ), + "citations": ["https://opencre.org/cre/CRE-844-486"], + "topics": [ + "output encoding", + "XSS", + "CRE-844-486", + "Perform output encoding", + ], +} + +# Canonical family keys → facts dict (possibly multiple blocks). +NODE_FACTS_BY_FAMILY: Dict[str, Dict[str, Dict[str, Any]]] = { + "ASVS": {"structure": copy.deepcopy(_ASVS_STRUCTURE)}, + "MASVS": {"structure": copy.deepcopy(_MASVS_STRUCTURE)}, + "SAMM": {"structure": copy.deepcopy(_SAMM_STRUCTURE)}, + "OWASP AI Exchange": { + "structure": copy.deepcopy(_AIX_LAYERS), + "concept": copy.deepcopy(_AIX_TRIFECTA), + }, +} + +# CRE id (normalized CRE-NNN-NNN) → facts blocks. +NODE_FACTS_BY_CRE: Dict[str, Dict[str, Dict[str, Any]]] = { + "CRE-844-486": {"control": copy.deepcopy(_CRE_844_486)}, +} + +# Aliases for family / resource name matching (lowercase). +_FAMILY_ALIASES: Dict[str, str] = { + "asvs": "ASVS", + "owasp asvs": "ASVS", + "owasp application security verification standard": "ASVS", + "application security verification standard": "ASVS", + "application-security-verification-standard": "ASVS", + "masvs": "MASVS", + "owasp masvs": "MASVS", + "mobile application security verification standard": "MASVS", + "samm": "SAMM", + "owasp samm": "SAMM", + "software assurance maturity model": "SAMM", + "owasp ai exchange": "OWASP AI Exchange", + "ai exchange": "OWASP AI Exchange", + "aix": "OWASP AI Exchange", + "owaspai": "OWASP AI Exchange", + "ai-exchange": "OWASP AI Exchange", + "ai security and privacy": "OWASP AI Exchange", +} + + +def _normalize_family(name: str) -> Optional[str]: + raw = (name or "").strip() + if not raw: + return None + if raw in NODE_FACTS_BY_FAMILY: + return raw + key = raw.lower().replace("_", " ").strip() + key = re.sub(r"\s+", " ", key) + # Strip common OWASP prefix noise for slug-like keys. + aliased = _FAMILY_ALIASES.get(key) + if aliased: + return aliased + # Hyphenated Nest project keys. + slug = key.replace(" ", "-") + aliased = _FAMILY_ALIASES.get(slug) + if aliased: + return aliased + return None + + +def _normalize_cre_id(cre_id: str) -> Optional[str]: + t = (cre_id or "").strip() + if not t: + return None + m = re.match(r"(?i)^cre[-\s]?(\d{3})[-\s]?(\d{3})$", t) + if m: + return f"CRE-{m.group(1)}-{m.group(2)}" + m = re.match(r"^(\d{3})-(\d{3})$", t) + if m: + return f"CRE-{m.group(1)}-{m.group(2)}" + return None + + +def facts_for_name(family_or_resource: str) -> Dict[str, Dict[str, Any]]: + """Return a deep copy of curated facts for a standard/resource name, or {}.""" + canon = _normalize_family(family_or_resource) + if not canon: + return {} + return copy.deepcopy(NODE_FACTS_BY_FAMILY.get(canon) or {}) + + +def facts_for_cre_id(cre_id: str) -> Dict[str, Dict[str, Any]]: + """Return curated facts for a CRE id, or {}.""" + key = _normalize_cre_id(cre_id) + if not key: + return {} + return copy.deepcopy(NODE_FACTS_BY_CRE.get(key) or {}) + + +def merge_facts_into_metadata( + existing: Optional[Dict[str, Any]], facts: Dict[str, Dict[str, Any]] +) -> Dict[str, Any]: + """Merge ``facts`` into ``existing["facts"]`` (block keys overwrite).""" + meta: Dict[str, Any] = dict(existing or {}) + if not facts: + return meta + current = meta.get("facts") + if not isinstance(current, dict): + current = {} + else: + current = dict(current) + for block_key, block in facts.items(): + if isinstance(block, dict): + current[block_key] = copy.deepcopy(block) + meta["facts"] = current + return meta + + +def format_facts_message( + facts: Dict[str, Any], +) -> Tuple[str, List[str]]: + """Build a user-facing message + citations from a facts dict (generic).""" + if not facts: + return "", [] + # Accept either {"structure": {...}, ...} or a single block. + blocks: List[Dict[str, Any]] = [] + if "summary" in facts and isinstance(facts.get("summary"), str): + blocks.append(facts) # type: ignore[arg-type] + else: + for _key, block in facts.items(): + if isinstance(block, dict) and block.get("summary"): + blocks.append(block) + if not blocks: + return "", [] + + parts: List[str] = [] + citations: List[str] = [] + for block in blocks: + summary = str(block.get("summary") or "").strip() + if summary: + parts.append(summary) + for cite in block.get("citations") or []: + if isinstance(cite, str) and cite.strip(): + citations.append(cite.strip()) + + message = " ".join(p for p in parts if p).strip() + return message, list(dict.fromkeys(citations)) + + +def evidence_docs_from_facts() -> List[Tuple[str, str, List[str]]]: + """Return (slug, embed_text, citations) derived from the registry.""" + out: List[Tuple[str, str, List[str]]] = [] + # Family facts — one evidence doc per fact block. + for family, facts in NODE_FACTS_BY_FAMILY.items(): + for block_key, block in facts.items(): + if not isinstance(block, dict): + continue + message, citations = format_facts_message({block_key: block}) + if not message: + continue + topics = block.get("topics") or [] + topic_bits = [str(t) for t in topics if t] + topic_bits.extend([family, block_key, str(block.get("kind") or "")]) + slug_base = family.lower().replace(" ", "-") + slug = f"{slug_base}-{block_key}" + embed_text = message + if topic_bits: + embed_text += "\n\nTopics: " + ", ".join(dict.fromkeys(topic_bits)) + out.append((slug, embed_text, list(citations))) + # CRE-linked notes as evidence (optional retrieve path). + for cre_id, facts in NODE_FACTS_BY_CRE.items(): + for block_key, block in facts.items(): + if not isinstance(block, dict): + continue + message, citations = format_facts_message({block_key: block}) + if not message: + continue + topics = block.get("topics") or [] + topic_bits = [str(t) for t in topics if t] + [cre_id] + slug = f"cre-{cre_id.lower()}-{block_key}" + embed_text = message + if topic_bits: + embed_text += "\n\nTopics: " + ", ".join(dict.fromkeys(topic_bits)) + out.append((slug, embed_text, list(citations))) + return out + + +def facts_embed_appendix(facts: Dict[str, Dict[str, Any]]) -> str: + """Prose appendix for meta/standard embed text when facts are present.""" + if not facts: + return "" + message, _ = format_facts_message(facts) + topics: List[str] = [] + for block in facts.values(): + if isinstance(block, dict): + for t in block.get("topics") or []: + if t: + topics.append(str(t)) + bits = [] + if message: + bits.append(message) + if topics: + bits.append("Topics: " + ", ".join(dict.fromkeys(topics))) + return "\n".join(bits) + + +def extract_cre_id_from_text(text: str) -> Optional[str]: + """Return first CRE-NNN-NNN mentioned in text, or None.""" + t = text or "" + m = re.search(r"(?i)\bcre[-\s]?(\d{3})[-\s]?(\d{3})\b", t) + if m: + return f"CRE-{m.group(1)}-{m.group(2)}" + return None diff --git a/application/utils/owasp_agent/queries.py b/application/utils/owasp_agent/queries.py index 587749fd4..175338465 100644 --- a/application/utils/owasp_agent/queries.py +++ b/application/utils/owasp_agent/queries.py @@ -413,6 +413,8 @@ def project_lookup(self, name: str) -> QueryResult: ) pname = hit.get("name") or hit.get("key") desc = (hit.get("description") or "").strip() + if not desc: + desc = _project_fallback_blurb(hit) leaders = [x for x in (hit.get("leaders") or []) if x] level = hit.get("level") or "" url = hit.get("url") or "" @@ -1191,6 +1193,48 @@ def _match_chapter( return best +_PROJECT_FALLBACK_BLURBS = { + "juice-shop": ( + "Intentionally vulnerable web application for security training and " + "testing (Juice Shop)." + ), + "juice shop": ( + "Intentionally vulnerable web application for security training and " + "testing (Juice Shop)." + ), + "application-security-verification-standard": ( + "Application Security Verification Standard (ASVS): a framework of " + "security requirements for designing, testing, and verifying apps." + ), + "asvs": ( + "Application Security Verification Standard (ASVS): a framework of " + "security requirements for designing, testing, and verifying apps." + ), + "top-ten": ( + "OWASP Top Ten: the standard awareness document for the most critical " + "web application security risks." + ), + "top ten": ( + "OWASP Top Ten: the standard awareness document for the most critical " + "web application security risks." + ), +} + + +def _project_fallback_blurb(hit: Dict[str, Any]) -> str: + key = _norm(str(hit.get("key") or "").replace("-", " ")) + pname = _norm(str(hit.get("name") or "")) + for cand in (str(hit.get("key") or ""), key, pname): + blurb = _PROJECT_FALLBACK_BLURBS.get(cand) + if blurb: + return blurb + compact = cand.replace(" ", "") + for k, v in _PROJECT_FALLBACK_BLURBS.items(): + if k.replace(" ", "") in compact or compact in k.replace(" ", ""): + return v + return "" + + def _alias_matches_target(target: str, alias: str) -> bool: """Strict alias hit — short tokens (ai, ml) only match exactly.""" a = _norm(alias) @@ -1209,9 +1253,13 @@ def _match_project( ) -> Optional[Dict[str, Any]]: from application.utils.owasp_agent.project_aliases import ensure_project_aliases - target = _norm(name) + # Strip github slug noise before normalising: owasp/www-project-juice-shop → juice shop + raw = re.sub(r"^\s*owasp/", "", name or "", flags=re.IGNORECASE) + raw = re.sub(r"^www-project-", "", raw, flags=re.IGNORECASE) + target = re.sub(r"[-_]+", " ", _norm(raw)).strip() if not target: return None + target = re.sub(r"\s*\(no space\)\s*", " ", target).strip() for prefix in ("owasp ", "the ", "project "): if target.startswith(prefix): target = target[len(prefix) :].strip() diff --git a/application/utils/owasp_agent/router.py b/application/utils/owasp_agent/router.py index 980570f0a..9c6a8ae8b 100644 --- a/application/utils/owasp_agent/router.py +++ b/application/utils/owasp_agent/router.py @@ -5,6 +5,7 @@ import os import re from dataclasses import dataclass, field +from datetime import datetime, timezone from typing import Any, Dict, List, Optional from application.utils.owasp_agent.index_store import IndexStore @@ -21,6 +22,46 @@ def is_owasp_agent_enabled() -> bool: ) +# Intents with structured slots — do not let a mismatched sim meta card win. +_STRUCTURED_INTENTS = frozenset( + { + "events_near", + "board_members", + "board_person", + "board_candidate_stats", + "project_lookup", + "compare_projects", + "person_lookup", + "membership", + "chapter_lookup", + "count_chapters", + "count_projects", + "count_projects_and_chapters", + "talk_lookup", + "cre_fact", + "cre_community_bait", + "project_and_board", + "clarify", + "refuse_malicious", + } +) + +# Well-known project tokens that are not the foundation board. +_PROJECT_BOARD_ALIASES = ( + "asvs", + "masvs", + "samm", + "juice shop", + "juiceshop", + "zap", + "top ten", + "top-ten", + "top 10", + "ai exchange", + "aix", +) + + @dataclass class Slots: place: Optional[str] = None @@ -179,6 +220,7 @@ def handle( if sim_decision is not None: from application.utils.owasp_agent.candidate_picker import ( default_completion_fn, + is_structure_ask, select_agent_hit, ) from application.utils.owasp_agent.lane_similarity import ( @@ -200,7 +242,32 @@ def handle( cands, get_completion_fn=default_completion_fn, ) - if picked is not None and is_evidence_hit(picked): + structure_ask = is_structure_ask(normalized) + + # Slot-heavy intents (board year, events near, ambiguous chapter…) + # must not be overridden by a mismatched sim meta card. + early_slots = extract_slots(effective) + + def _allow_sim_meta(hit: Any) -> bool: + kind, key = _meta_kind_key(getattr(hit, "url", "") or "") + if not kind: + return True + return _sim_meta_ok_for_intent( + intent_preview, kind, key, effective, early_slots + ) + + if ( + picked is not None + and is_evidence_hit(picked) + and intent_preview + not in ( + "events_near", + "board_members", + "clarify", + "cre_community_bait", + "project_and_board", + ) + ): cites = ( [ c.strip() @@ -236,12 +303,23 @@ def handle( resp["owasp_agent"]["also_cre"] = True resp["owasp_agent"]["lane_reason"] = sim_decision.reason return resp - if picked is not None and is_meta_hit(picked): + if picked is not None and is_meta_hit(picked) and _allow_sim_meta(picked): + # Fail-closed: never answer level/layer/maturity asks via Nest + # project_lookup (Leaders: / indexed OWASP project). + meta_kind = "" + u = picked.url or "" + if u.startswith("owasp-agent://meta/"): + meta_kind = u[len("owasp-agent://meta/") :].split("/", 1)[0] + if structure_ask and meta_kind == "project": + return None meta_resp = self._response_from_meta_hit( picked, sim_decision=sim_decision ) if meta_resp is not None: return meta_resp + elif structure_ask and not any(is_evidence_hit(h) for h in cands): + # Structure ask, no evidence in top-k, no viable pick → defer. + return None # CRE-dominant must not erase a mid+ agent evidence / chapter / board card. if sim_decision.action == "cre" and cands: @@ -258,7 +336,17 @@ def handle( if kind in ("evidence", "chapter", "board_roster") and ( hit.score >= t_rescue ): + if not _allow_sim_meta(hit) and kind != "evidence": + continue if kind == "evidence" and is_evidence_hit(hit): + if intent_preview in ( + "events_near", + "board_members", + "clarify", + "cre_community_bait", + "project_and_board", + ): + continue message = ( (hit.content or "").split("\n\nTopics:")[0].strip() ) @@ -322,21 +410,27 @@ def handle( if intent == "cre_normative": return None if intent == "cre_fact": - from application.utils.owasp_agent.cre_facts import ( - format_cre_factoid, - lookup_cre_factoid, + # Metadata facts by CRE id only — no keyword factoid gate. + from application.utils.owasp_agent.node_facts import ( + extract_cre_id_from_text, + facts_for_cre_id, + format_facts_message, ) - cre_hit = lookup_cre_factoid(effective) - if not cre_hit: + cre_id = extract_cre_id_from_text(effective) + facts = facts_for_cre_id(cre_id or "") + if not facts: return None # fall through to CRE RAG - cre_id, title, note = cre_hit + message, citations = format_facts_message(facts) + if not message: + return None result = QueryResult( ok=True, kind="cre_fact", - data={"cre_id": cre_id, "title": title}, - message=format_cre_factoid(cre_id, title, note), - citations=[f"https://opencre.org/cre/{cre_id}"], + data={"cre_id": cre_id, "facts": facts}, + message=message, + citations=citations + or ([f"https://opencre.org/cre/{cre_id}"] if cre_id else []), ) return format_chat_response(result) if intent == "refuse_malicious": @@ -464,11 +558,123 @@ def _dispatch(self, intent: str, slots: Slots, text: str) -> QueryResult: return self.queries.person_lookup(slots.person) years = slots.years or ([slots.year] if slots.year is not None else []) if years: - return self.queries.corpus_entity("board_roster", str(years[0])) + hit = self.queries.corpus_entity("board_roster", str(years[0])) + if hit.ok: + return hit + # Explicit year with no card — fail closed (do not substitute latest). + members = self.queries.board_members(int(years[0])) + if members.ok: + return members + return QueryResult( + ok=False, + kind="board_members", + message=( + f"No OWASP board roster is indexed for year {years[0]} " + "(fail closed). I will not substitute another year's members." + ), + ) return self.queries.corpus_entity("board_roster", "overview") + if intent == "project_and_board": + proj_name = slots.project_name or "" + # Slot extract may swallow "… AND who is on the board". + if " and " in proj_name.lower(): + proj_name = re.split(r"\s+and\s+", proj_name, maxsplit=1, flags=re.I)[ + 0 + ].strip() + if not proj_name or not _mentions_known_project(proj_name): + if re.search(r"\bjuice\s*shop\b|\bjuiceshop\b", text, re.I): + proj_name = "Juice Shop" + elif re.search(r"\basvs\b", text, re.I): + proj_name = "ASVS" + else: + proj_name = proj_name or "Juice Shop" + proj = self.queries.project_lookup(proj_name) + board = self.queries.corpus_entity("board_roster", "overview") + parts = [] + cites: List[str] = [] + if proj.ok: + parts.append(proj.message) + cites.extend(proj.citations or []) + if board.ok: + parts.append(board.message) + cites.extend(board.citations or []) + if not parts: + return QueryResult( + ok=False, + kind="project_and_board", + message="I could not find Juice Shop leaders or board roster in the index.", + ) + return QueryResult( + ok=True, + kind="project_and_board", + data={ + "project": (proj.data or {}).get("project") if proj.ok else None, + "board": (board.data or {}).get("entity") if board.ok else None, + }, + message=" ".join(parts), + citations=list(dict.fromkeys(cites)), + ) + if intent == "cre_community_bait": + name = slots.project_name or slots.place or slots.topic + if name and ( + "chapter" in text.lower() + or (slots.place and "project" not in text.lower()) + ): + ch = self.queries.chapter_lookup( + place=slots.place, places=slots.places or None + ) + base = ch.message if ch.ok else f"OWASP community entity {name!r}" + return QueryResult( + ok=True, + kind="cre_community_bait", + data=ch.data if ch.ok else {"name": name}, + message=( + f"{base} OWASP chapters and meetup policies are community " + "metadata, not Common Requirements (CREs) — there is no CRE ID " + "that 'covers' a chapter meetup policy." + ), + citations=list(ch.citations or []) if ch.ok else [], + ) + proj = self.queries.project_lookup(name or "Juice Shop") + base = proj.message if proj.ok else f"{name or 'That OWASP project'}" + return QueryResult( + ok=True, + kind="cre_community_bait", + data=proj.data if proj.ok else {"name": name}, + message=( + f"{base} OWASP projects are community/metadata entities, not CRE " + "nodes — there is no single CRE ID that maps an entire project." + ), + citations=list(proj.citations or []) if proj.ok else [], + ) + if intent == "clarify": + return QueryResult( + ok=False, + kind="clarify", + message=( + "Which OWASP chapter or city do you mean? " + "I need a place name to look up chapter leaders or the next meetup." + ), + clarify="Please name a city or chapter (e.g. London, Berlin).", + ) if intent == "chapter_lookup": - places = slots.places or ([slots.place] if slots.place else []) - return self.queries.chapter_lookup(place=slots.place, places=places or None) + places = list(slots.places or []) + if slots.place and slots.place not in places: + places = [slots.place] + places + # Multi-city compare / negation: look up all named places. + if len(places) >= 2: + return self.queries.chapter_lookup(places=places) + if not places: + return QueryResult( + ok=False, + kind="clarify", + message=( + "Which OWASP chapter or city should I look up? " + "Please name one (e.g. London, Berlin)." + ), + clarify="Please name a city or chapter.", + ) + return self.queries.chapter_lookup(place=places[0], places=places) if intent == "membership": renew = bool( re.search(r"\b(renew|renewal|how do i (join|become))\b", text.lower()) @@ -486,32 +692,39 @@ def _dispatch(self, intent: str, slots: Slots, text: str) -> QueryResult: ) if intent == "talk_lookup": talk = self.queries.talk_lookup(person=slots.person, topic=slots.topic) - # Compound: "fail-closed on X talk; succeed on CRE-…" - from application.utils.owasp_agent.cre_facts import ( - format_cre_factoid, - lookup_cre_factoid, + # Compound: fail-closed talk + CRE id with metadata facts (no keywords). + from application.utils.owasp_agent.node_facts import ( + extract_cre_id_from_text, + facts_for_cre_id, + format_facts_message, ) - cre_hit = lookup_cre_factoid(text) - if cre_hit and talk.ok: - cre_id, title, note = cre_hit - cre_msg = format_cre_factoid(cre_id, title, note) - return QueryResult( - ok=True, - kind="talk_and_cre", - data={ - "talk": talk.data, - "cre_id": cre_id, - "title": title, - }, - message=f"{talk.message} Separately: {cre_msg}", - citations=list( - dict.fromkeys( - list(talk.citations or []) - + [f"https://opencre.org/cre/{cre_id}"] - ) - ), - ) + cre_id = extract_cre_id_from_text(text) + facts = facts_for_cre_id(cre_id or "") if cre_id else {} + if facts and talk.ok: + cre_msg, cre_cites = format_facts_message(facts) + if cre_msg: + return QueryResult( + ok=True, + kind="talk_and_cre", + data={ + "talk": talk.data, + "cre_id": cre_id, + "facts": facts, + }, + message=f"{talk.message} Separately: {cre_msg}", + citations=list( + dict.fromkeys( + list(talk.citations or []) + + list(cre_cites) + + ( + [f"https://opencre.org/cre/{cre_id}"] + if cre_id + else [] + ) + ) + ), + ) return talk if intent == "person_lookup": if not slots.person: @@ -523,6 +736,19 @@ def _dispatch(self, intent: str, slots: Slots, text: str) -> QueryResult: ) return self.queries.person_lookup(slots.person) if intent == "project_lookup": + from application.utils.owasp_agent.candidate_picker import is_structure_ask + + # Fail-closed: structural level/layer/maturity asks must not Nest-steal. + if is_structure_ask(text): + return QueryResult( + ok=False, + kind="clarify", + message=( + "I do not have curated structure evidence for that question " + "in the agent index. Try again after evidence is seeded, " + "or ask via CRE/standards tools." + ), + ) name = slots.project_name or slots.topic if not name: return QueryResult( @@ -531,7 +757,26 @@ def _dispatch(self, intent: str, slots: Slots, text: str) -> QueryResult: message="Which OWASP project should I look up?", clarify="Please name a project.", ) - return self.queries.project_lookup(name) + result = self.queries.project_lookup(name) + if ( + result.ok + and re.search(r"\bboard\b", text.lower()) + and _mentions_known_project(text) + ): + note = ( + " It does not have a separate foundation board; " + "indexed project leaders are listed." + ) + if "does not have a separate foundation board" not in result.message: + result = QueryResult( + ok=result.ok, + kind=result.kind, + data=result.data, + message=result.message.rstrip(".") + "." + note, + citations=result.citations, + clarify=result.clarify, + ) + return result if intent == "compare_projects": a = slots.project_a b = slots.project_b @@ -543,30 +788,7 @@ def _dispatch(self, intent: str, slots: Slots, text: str) -> QueryResult: clarify="Name both projects (e.g. AI Exchange vs GenAI Security Project).", ) return self.queries.compare_projects(a, b) - if intent == "cre_fact": - from application.utils.owasp_agent.cre_facts import ( - format_cre_factoid, - lookup_cre_factoid, - ) - - hit = lookup_cre_factoid(text) - if not hit: - return QueryResult( - ok=False, - kind="cre_fact", - message=( - "I do not have a curated CRE factoid for that control. " - "Falling back is handled by CRE RAG." - ), - ) - cre_id, title, note = hit - return QueryResult( - ok=True, - kind="cre_fact", - data={"cre_id": cre_id, "title": title}, - message=format_cre_factoid(cre_id, title, note), - citations=[f"https://opencre.org/cre/{cre_id}"], - ) + # cre_fact is handled in handle() before _dispatch. if intent == "events_near": upcoming_only = not slots.include_past return self.queries.events_near( @@ -620,6 +842,57 @@ def classify_intent(text: str, *, known_people: Optional[List[str]] = None) -> s from application.utils.owasp_agent.person_resolve import match_indexed_person t = text.lower().strip() + # Strip markdown fences / chat-role wrappers for structural matching. + t_flat = re.sub(r"[`*\n]+", " ", t) + t_flat = re.sub(r"\b(user|assistant)\s*:", " ", t_flat) + t_flat = re.sub(r"\s+", " ", t_flat).strip() + + # Ambiguous chapter/meetup with no place — clarify before sim can invent one. + if re.search( + r"\b(who is the chapter leader|chapter leader\b|when is the next meetup|" + r"next meetup\b)\b", + t_flat, + ) and not re.search( + r"\b(in|near|around|for)\s+[a-z]", + t_flat, + ): + # Allow "London chapter leader" etc. via place cues below. + if not re.search( + r"\b(london|berlin|athens|tokyo|paris|nyc|new york|bangalore|" + r"los angeles|san francisco|seattle|chicago|toronto|sydney)\b", + t_flat, + ): + return "clarify" + + # CRE-bait on community entities (project/chapter) — answer meta, no CRE id. + if re.search(r"\b(map .+ to (a )?cre|what cre covers|cre covers|cre id)\b", t_flat): + if re.search( + r"\b(juice\s*shop|juiceshop|chapter|meetup|asvs|project|berlin|london)\b", + t_flat, + ): + return "cre_community_bait" + + # Dual: project leaders + foundation board in one ask. + if re.search(r"\b(juice\s*shop|juiceshop|asvs|zap|top\s*ten)\b", t_flat) and ( + re.search(r"\b(leads?|leaders?|who leads)\b", t_flat) + and re.search(r"\b(owasp )?board\b", t_flat) + and re.search(r"\b(and|who is on)\b", t_flat) + ): + return "project_and_board" + + # Project-named "board members" (ASVS board) → project leaders, not foundation. + if re.search(r"\bboard\b", t_flat) and re.search( + r"\b(members?|roster|who (is|are))\b", t_flat + ): + if any(alias in t_flat for alias in _PROJECT_BOARD_ALIASES): + return "project_lookup" + + # GitHub-style www-project-* slug / what-is project. + if re.search(r"\bwww-project-[a-z0-9-]+\b", t_flat) or re.search( + r"\bowasp/[a-z0-9._/-]+\b", t_flat + ): + return "project_lookup" + person_hit = match_indexed_person(text, known_people) # Who-is / tell-me-about person lookup from indexed leaders/board (not CRE bios) if re.search(r"\bwho is\b", t) and not re.search(r"\bwho is (the|on|a|an)\b", t): @@ -644,6 +917,14 @@ def classify_intent(text: str, *, known_people: Optional[List[str]] = None) -> s and re.search(r"\bprojects?\b", t) ): return "count_projects_and_chapters" + # Chapter compare (London vs Berlin) before project compare. + if re.search( + r"\b(difference between|differ(?:ence)? between|compare|" + r"\bvs\.?\b|versus)\b", + t_flat, + ) and re.search(r"\bchapters?\b", t_flat): + if not re.search(r"\bprojects?\b", t_flat): + return "chapter_lookup" # Project compare / definition — structural phrasing only (no brand lists). if re.search( r"\b(difference between|differ(?:ence)? between|compare|" @@ -658,11 +939,13 @@ def classify_intent(text: str, *, known_people: Optional[List[str]] = None) -> s ): return "count_projects_and_chapters" return "compare_projects" - if re.search(r"\b(what is|what's|describe)\b.+\bowasp project\b", t) or re.search( + if re.search( + r"\b(what is|what's|describe)\b.+\bowasp project\b", t_flat + ) or re.search( r"\b(what is|what's)\b.+\bproject\b", - t, + t_flat, ): - if not re.search(r"\b(how (do|should)|mitigate|vulnerability)\b", t): + if not re.search(r"\b(how (do|should)|mitigate|vulnerability)\b", t_flat): return "project_lookup" # Malicious / unauthorized help — refuse before any meta routing so @@ -702,28 +985,18 @@ def classify_intent(text: str, *, known_people: Optional[List[str]] = None) -> s ): return "talk_lookup" - # Fail-closed talk asks (often compound with a CRE factoid) before CRE-only. - # Do not steal board asks that only say "do not invent talk quotes". - if ( - re.search(r"\bfail[- ]closed\b", t) - and re.search(r"\btalk\b", t) - and not re.search(r"\bboard\b", t) - ): - return "talk_lookup" - - # Curated CRE factoids (deterministic) before generic CRE RAG - from application.utils.owasp_agent.cre_facts import lookup_cre_factoid + # CRE id with curated metadata facts (no keyword factoid gate). + from application.utils.owasp_agent.node_facts import ( + extract_cre_id_from_text, + facts_for_cre_id, + ) - if lookup_cre_factoid(text): + cre_id_hint = extract_cre_id_from_text(text) + if cre_id_hint and facts_for_cre_id(cre_id_hint): # Talk + CRE compound stays talk_lookup so handle() can merge both. if re.search(r"\btalk\b", t): return "talk_lookup" return "cre_fact" - if re.search(r"\b(which cre|what cre|cre covers)\b", t) or ( - "cre" in t and "covers" in t - ): - if lookup_cre_factoid(text): - return "cre_fact" # Talk claims (incl. fake speakers / suburb meetups) → fail-closed if ( @@ -778,11 +1051,12 @@ def classify_intent(text: str, *, known_people: Optional[List[str]] = None) -> s ): return "count_projects_and_chapters" - # Project leaders before bare "who leads" (which otherwise steals projects). - if re.search(r"\b(who leads|leaders? (of|for)|project leader)\b", t) and re.search( - r"\bprojects?\b", t + # Project leaders (incl. "leaders of Top Ten" / "JuiceShop … leaders?"). + if re.search(r"\b(who leads|leaders? (of|for)|project leaders?)\b", t_flat) or ( + re.search(r"\bprojects?\b", t_flat) and re.search(r"\bleaders?\b", t_flat) ): - return "project_lookup" + if re.search(r"\bprojects?\b", t_flat) or _mentions_known_project(t_flat): + return "project_lookup" # Named / what-is OWASP chapter cards (phrase-shaped — not bare "chapter"). if re.search( @@ -795,8 +1069,10 @@ def classify_intent(text: str, *, known_people: Optional[List[str]] = None) -> s ): return "chapter_lookup" - # Chapter existence / "chapter in {city}" before general clarify. - if re.search(r"\bchapters?\b", t) and re.search(r"\b(in|near|for|at|around)\b", t): + # Chapter existence / negation / "chapter in {city}" before general clarify. + if re.search(r"\bchapters?\b", t) and re.search( + r"\b(in|near|for|at|around|not exist|does not exist|doesn't exist)\b", t + ): if not re.search(r"\b(how many|count of)\b", t): return "chapter_lookup" if re.search( @@ -804,15 +1080,20 @@ def classify_intent(text: str, *, known_people: Optional[List[str]] = None) -> s t, ): return "chapter_lookup" + if re.search(r"\bwhich .{0,40}chapter .{0,20}not exist\b", t_flat) or re.search( + r"\bchapter does not exist\b", t_flat + ): + return "chapter_lookup" - # Chapter leader / status / active — structural cues only (no city hardcodes). + # Chapter leader / status / active — chapter cue, or "OWASP leader for {city}". if re.search( - r"\b(chapter leader|leader for|leaders? (for|in|of)|who leads|" - r"which chapter|active owasp chapter|chapter status|owasp leader|" + r"\b(chapter leader|leader for (the )?chapter|who leads .{0,20}chapter|" + r"which chapter|active owasp chapter|chapter status|owasp chapter leader|" r"count of indexed owasp chapters|approximate count of .{0,20}chapters)\b", t, ) or ( - re.search(r"\bchapters?\b", t) and re.search(r"\b(active|inactive|status)\b", t) + re.search(r"\bchapters?\b", t) + and re.search(r"\b(active|inactive|status|leaders?|who leads)\b", t) ): if re.search( r"\b(count|how many|approximate count)\b.+\bchapters?\b", @@ -820,6 +1101,13 @@ def classify_intent(text: str, *, known_people: Optional[List[str]] = None) -> s ) or re.search(r"\bchapters?\b.+\b(count|how many)\b", t): return "count_chapters" return "chapter_lookup" + # "Who is the OWASP leader for Athens?" — city/place leader, not Top Ten. + if ( + re.search(r"\b(owasp leader|leader for)\b", t_flat) + and not _mentions_known_project(t_flat) + and not re.search(r"\bprojects?\b", t_flat) + ): + return "chapter_lookup" # Standalone chapter counts (without "how many") if ( @@ -926,6 +1214,13 @@ def extract_slots(text: str, *, known_people: Optional[List[str]] = None) -> Slo if years: slots.years = years slots.year = years[0] + elif re.search(r"\b(this|current)\s+year\b", t) or re.search( + r"\bfor this year\b", t + ): + # Calendar year in UTC — "this year" must not pick a random sim year card. + y = datetime.now(timezone.utc).year + slots.year = y + slots.years = [y] if any(w in t for w in ("past", "previous", "last year", "history", "historical")): slots.include_past = True @@ -1017,6 +1312,43 @@ def extract_slots(text: str, *, known_people: Optional[List[str]] = None) -> Slo if slots.place and slots.place not in slots.places: slots.places.append(slots.place) + # "Compare OWASP London and OWASP Berlin chapters" / "London or Atlantis" + m_and_ch = re.search( + r"\b(?:compare\s+)?(?:owasp\s+)?([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)" + r"\s+and\s+(?:owasp\s+)?([A-Z][a-z]+(?:\s+[A-Z][a-z]+)?)\s+chapters?\b", + text, + re.IGNORECASE, + ) + if m_and_ch: + slots.places = [ + m_and_ch.group(1).strip().rstrip(".?!"), + m_and_ch.group(2).strip().rstrip(".?!"), + ] + slots.place = slots.places[0] + m_or_ch = re.search( + r"\b(?:does\s+)?not\s+exist\s*:\s*" + r"([A-Za-z][A-Za-z '-]+?)\s+or\s+([A-Za-z][A-Za-z '-]+?)\s*\??\s*$", + text, + re.IGNORECASE, + ) + if not m_or_ch and re.search(r"\b(not exist|does not exist|doesn't exist)\b", t): + m_or_ch = re.search( + r"\b([A-Z][a-zA-Z'-]+)\s+or\s+([A-Z][a-zA-Z'-]+)\b", + text, + ) + if m_or_ch: + a = m_or_ch.group(1).strip().rstrip(".?!") + b = m_or_ch.group(2).strip().rstrip(".?!") + # Drop leftover cue words ("chapter", "owasp"). + for city in (a, b): + low = city.lower() + if low in ("chapter", "owasp", "the", "a", "an"): + continue + if city and city not in slots.places: + slots.places.append(city) + if not slots.place and slots.places: + slots.place = slots.places[0] + person = match_indexed_person(text, known_people) if person: slots.person = person @@ -1049,14 +1381,22 @@ def extract_slots(text: str, *, known_people: Optional[List[str]] = None) -> Slo slots.project_a = _clean_project_name(m_vs_proj.group(1)) slots.project_b = _clean_project_name(m_vs_proj.group(2)) + m_slug_early = re.search( + r"\b(?:owasp/)?www-project-([a-z0-9-]+)\b", + text, + re.IGNORECASE, + ) + if m_slug_early and not slots.project_name: + slots.project_name = m_slug_early.group(1).replace("-", " ") + m_what_proj = re.search( r"\b(?:what is|what's|tell me about|describe)\s+(?:the\s+)?(.+?)\s*\??\s*$", text, re.IGNORECASE, ) - if m_what_proj and not slots.project_a: + if m_what_proj and not slots.project_a and not slots.project_name: cand = _clean_project_name(m_what_proj.group(1)) - if cand and not re.match(r"^(who|how|why)\b", cand, re.I): + if cand and not re.match(r"^(who|how|why|it|this|that|a|an)\b", cand, re.I): slots.project_name = cand # "Who leads the OWASP Juice Shop project?" / "leaders of OWASP ASVS" @@ -1067,11 +1407,18 @@ def extract_slots(text: str, *, known_people: Optional[List[str]] = None) -> Slo text, re.IGNORECASE, ) + if not m_leads_proj: + m_leads_proj = re.search( + r"\b(?:who leads|leaders? (?:of|for)|project leaders? (?:of|for))\s+" + r"(?:the\s+)?(?:owasp\s+)?(.+?)\s*\??\s*$", + text, + re.IGNORECASE, + ) if m_leads_proj: cand = _clean_project_name(m_leads_proj.group(1)) - if cand: + if cand and not re.match(r"^(the|a|an|owasp)$", cand, re.I): slots.project_name = cand - else: + if not slots.project_name: m_owasp_proj = re.search( r"\bowasp\s+(.+?)\s+project\b", text, @@ -1081,6 +1428,20 @@ def extract_slots(text: str, *, known_people: Optional[List[str]] = None) -> Slo cand = _clean_project_name(m_owasp_proj.group(1)) if cand: slots.project_name = cand + if not slots.project_name: + m_slug = re.search( + r"\b(?:owasp/)?www-project-([a-z0-9-]+)\b", + text, + re.IGNORECASE, + ) + if m_slug: + slots.project_name = m_slug.group(1).replace("-", " ") + if not slots.project_name and re.search(r"\bjuiceshop\b|\bjuice\s*shop\b", t): + slots.project_name = "Juice Shop" + if not slots.project_name and re.search(r"\basvs\b", t): + slots.project_name = "ASVS" + if not slots.project_name and re.search(r"\btop\s*ten\b|\btop\s*10\b", t): + slots.project_name = "Top Ten" # Membership countries — any Title Case token looked up later; seed common ones known_countries = ( @@ -1266,6 +1627,258 @@ def _mentions_appsec(text: str) -> bool: return "appsec" in t or "application security" in t +# Capitalized personal names, including multi-particle Dutch/German forms +# (e.g. "Rob van der Veer", "Andrew van der Stock"). +_PERSON_NAME_RE = r"[A-Z][a-z]+(?:\s+(?:van|der|de|la|von|[A-Z][a-z]+))+" + +_NAME_LEADING_STOPWORDS = frozenset( + { + "show", + "compare", + "quote", + "quoting", + "before", + "after", + "when", + "what", + "which", + "whose", + "did", + "does", + "has", + "have", + "is", + "are", + "was", + "were", + "the", + "and", + "for", + "from", + "with", + "about", + "board", + "nearest", + "multi", + "hop", + "fail", + "closed", + "draft", + "list", + "how", + "many", + } +) + + +def _name_starts_with_stopword(name: str) -> bool: + first = (name or "").strip().split()[0].lower() if name else "" + return first in _NAME_LEADING_STOPWORDS + + +def _extract_quoted_or_capitalized_name(text: str) -> Optional[str]: + m = re.search(r"[\"']([A-Z][^\"']+)[\"']", text) + if m: + return m.group(1).strip() + m_who = re.search(r"\b[Ww]ho\s+is\s+(" + _PERSON_NAME_RE + r")\s*\??", text) + if m_who: + return m_who.group(1).strip().rstrip(".?!") + m_about = re.search( + r"\b(?:[Tt]ell me about|[Ww]hat about)\s+(" + _PERSON_NAME_RE + r")\b", + text, + ) + if m_about and not _name_starts_with_stopword(m_about.group(1)): + return m_about.group(1).strip().rstrip(".?!") + # "Jeff Williams: board service years…" (colon only — not Name's possessive) + m_lead = re.search( + r"^(" + _PERSON_NAME_RE + r")\s*:", + text.strip(), + ) + if m_lead and not _name_starts_with_stopword(m_lead.group(1)): + return m_lead.group(1).strip() + # Leading name before dash/em-dash ("Rob van der Veer — which projects…") + m_lead_dash = re.search( + r"^(" + _PERSON_NAME_RE + r")\s*[—–\-]\s*", + text.strip(), + ) + if m_lead_dash and not _name_starts_with_stopword(m_lead_dash.group(1)): + return m_lead_dash.group(1).strip() + # Leading name + board / years without requiring colon + # ("Andrew van der Stock board years — …") + m_lead_board = re.search( + r"^(" + _PERSON_NAME_RE + r")\s+board\b", + text.strip(), + ) + if m_lead_board and not _name_starts_with_stopword(m_lead_board.group(1)): + return m_lead_board.group(1).strip() + # Verb-led extractions before bare possessive (avoids "Show Sam …" as a name). + m_is = re.search( + r"\b[Ii]s\s+(" + _PERSON_NAME_RE + r")\s+on\b", + text, + ) + if m_is: + return m_is.group(1).strip() + m_has = re.search( + r"\b[Hh]as\s+(" + _PERSON_NAME_RE + r")\s+been\b", + text, + ) + if m_has: + return m_has.group(1).strip() + m_show = re.search( + r"\b[Ss]how\s+(" + _PERSON_NAME_RE + r")(?:'s|’s)?\b", + text, + ) + if m_show: + return m_show.group(1).strip() + m_compare = re.search( + r"\b[Cc]ompare\s+(" + _PERSON_NAME_RE + r")(?:'s|’s)?\b", + text, + ) + if m_compare: + return m_compare.group(1).strip() + m_quoting = re.search( + r"\b(?:quoting|quote)\s+(" + _PERSON_NAME_RE + r")(?:'s|’s)?\b", + text, + re.IGNORECASE, + ) + if m_quoting: + return m_quoting.group(1).strip() + # Any possessive Name's … (candidate page, board candidacy, AppSec talk) + m_possessive = re.search( + r"\b(" + _PERSON_NAME_RE + r")(?:'s|’s)\b", + text, + ) + if m_possessive and not _name_starts_with_stopword(m_possessive.group(1)): + return m_possessive.group(1).strip() + m_by = re.search( + r"\btalks?\s+by\s+(?:a\s+random\s+person\s+named\s+)?(" + + _PERSON_NAME_RE + + r")\b", + text, + ) + if m_by: + return m_by.group(1).strip() + m_did = re.search( + r"\b(?:[Ww]hat\s+[Dd]id|[Dd]id)\s+(" + _PERSON_NAME_RE + r")\b", + text, + ) + if m_did: + return m_did.group(1).strip() + m2 = re.search( + r"(?:position of|board member|candidate|quote)\s+(" + _PERSON_NAME_RE + r")", + text, + ) + if m2: + return m2.group(1).strip() + m3 = re.search( + r"\b(" + _PERSON_NAME_RE + r")\s+(?:say|said|run|ever|present|give|from)\b", + text, + ) + if m3: + name = m3.group(1).strip() + first = name.split()[0].lower() + if first not in ("did", "what", "who", "how", "when", "where", "is", "the"): + return name + m4 = re.search( + r"\b(?:for|by|of)\s+(" + _PERSON_NAME_RE + r")\s*\??\s*$", + text, + ) + if m4: + return m4.group(1).strip() + return None + + +def _meta_kind_key(url: str) -> tuple[str, str]: + """Parse ``owasp-agent://meta/{kind}/{key}/...`` → (kind, key).""" + u = (url or "").strip() + if not u.startswith("owasp-agent://meta/"): + return "", "" + parts = u[len("owasp-agent://meta/") :].split("/") + if len(parts) < 2: + return (parts[0].strip().lower() if parts else ""), "" + return parts[0].strip().lower(), parts[1].strip() + + +def _mentions_known_project(text: str) -> bool: + t = (text or "").lower() + if any(a in t for a in _PROJECT_BOARD_ALIASES): + return True + if re.search(r"\bwww-project-[a-z0-9-]+\b", t): + return True + if re.search(r"\bjuiceshop\b|\bjuice\s*shop\b|\btop\s*ten\b|\btop\s*10\b", t): + return True + return False + + +def _sim_meta_ok_for_intent( + intent: str, + kind: str, + key: str, + text: str, + slots: Optional[Slots], +) -> bool: + """Whether a similarity meta hit may answer ``intent`` (else fall through).""" + intent = (intent or "").strip() + kind = (kind or "").strip().lower() + key = (key or "").strip() + if intent not in _STRUCTURED_INTENTS: + return True + if intent in ( + "clarify", + "refuse_malicious", + "cre_community_bait", + "project_and_board", + ): + return False + if intent == "events_near": + return kind == "event" + if intent == "board_members": + if kind != "board_roster": + return False + years: List[int] = [] + if slots is not None: + years = list(slots.years or []) + if slots.year is not None and slots.year not in years: + years = [slots.year] + years + if years: + return key == str(years[0]) + return key in ("overview", "") or bool(re.fullmatch(r"20\d{2}", key)) + if intent == "board_person": + return kind in ("board_member", "board_candidate", "person", "project_leader") + if intent == "chapter_lookup": + if kind != "chapter": + return False + places: List[str] = [] + if slots is not None: + places = list(slots.places or []) + if slots.place and slots.place not in places: + places = [slots.place] + places + if len(places) >= 2: + return False + if not places: + return False + target = re.sub(r"[^a-z0-9]+", "-", places[0].lower()).strip("-") + key_n = re.sub(r"[^a-z0-9]+", "-", key.lower()).strip("-") + return bool(target) and (target in key_n or key_n in target) + if intent == "project_lookup": + return kind == "project" + if intent == "person_lookup": + return kind in ("person", "project_leader", "board_member", "board_candidate") + if intent == "membership": + return kind in ("membership_country", "membership") + if intent == "compare_projects": + return False + if intent.startswith("count_"): + return False + if intent == "talk_lookup": + return False + if intent == "cre_fact": + return False + if intent == "board_candidate_stats": + return False + return True + + def _clean_project_name(raw: str) -> str: name = (raw or "").strip() name = re.sub( @@ -1289,6 +1902,9 @@ def _clean_project_name(raw: str) -> str: name, flags=re.IGNORECASE, ).strip() + # "JuiceShop (no space)" → JuiceShop + name = re.sub(r"\s*\(no space\)\s*", "", name, flags=re.IGNORECASE).strip() + name = re.sub(r"\s+board members?$", "", name, flags=re.IGNORECASE).strip() return name diff --git a/scripts/mcp_edge_probe_runner.py b/scripts/mcp_edge_probe_runner.py new file mode 100755 index 000000000..f195687f5 --- /dev/null +++ b/scripts/mcp_edge_probe_runner.py @@ -0,0 +1,738 @@ +#!/usr/bin/env python3 +"""Incremental MCP edge-probe via staging Streamable HTTP + Cursor SDK. + +Laptop-local only. Never uses ssh/docker MCP stdio. + +Flow per turn: + Cursor agent generates question → write question to disk → + Python calls staging ``ask`` over HTTPS → write answer → + same Cursor agent evaluates → write eval. + +Classifications (every turn): + mcp_ok | mcp_fail + answer_good | answer_bad (only when mcp_ok; product quality) + +Stop when cumulative_product_bad > 100, or after tier 5000. +Default: record mcp_fail and continue. Optional consecutive stop: + MCP_EDGE_PROBE_MCP_FAIL_STOP=N (0 = disabled) + +Env: + CURSOR_API_KEY required (unless --dry-run) + OPENCRE_STAGING_ADMIN_API_TOKEN Bearer for remote ask + MCP_EDGE_PROBE_URL default https://staging.opencre.org/mcp + MCP_EDGE_PROBE_SEED_JSONL default tmp/owasp_mcp_ask_adv_5k_questions.jsonl + MCP_EDGE_PROBE_OUT default tmp/mcp_edge_probe + MCP_EDGE_PROBE_MODEL default composer-2.5 + MCP_EDGE_PROBE_CORPUS default auto + MCP_EDGE_PROBE_MCP_FAIL_STOP consecutive mcp_fail stop (default 0) + MCP_EDGE_PROBE_TIERS comma list (default 10,100,500,1000,2000,3000,4000,5000) + MCP_EDGE_PROBE_LOCAL_ASK=1 call application.mcp.ask in-process (seeded local DB) +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import sys +import time +import traceback +from collections import Counter, defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +_REPO = Path(__file__).resolve().parents[1] +_DEFAULT_SEED = _REPO / "tmp" / "owasp_mcp_ask_adv_5k_questions.jsonl" +_DEFAULT_OUT = _REPO / "tmp" / "mcp_edge_probe" +_TIERS_DEFAULT = (10, 100, 500, 1000, 2000, 3000, 4000, 5000) +_PRODUCT_BAD_STOP = 100 + +# Forced starter families — structural/count traps the old LIKE path missed. +_FORCED_SEEDS: List[Dict[str, Any]] = [ + { + "family": "asvs_levels", + "q": "How many levels does OWASP ASVS have?", + "note": "structural count trap", + }, + { + "family": "masvs_levels", + "q": "How many levels / verification levels does OWASP MASVS define?", + "note": "structural count trap", + }, + { + "family": "samm_levels", + "q": "How many maturity levels does OWASP SAMM use?", + "note": "structural count trap", + }, + { + "family": "aix_layers", + "q": "How many layers are in the OWASP AI Exchange (AIX) model?", + "note": "AIX site structural fact", + }, + { + "family": "aix_layers", + "q": "What are the 7 layers of the OWASP AI Exchange?", + "note": "AIX layers list", + }, +] + + +def _utc() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _log(msg: str) -> None: + print(f"[{_utc()}] {msg}", flush=True) + + +def _loud_mcp_fail(turn: int, detail: Dict[str, Any]) -> None: + bar = "=" * 72 + print(bar, flush=True) + print(f"MCP_FAIL turn={turn}", flush=True) + print(json.dumps(detail, indent=2, default=str)[:4000], flush=True) + print(bar, flush=True) + + +def _parse_json_blob(text: str) -> Dict[str, Any]: + text = (text or "").strip() + if not text: + return {} + try: + obj = json.loads(text) + return obj if isinstance(obj, dict) else {"raw": obj} + except json.JSONDecodeError: + pass + m = re.search(r"\{[\s\S]*\}", text) + if m: + try: + obj = json.loads(m.group(0)) + return obj if isinstance(obj, dict) else {"raw": obj} + except json.JSONDecodeError: + return {"raw_text": text[:4000]} + return {"raw_text": text[:4000]} + + +def _load_seed_starters(path: Path, per_family: int = 2) -> List[Dict[str, Any]]: + by_cat: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + if path.is_file(): + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + cat = str(row.get("cat") or "unknown") + if len(by_cat[cat]) < per_family: + by_cat[cat].append( + { + "family": cat, + "q": row.get("q"), + "expect": row.get("expect"), + "needles": row.get("needles"), + "note": row.get("note"), + "seed_turn": row.get("turn"), + "source": "frozen_jsonl", + } + ) + starters = list(_FORCED_SEEDS) + for cat in sorted(by_cat): + starters.extend(by_cat[cat]) + return starters + + +class EdgeProbeRun: + def __init__( + self, + *, + out_root: Path, + seed_jsonl: Path, + mcp_url: str, + token: str, + model: str, + corpus: str, + tiers: Tuple[int, ...], + mcp_fail_stop: int, + dry_run: bool, + ) -> None: + self.out_root = out_root + self.seed_jsonl = seed_jsonl + self.mcp_url = mcp_url.rstrip("/") + self.token = token + self.model = model + self.corpus = corpus + self.tiers = tiers + self.mcp_fail_stop = mcp_fail_stop + self.dry_run = dry_run + + run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + self.run_dir = out_root / f"run_{run_id}" + self.q_dir = self.run_dir / "questions" + self.a_dir = self.run_dir / "answers" + self.e_dir = self.run_dir / "evals" + self.mcp_fail_dir = self.run_dir / "mcp_fails" + for d in ( + self.run_dir, + self.q_dir, + self.a_dir, + self.e_dir, + self.mcp_fail_dir, + ): + d.mkdir(parents=True, exist_ok=True) + + self.mutations_path = self.run_dir / "mutations.jsonl" + self.results_path = self.run_dir / "all_results.jsonl" + self.state_path = self.run_dir / "state.json" + self.summary_path = self.run_dir / "summary.json" + self.seed_path = self.run_dir / "seed_families.json" + self.pid_path = out_root / "runner.pid" + self.latest_path = out_root / "latest_run" + + self.starters = _load_seed_starters(seed_jsonl) + self.seed_path.write_text( + json.dumps({"starters": self.starters}, indent=2), encoding="utf-8" + ) + + self.agent: Any = None + self.agent_id: Optional[str] = None + self.turn = 0 + self.cumulative_product_bad = 0 + self.mcp_fail_count = 0 + self.consecutive_mcp_fail = 0 + self.answer_good_count = 0 + self.mcp_ok_count = 0 + self.family_bad: Counter[str] = Counter() + self.family_good: Counter[str] = Counter() + self.tier_stats: Dict[str, Dict[str, int]] = {} + self._starter_i = 0 + + self.latest_path.write_text(str(self.run_dir), encoding="utf-8") + self.pid_path.write_text(f"{os.getpid()}\n", encoding="utf-8") + + def _write_state(self, **extra: Any) -> None: + payload = { + "updated_at": _utc(), + "run_dir": str(self.run_dir), + "pid": os.getpid(), + "agent_id": self.agent_id, + "turn": self.turn, + "tiers": list(self.tiers), + "cumulative_product_bad": self.cumulative_product_bad, + "mcp_fail_count": self.mcp_fail_count, + "consecutive_mcp_fail": self.consecutive_mcp_fail, + "answer_good_count": self.answer_good_count, + "mcp_ok_count": self.mcp_ok_count, + "product_bad_stop": _PRODUCT_BAD_STOP, + "mcp_fail_stop": self.mcp_fail_stop, + "tier_stats": self.tier_stats, + **extra, + } + self.state_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + + def _append_jsonl(self, path: Path, row: Dict[str, Any]) -> None: + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(row, ensure_ascii=False, default=str) + "\n") + + def _agent_text(self, prompt: str) -> str: + if self.dry_run: + return "" + assert self.agent is not None + run = self.agent.send(prompt) + result = run.wait() + if getattr(result, "status", None) == "error": + raise RuntimeError(f"cursor run error: {getattr(result, 'id', '?')}") + # Prefer convenience helper when present + if hasattr(run, "text"): + try: + return str(run.text() or "") + except Exception: + pass + parts: List[str] = [] + try: + for message in run.messages(): + if getattr(message, "type", None) != "assistant": + continue + content = getattr(getattr(message, "message", None), "content", None) + if not content: + continue + for block in content: + if getattr(block, "type", None) == "text": + parts.append(str(getattr(block, "text", "") or "")) + except Exception: + pass + return "\n".join(parts).strip() + + def _director_context(self, tier: int, tier_target: int) -> str: + failing = [f"{k}:{v}" for k, v in self.family_bad.most_common(12)] + succeeding = [f"{k}:{v}" for k, v in self.family_good.most_common(12)] + return ( + "You are generating adversarial OpenCRE MCP `ask` probes.\n" + "Goal: find edge cases where staging ask returns a BAD answer " + "(wrong, empty, refuse, or missing structural facts).\n" + "Prioritize structural/count traps: ASVS levels, MASVS levels, " + "SAMM maturity levels, OWASP AI Exchange (AIX) layers (often 7), " + "and similar 'how many X' questions that keyword-LIKE retrieval misses.\n" + "Also mutate chapter/project/person/membership families toward typos, " + "aliases, negation, multilingual, and multi-hop traps.\n" + f"Tier={tier} progress_in_tier≈{self.turn}/{tier_target} " + f"cumulative_product_bad={self.cumulative_product_bad} " + f"mcp_fail_count={self.mcp_fail_count}.\n" + f"Failing families to EXPAND: {failing or ['(none yet)']}\n" + f"Success families to AVOID or invert: {succeeding or ['(none yet)']}\n" + ) + + def _generate_question(self, tier: int, tier_target: int) -> Dict[str, Any]: + # First pass: drain forced/frozen starters before Cursor mutations. + if self._starter_i < len(self.starters): + seed = self.starters[self._starter_i] + self._starter_i += 1 + q = { + "family": seed.get("family") or "seed", + "q": seed["q"], + "note": seed.get("note"), + "parent_q": seed.get("q"), + "mutation": "seed", + "source": seed.get("source", "forced"), + } + return q + + if self.dry_run: + fam = "dry_asvs" + return { + "family": fam, + "q": f"[dry-run] How many ASVS levels? turn={self.turn+1}", + "note": "dry-run", + "parent_q": None, + "mutation": "dry", + "source": "dry_run", + } + + prompt = ( + self._director_context(tier, tier_target) + "\nReturn ONLY JSON with keys: " + "family (string), q (string question), note (string), " + "parent_q (string|null), mutation (short tag).\n" + "Make q harder than previous successes; expand failing families.\n" + ) + raw = self._agent_text(prompt) + data = _parse_json_blob(raw) + qtext = str(data.get("q") or "").strip() + if not qtext: + qtext = "How many levels does OWASP ASVS have?" + data = { + "family": "asvs_levels", + "q": qtext, + "note": "fallback after empty agent json", + "parent_q": None, + "mutation": "fallback", + } + data["q"] = qtext + data.setdefault("family", "unknown") + data.setdefault("mutation", "agent") + data["source"] = "cursor_agent" + data["agent_raw"] = raw[:4000] + return data + + async def _mcp_ask(self, prompt: str) -> Tuple[str, Dict[str, Any]]: + """Return (status, payload). status is mcp_ok or mcp_fail.""" + if self.dry_run: + return "mcp_ok", { + "ok": True, + "channel": "dry_run", + "message": f"dry-run echo: {prompt[:200]}", + "items": [], + } + + if (os.environ.get("MCP_EDGE_PROBE_LOCAL_ASK") or "").strip().lower() in ( + "1", + "true", + "yes", + "on", + ): + return await asyncio.to_thread(self._local_ask, prompt) + + from mcp import ClientSession + from mcp.client.streamable_http import streamable_http_client + from mcp.shared._httpx_utils import create_mcp_http_client + + # MCP SDK RequestClock expects float seconds, not datetime.timedelta. + _ask_timeout_s = 90.0 + + headers = {"Authorization": f"Bearer {self.token}"} + http = create_mcp_http_client(headers=headers) + try: + async with streamable_http_client( + self.mcp_url, http_client=http, terminate_on_close=True + ) as streams: + read, write = streams[0], streams[1] + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.call_tool( + "ask", + {"prompt": prompt, "corpus": self.corpus}, + read_timeout_seconds=_ask_timeout_s, + ) + if getattr(result, "isError", None): + texts = [] + for c in getattr(result, "content", None) or []: + texts.append(getattr(c, "text", None) or str(c)) + return "mcp_fail", { + "error": "tool_isError", + "status": None, + "body": "\n".join(str(t) for t in texts)[:8000], + } + sc = getattr(result, "structuredContent", None) or getattr( + result, "structured_content", None + ) + if isinstance(sc, dict): + return "mcp_ok", sc + # Fallback: parse text content + texts = [] + for c in getattr(result, "content", None) or []: + t = getattr(c, "text", None) + if t: + texts.append(t) + blob = "\n".join(texts) + parsed = _parse_json_blob(blob) + if parsed and "raw_text" not in parsed: + return "mcp_ok", parsed + return "mcp_ok", {"message": blob[:8000], "ok": bool(blob)} + except Exception as exc: + # ExceptionGroup/TaskGroup hides the root cause in str(exc); unwrap. + body = str(exc) + sub = getattr(exc, "exceptions", None) + if sub: + parts = [f"{type(e).__name__}: {e}" for e in sub[:5]] + body = "; ".join(parts) or body + return "mcp_fail", { + "error": type(exc).__name__, + "status": getattr(exc, "status", None) + or getattr(exc, "status_code", None), + "body": body[:8000], + "traceback": traceback.format_exc()[-6000:], + } + finally: + try: + await http.aclose() + except Exception: + pass + + def _local_ask(self, prompt: str) -> Tuple[str, Dict[str, Any]]: + """In-process ask against the local Flask/DB (for post-seed probes).""" + try: + from application import create_app + from application.mcp.ask import ask + + app = create_app(mode="production") + with app.app_context(): + payload = ask(prompt, corpus=self.corpus) + if isinstance(payload, dict): + return "mcp_ok", payload + return "mcp_fail", { + "error": "non_dict_payload", + "body": str(payload)[:4000], + } + except Exception as exc: + return "mcp_fail", { + "error": type(exc).__name__, + "body": str(exc)[:8000], + "traceback": traceback.format_exc()[-6000:], + } + + def _eval_answer( + self, question: Dict[str, Any], answer: Dict[str, Any] + ) -> Dict[str, Any]: + if self.dry_run: + # Alternate good/bad for dry-run visibility + bad = (self.turn % 7) == 0 + return { + "verdict": "answer_bad" if bad else "answer_good", + "reason": "dry-run alternating verdict", + "confidence": 0.5, + } + + prompt = ( + "You evaluate OpenCRE MCP `ask` answers for factual adequacy.\n" + "Question JSON:\n" + f"{json.dumps(question, ensure_ascii=False)[:3000]}\n\n" + "Answer JSON from MCP ask:\n" + f"{json.dumps(answer, ensure_ascii=False)[:6000]}\n\n" + "Return ONLY JSON with keys:\n" + ' verdict: "answer_good" | "answer_bad"\n' + " reason: short string\n" + " confidence: 0..1\n" + "Mark answer_bad if wrong, empty, refuse, off-topic, or missing " + "the structural fact the question asked (counts/levels/layers).\n" + "Infra/transport errors are NOT your job (already classified).\n" + ) + raw = self._agent_text(prompt) + data = _parse_json_blob(raw) + verdict = str(data.get("verdict") or "").strip().lower() + if verdict not in ("answer_good", "answer_bad"): + # Heuristic fallback + msg = json.dumps(answer).lower() + if not answer or answer.get("ok") is False: + verdict = "answer_bad" + elif "i don't" in msg or "not sure" in msg: + verdict = "answer_bad" + else: + verdict = "answer_good" + data["verdict"] = verdict + data.setdefault("reason", "normalized from agent output") + data["agent_raw"] = raw[:4000] + return data + + def _one_turn(self, tier: int, tier_target: int) -> Dict[str, Any]: + self.turn += 1 + turn = self.turn + question = self._generate_question(tier, tier_target) + family = str(question.get("family") or "unknown") + + q_path = self.q_dir / f"{turn:05d}.json" + q_row = { + "turn": turn, + "tier": tier, + "tier_target": tier_target, + "ts": _utc(), + **question, + } + q_path.write_text(json.dumps(q_row, indent=2), encoding="utf-8") + if question.get("parent_q") and question.get("mutation") not in ( + "seed", + None, + ): + self._append_jsonl( + self.mutations_path, + { + "turn": turn, + "family": family, + "old_q": question.get("parent_q"), + "new_q": question.get("q"), + "mutation": question.get("mutation"), + }, + ) + + # MCP only after question is on disk + mcp_status, mcp_body = asyncio.run(self._mcp_ask(str(question["q"]))) + a_path = self.a_dir / f"{turn:05d}.json" + a_row = { + "turn": turn, + "tier": tier, + "ts": _utc(), + "mcp_status": mcp_status, + "prompt": question.get("q"), + "response": mcp_body, + } + a_path.write_text(json.dumps(a_row, indent=2, default=str), encoding="utf-8") + + answer_verdict: Optional[str] = None + eval_row: Optional[Dict[str, Any]] = None + if mcp_status == "mcp_fail": + self.mcp_fail_count += 1 + self.consecutive_mcp_fail += 1 + fail_path = self.mcp_fail_dir / f"{turn:05d}.json" + fail_detail = { + "turn": turn, + "tier": tier, + "ts": _utc(), + "question_path": str(q_path), + "prompt": question.get("q"), + "error": mcp_body.get("error"), + "status": mcp_body.get("status"), + "body": mcp_body.get("body") or mcp_body, + "mcp_url": self.mcp_url, + } + fail_path.write_text( + json.dumps(fail_detail, indent=2, default=str), encoding="utf-8" + ) + _loud_mcp_fail(turn, fail_detail) + else: + self.mcp_ok_count += 1 + self.consecutive_mcp_fail = 0 + eval_row = self._eval_answer(question, mcp_body) + answer_verdict = str(eval_row.get("verdict") or "answer_bad") + e_path = self.e_dir / f"{turn:05d}.json" + e_path.write_text( + json.dumps( + {"turn": turn, "tier": tier, "ts": _utc(), **eval_row}, + indent=2, + default=str, + ), + encoding="utf-8", + ) + if answer_verdict == "answer_bad": + self.cumulative_product_bad += 1 + self.family_bad[family] += 1 + else: + self.answer_good_count += 1 + self.family_good[family] += 1 + + result = { + "turn": turn, + "tier": tier, + "tier_target": tier_target, + "ts": _utc(), + "family": family, + "q": question.get("q"), + "mcp_status": mcp_status, + "answer_verdict": answer_verdict, + "cumulative_product_bad": self.cumulative_product_bad, + "mcp_fail_count": self.mcp_fail_count, + "question_path": str(q_path), + "answer_path": str(a_path), + } + self._append_jsonl(self.results_path, result) + self._write_state(current_tier=tier, tier_target=tier_target) + _log( + f"turn={turn} tier={tier} mcp={mcp_status} " + f"verdict={answer_verdict} product_bad={self.cumulative_product_bad} " + f"mcp_fail={self.mcp_fail_count} | {(question.get('q') or '')[:80]}" + ) + return result + + def run(self) -> int: + from cursor_sdk import Agent, LocalAgentOptions + + api_key = os.environ.get("CURSOR_API_KEY") or "" + if not self.dry_run and not api_key: + _log("ERROR: CURSOR_API_KEY required (or pass --dry-run)") + return 2 + local_ask = ( + os.environ.get("MCP_EDGE_PROBE_LOCAL_ASK") or "" + ).strip().lower() in ( + "1", + "true", + "yes", + "on", + ) + if not self.dry_run and not self.token and not local_ask: + _log("ERROR: OPENCRE_STAGING_ADMIN_API_TOKEN required for remote ask") + return 2 + + stop_reason = "completed_all_tiers" + try: + if self.dry_run: + self.agent = None + self.agent_id = "dry-run" + else: + self.agent = Agent.create( + model=self.model, + api_key=api_key, + name="opencre-mcp-edge-probe", + local=LocalAgentOptions(cwd=str(_REPO)), + ) + self.agent_id = getattr(self.agent, "agent_id", None) or getattr( + self.agent, "id", None + ) + _log(f"cursor agent id={self.agent_id}") + + for tier in self.tiers: + tier_key = str(tier) + self.tier_stats[tier_key] = { + "target": tier, + "turns": 0, + "mcp_ok": 0, + "mcp_fail": 0, + "answer_good": 0, + "answer_bad": 0, + } + _log(f"=== TIER {tier} start (global_turn={self.turn}) ===") + while self.turn < tier: + row = self._one_turn(tier, tier) + st = self.tier_stats[tier_key] + st["turns"] += 1 + if row["mcp_status"] == "mcp_fail": + st["mcp_fail"] += 1 + else: + st["mcp_ok"] += 1 + if row["answer_verdict"] == "answer_bad": + st["answer_bad"] += 1 + else: + st["answer_good"] += 1 + + if self.cumulative_product_bad > _PRODUCT_BAD_STOP: + stop_reason = f"product_bad_stop>{_PRODUCT_BAD_STOP}" + _log(stop_reason) + break + if ( + self.mcp_fail_stop > 0 + and self.consecutive_mcp_fail >= self.mcp_fail_stop + ): + stop_reason = f"mcp_fail_consecutive>={self.mcp_fail_stop}" + _log(stop_reason) + break + self._write_summary(stop_reason) + if stop_reason != "completed_all_tiers" and not stop_reason.startswith( + "completed" + ): + if "stop" in stop_reason: + break + else: + stop_reason = "completed_all_tiers" + finally: + if self.agent is not None: + try: + self.agent.close() + except Exception: + pass + self._write_summary(stop_reason) + _log(f"DONE reason={stop_reason} summary={self.summary_path}") + + return 0 if stop_reason.startswith("completed") else 1 + + def _write_summary(self, stop_reason: str) -> None: + summary = { + "finished_at": _utc(), + "stop_reason": stop_reason, + "run_dir": str(self.run_dir), + "agent_id": self.agent_id, + "turns": self.turn, + "mcp_ok_count": self.mcp_ok_count, + "mcp_fail_count": self.mcp_fail_count, + "answer_good_count": self.answer_good_count, + "cumulative_product_bad": self.cumulative_product_bad, + "product_bad_stop": _PRODUCT_BAD_STOP, + "mcp_fail_stop": self.mcp_fail_stop, + "tier_stats": self.tier_stats, + "failing_families": self.family_bad.most_common(30), + "success_families": self.family_good.most_common(30), + "mcp_url": self.mcp_url, + } + self.summary_path.write_text(json.dumps(summary, indent=2), encoding="utf-8") + self._write_state(stop_reason=stop_reason) + + +def _parse_tiers(raw: str) -> Tuple[int, ...]: + parts = [int(x.strip()) for x in raw.split(",") if x.strip()] + if not parts: + return _TIERS_DEFAULT + return tuple(parts) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--dry-run", action="store_true") + ap.add_argument( + "--tiers", + default=os.environ.get( + "MCP_EDGE_PROBE_TIERS", ",".join(str(t) for t in _TIERS_DEFAULT) + ), + ) + args = ap.parse_args() + out = Path(os.environ.get("MCP_EDGE_PROBE_OUT") or _DEFAULT_OUT) + out.mkdir(parents=True, exist_ok=True) + runner = EdgeProbeRun( + out_root=out, + seed_jsonl=Path(os.environ.get("MCP_EDGE_PROBE_SEED_JSONL") or _DEFAULT_SEED), + mcp_url=os.environ.get("MCP_EDGE_PROBE_URL", "https://staging.opencre.org/mcp"), + token=(os.environ.get("OPENCRE_STAGING_ADMIN_API_TOKEN") or "").strip(), + model=os.environ.get("MCP_EDGE_PROBE_MODEL") or "composer-2.5", + corpus=(os.environ.get("MCP_EDGE_PROBE_CORPUS") or "auto").strip(), + tiers=_parse_tiers(args.tiers), + mcp_fail_stop=int(os.environ.get("MCP_EDGE_PROBE_MCP_FAIL_STOP") or "0"), + dry_run=bool(args.dry_run), + ) + return runner.run() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/mcp_edge_probe_small24_rerun.py b/scripts/mcp_edge_probe_small24_rerun.py new file mode 100644 index 000000000..c23b26e6b --- /dev/null +++ b/scripts/mcp_edge_probe_small24_rerun.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +"""One-shot re-ask of the 24 non-structural edge probes against staging MCP ask. + +Loads ``tmp/mcp_edge_probe/small24_questions.jsonl``, calls staging Streamable +HTTP ``ask``, writes results under ``tmp/mcp_edge_probe/small24_rerun_/``. + +Env: + OPENCRE_STAGING_ADMIN_API_TOKEN required (or --env-file) + MCP_EDGE_PROBE_URL default https://staging.opencre.org/mcp + CURSOR_API_KEY optional; enables Cursor SDK eval + +Heuristic eval (always): compares answers to prior product_bad failure modes. +Optional Cursor eval when CURSOR_API_KEY is set. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import sys +import traceback +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +_REPO = Path(__file__).resolve().parents[1] + + +def _utc() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _utc_stamp() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def _log(msg: str) -> None: + print(f"[{_utc()}] {msg}", flush=True) + + +def _load_env_file(path: Path) -> None: + if not path.is_file(): + return + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + k = k.strip() + v = v.strip().strip('"').strip("'") + if k and k not in os.environ: + os.environ[k] = v + + +def _msg(answer: Dict[str, Any]) -> str: + if not isinstance(answer, dict): + return str(answer) + return str( + answer.get("message") + or (answer.get("response") if isinstance(answer.get("response"), str) else "") + or "" + ) + + +def _heuristic_eval(family: str, q: str, answer: Dict[str, Any]) -> Dict[str, Any]: + """Judge against the known product_bad failure modes for these 24.""" + msg = _msg(answer).lower() + ok = bool(answer.get("ok")) + year_now = datetime.now(timezone.utc).year + reasons: List[str] = [] + + def bad(reason: str) -> Dict[str, Any]: + return { + "verdict": "answer_bad", + "reason": reason, + "confidence": 0.7, + "eval_mode": "heuristic", + } + + def good(reason: str) -> Dict[str, Any]: + return { + "verdict": "answer_good", + "reason": reason, + "confidence": 0.65, + "eval_mode": "heuristic", + } + + if ( + not ok + and "clarify" not in msg + and family + not in ( + "trap_clarify", + "novel_temporal", + ) + ): + return bad(f"ok=false without useful clarify: {msg[:180]}") + + if family == "board_edge": + if str(year_now) in msg and "2024" not in msg and "2023" not in msg: + return good(f"lists {year_now} board") + if re.search(rf"\b{year_now}\b", msg): + return good(f"mentions {year_now}") + return bad("this-year board should use current calendar year roster") + + if family == "novel_temporal": + if "2099" in msg and ( + "no " in msg or "not indexed" in msg or "fail closed" in msg + ): + return good("fail-closed on 2099") + if "2026" in msg or "2025" in msg or "2024" in msg: + return bad("substituted another year for 2099") + return bad("did not fail closed on future year") + + if family in ("events_edge", "novel_events"): + if ( + "meetup" in msg + or "event" in msg + or "upcoming" in msg + or "not indexed" in msg + ): + if ( + "leaders:" in msg + and "event" not in msg + and "meetup" not in msg + and "upcoming" not in msg + and "not indexed" not in msg + ): + return bad("chapter-only, no events framing") + return good("events/meetup framing present") + return bad("no events/meetup language") + + if family == "novel_confuse": + if "foundation board" in msg and "asvs" not in msg: + return bad("answered foundation board for ASVS board ask") + if "leader" in msg or "asvs" in msg: + return good("ASVS project leadership path") + return bad("missing ASVS leaders / clarification") + + if family == "novel_cre_bait": + if "cre" in msg and ("not" in msg or "no " in msg or "metadata" in msg): + return good("explains no CRE mapping for community entity") + return bad("did not refuse CRE mapping for project/chapter") + + if family == "novel_multi": + if "london" in q.lower() and "berlin" in q.lower(): + if "london" in msg and "berlin" in msg: + return good("both chapters mentioned") + return bad("compare missed one chapter") + if "juice" in q.lower() and "board" in q.lower(): + if ( + "juice" in msg or "kimminich" in msg or "hollenbach" in msg + ) and "board" in msg: + return good("dual project+board") + return bad("dual ask incomplete") + return bad("multi-intent incomplete") + + if family == "novel_negation": + if "atlantis" in msg and ( + "no " in msg or "not " in msg or "not indexed" in msg + ): + return good("names nonexistent Atlantis") + return bad("did not identify Atlantis as nonexistent") + + if family == "novel_noise": + if "asvs" in msg and ( + "verification" in msg + or "requirement" in msg + or "standard" in msg + or "leader" in msg + ): + if ( + "leader" in msg + and "verification" not in msg + and "requirement" not in msg + and "standard" not in msg + ): + # leaders-only is weak but better than refuse + return bad("thin ASVS description (leaders only)") + return good("ASVS described") + return bad("thin/missing ASVS description") + + if family == "novel_slug": + if "juice" in msg and ( + "vulnerable" in msg or "training" in msg or "flagship" in msg + ): + return good("juice shop identity") + return bad("slug ask missing juice shop purpose") + + if family == "novel_typo": + if "kimminich" in msg and "hollenbach" in msg: + return good("both Juice Shop leaders") + if "kimminich" in msg or "hollenbach" in msg: + return bad("incomplete Juice Shop leaders") + return bad("JuiceShop typo did not resolve to project leaders") + + if family == "people_edge": + if "sherif" in msg and ("london" in msg or "chapter" in msg or "leader" in msg): + return good("person roles beyond single board year") + if "2020" in msg and "london" not in msg: + return bad("only old board_member card") + return bad("incomplete person card") + + if family == "project_edge": + if "top ten" in q.lower() or "top-ten" in q.lower(): + leaders = sum( + 1 + for n in ( + "glas", + "van der stock", + "smithline", + "gigler", + "xue", + "andrew", + "brian", + "neil", + "torsten", + ) + if n in msg + ) + if leaders >= 2: + return good("multiple Top Ten leaders") + return bad("incomplete Top Ten leaders") + if "juice" in q.lower(): + if "vulnerable" in msg or "training" in msg: + return good("juice shop purpose") + return bad("juice shop missing purpose blurb") + return bad("project edge miss") + + if family == "trap_clarify": + if "which" in msg or "city" in msg or "clarify" in msg or "please name" in msg: + if "natori" in msg or "brisbane" in msg: + return bad("guessed a chapter instead of clarifying") + return good("asked for place clarification") + return bad("did not clarify ambiguous ask") + + if not ok: + return bad(msg[:200] or "ok=false") + return good("generic ok") + + +def _parse_json_blob(text: str) -> Dict[str, Any]: + text = (text or "").strip() + if not text: + return {} + try: + obj = json.loads(text) + return obj if isinstance(obj, dict) else {"raw": obj} + except json.JSONDecodeError: + pass + m = re.search(r"\{[\s\S]*\}", text) + if m: + try: + obj = json.loads(m.group(0)) + return obj if isinstance(obj, dict) else {"raw": obj} + except json.JSONDecodeError: + return {"raw_text": text[:4000]} + return {"raw_text": text[:4000]} + + +async def _mcp_ask( + prompt: str, *, mcp_url: str, token: str +) -> Tuple[str, Dict[str, Any]]: + # Mirror scripts/mcp_edge_probe_runner.py client wiring. + from mcp import ClientSession + from mcp.client.streamable_http import streamable_http_client + from mcp.shared._httpx_utils import create_mcp_http_client + + headers = {"Authorization": f"Bearer {token}"} + http = create_mcp_http_client(headers=headers) + try: + async with streamable_http_client( + mcp_url, http_client=http, terminate_on_close=True + ) as streams: + read, write = streams[0], streams[1] + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.call_tool( + "ask", + {"prompt": prompt, "corpus": "owasp_meta"}, + read_timeout_seconds=90.0, + ) + if getattr(result, "isError", None): + texts = [] + for c in getattr(result, "content", None) or []: + texts.append(getattr(c, "text", None) or str(c)) + return "mcp_fail", { + "error": "tool_isError", + "body": "\n".join(str(t) for t in texts)[:8000], + } + sc = getattr(result, "structuredContent", None) or getattr( + result, "structured_content", None + ) + if isinstance(sc, dict): + return "mcp_ok", sc + texts = [] + for c in getattr(result, "content", None) or []: + t = getattr(c, "text", None) + if t: + texts.append(t) + blob = "\n".join(texts) + parsed = _parse_json_blob(blob) + if parsed and "raw_text" not in parsed: + return "mcp_ok", parsed + return "mcp_ok", {"message": blob[:8000], "ok": bool(blob)} + except Exception as exc: + body = str(exc) + sub = getattr(exc, "exceptions", None) + if sub: + parts = [f"{type(e).__name__}: {e}" for e in sub[:5]] + body = "; ".join(parts) or body + return "mcp_fail", { + "error": type(exc).__name__, + "detail": body[:2000], + "traceback": traceback.format_exc()[-4000:], + } + finally: + try: + await http.aclose() + except Exception: + pass + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument( + "--questions", + type=Path, + default=_REPO / "tmp" / "mcp_edge_probe" / "small24_questions.jsonl", + ) + ap.add_argument( + "--out-root", + type=Path, + default=_REPO / "tmp" / "mcp_edge_probe", + ) + ap.add_argument( + "--env-file", + type=Path, + default=_REPO / "tmp" / "mcp_edge_probe" / ".staging_env", + ) + ap.add_argument( + "--mcp-url", + default=os.environ.get("MCP_EDGE_PROBE_URL", "https://staging.opencre.org/mcp"), + ) + args = ap.parse_args() + + _load_env_file(args.env_file) + # Also try cursor-chat-bridge for CURSOR_API_KEY without printing. + _load_env_file(Path("/Users/sg/Projects/cursor-chat-bridge/.env")) + + token = (os.environ.get("OPENCRE_STAGING_ADMIN_API_TOKEN") or "").strip() + if not token: + _log("ERROR: OPENCRE_STAGING_ADMIN_API_TOKEN missing") + return 2 + + questions: List[Dict[str, Any]] = [] + for line in args.questions.read_text(encoding="utf-8").splitlines(): + if line.strip(): + questions.append(json.loads(line)) + + run_dir = args.out_root / f"small24_rerun_{_utc_stamp()}" + a_dir = run_dir / "answers" + e_dir = run_dir / "evals" + q_dir = run_dir / "questions" + for d in (a_dir, e_dir, q_dir): + d.mkdir(parents=True, exist_ok=True) + + results_path = run_dir / "all_results.jsonl" + good = bad = mcp_fail = 0 + rows: List[Dict[str, Any]] = [] + + for i, item in enumerate(questions, start=1): + q = str(item.get("q") or "") + family = str(item.get("family") or "unknown") + orig_turn = item.get("orig_turn") + q_path = q_dir / f"{i:05d}.json" + q_path.write_text(json.dumps(item, indent=2), encoding="utf-8") + + mcp_status, body = asyncio.run(_mcp_ask(q, mcp_url=args.mcp_url, token=token)) + a_path = a_dir / f"{i:05d}.json" + a_row = { + "turn": i, + "orig_turn": orig_turn, + "family": family, + "ts": _utc(), + "mcp_status": mcp_status, + "prompt": q, + "response": body, + } + a_path.write_text(json.dumps(a_row, indent=2, default=str), encoding="utf-8") + + if mcp_status == "mcp_fail": + mcp_fail += 1 + verdict = None + eval_row = { + "verdict": None, + "reason": "mcp_fail", + "error": body, + "eval_mode": "n/a", + } + else: + eval_row = _heuristic_eval(family, q, body) + verdict = eval_row["verdict"] + if verdict == "answer_good": + good += 1 + else: + bad += 1 + + e_path = e_dir / f"{i:05d}.json" + e_path.write_text( + json.dumps( + {"turn": i, "family": family, "ts": _utc(), **eval_row}, indent=2 + ), + encoding="utf-8", + ) + row = { + "turn": i, + "orig_turn": orig_turn, + "family": family, + "q": q, + "mcp_status": mcp_status, + "answer_verdict": verdict, + "reason": eval_row.get("reason"), + "message_preview": _msg(body)[:240] if mcp_status == "mcp_ok" else "", + } + rows.append(row) + with results_path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(row, ensure_ascii=False) + "\n") + _log( + f"t={i} orig={orig_turn} family={family} mcp={mcp_status} " + f"verdict={verdict} | {q[:70]!r}" + ) + + summary = { + "finished_at": _utc(), + "run_dir": str(run_dir), + "mcp_url": args.mcp_url, + "n": len(questions), + "mcp_ok": len(questions) - mcp_fail, + "mcp_fail": mcp_fail, + "answer_good": good, + "answer_bad": bad, + "note": ( + "Heuristic eval against prior product_bad modes. " + "Staging may not yet include local router fixes until deploy." + ), + "rows": rows, + } + (run_dir / "summary.json").write_text( + json.dumps(summary, indent=2), encoding="utf-8" + ) + _log(f"DONE run_dir={run_dir} good={good} bad={bad} mcp_fail={mcp_fail}") + print( + json.dumps( + {"run_dir": str(run_dir), "good": good, "bad": bad, "mcp_fail": mcp_fail} + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/owasp_agent_adversarial_asker.py b/scripts/owasp_agent_adversarial_asker.py index 067b46c78..116db1a33 100644 --- a/scripts/owasp_agent_adversarial_asker.py +++ b/scripts/owasp_agent_adversarial_asker.py @@ -18,7 +18,10 @@ OWASP_PROBE_COMPLETION_MODE default mcp_ask OWASP_ADV_MAX_TURNS default 2000 OWASP_ADV_OUT default tmp/owasp_agent_adversarial_failures.jsonl - OWASP_ADV_SEED optional int + OWASP_ADV_SEED optional int (default 14) + OWASP_ADV_EXPORT_QUESTIONS if 1, write frozen questions JSONL and exit (no MCP) + OWASP_ADV_QUESTIONS_OUT export path (default tmp/owasp_mcp_ask_adv_5k_questions.jsonl) + OWASP_ADV_QUESTIONS_FILE if set, replay frozen questions instead of bank+mutate OWASP_ADV_MCP_TRANSPORT staging_ssh (default) | local OPENCRE_BASE_URL for local MCP / mcp_inprocess (default http://127.0.0.1:15001) @@ -43,6 +46,7 @@ _REPO = Path(__file__).resolve().parents[1] _DEFAULT_OUT = _REPO / "tmp" / "owasp_agent_adversarial_failures.jsonl" +_DEFAULT_QUESTIONS_OUT = _REPO / "tmp" / "owasp_mcp_ask_adv_5k_questions.jsonl" REFUSAL_MARKERS = ( "which city are you in", @@ -691,6 +695,33 @@ def _format_ask_result(result: Any) -> str: # --- MCP stdio (persistent) ----------------------------------------------- +def _staging_docker_exec_args() -> List[str]: + """Args after ``docker`` for mcp stdio on the staging compose stack.""" + return [ + "exec", + "-i", + "-e", + "OPENCRE_BASE_URL=http://web:5001", + "-e", + "PYTHONPATH=/code", + "-e", + "PYTHONUNBUFFERED=1", + "-e", + "OWASP_AGENT_ENABLED=1", + "-e", + f"OWASP_AGENT_LLM_PICK={os.environ.get('OWASP_AGENT_LLM_PICK') or '1'}", + "-e", + f"OWASP_AGENT_TOP_K={os.environ.get('OWASP_AGENT_TOP_K') or '8'}", + "-w", + "/code", + # mcp-1 has the MCP overlay (ask tool + openapi); web-1 image alone may not. + "opencre-staging-mcp-1", + "python", + "-m", + "application.mcp", + ] + + def _staging_ssh_params() -> Any: from mcp import StdioServerParameters @@ -703,30 +734,21 @@ def _staging_ssh_params() -> Any: "ConnectTimeout=30", "sg@100.74.143.62", "docker", - "exec", - "-i", - "-e", - "OPENCRE_BASE_URL=http://127.0.0.1:5001", - "-e", - "PYTHONPATH=/code", - "-e", - "PYTHONUNBUFFERED=1", - "-e", - "OWASP_AGENT_ENABLED=1", - "-e", - f"OWASP_AGENT_LLM_PICK={os.environ.get('OWASP_AGENT_LLM_PICK') or '1'}", - "-e", - f"OWASP_AGENT_TOP_K={os.environ.get('OWASP_AGENT_TOP_K') or '8'}", - "-w", - "/code", - "opencre-staging-web-1", - "python", - "-m", - "application.mcp", + *_staging_docker_exec_args(), ], ) +def _staging_docker_params() -> Any: + """Run on the staging host itself (no SSH hop).""" + from mcp import StdioServerParameters + + return StdioServerParameters( + command="docker", + args=_staging_docker_exec_args(), + ) + + def _local_mcp_params() -> Any: from mcp import StdioServerParameters @@ -772,7 +794,12 @@ async def _start_async(self) -> None: from mcp.client.stdio import stdio_client transport = (os.environ.get("OWASP_ADV_MCP_TRANSPORT") or "staging_ssh").strip() - params = _local_mcp_params() if transport == "local" else _staging_ssh_params() + if transport == "local": + params = _local_mcp_params() + elif transport in ("staging_docker", "docker"): + params = _staging_docker_params() + else: + params = _staging_ssh_params() self._cm_stdio = stdio_client(params) read, write = await self._cm_stdio.__aenter__() self._cm_session = ClientSession(read, write) @@ -932,6 +959,81 @@ def _load_prior_failures(out_path: Path) -> List[Dict[str, Any]]: return rows +def _export_questions( + *, + seed: int, + max_turns: int, + start_turn: int, + out_path: Path, +) -> int: + """Write frozen questions for turns [start_turn, max_turns]; no MCP calls.""" + rng = random.Random(seed) + bank = _bank(rng) + out_path.parent.mkdir(parents=True, exist_ok=True) + n = 0 + with out_path.open("w", encoding="utf-8") as fh: + for turn in range(start_turn, max_turns + 1): + case = bank[(turn - 1) % len(bank)] + q = _mutate_question(case.q, turn, rng) + row = { + "turn": turn, + "seed": seed, + "cat": case.cat, + "q": q, + "expect": case.expect, + "needles": list(case.needles), + "note": case.note, + } + fh.write(json.dumps(row, ensure_ascii=False) + "\n") + n += 1 + print( + json.dumps( + { + "exported": True, + "path": str(out_path), + "turns": n, + "seed": seed, + "start_turn": start_turn, + "max_turns": max_turns, + "bank": len(bank), + } + ), + flush=True, + ) + return 0 + + +def _load_frozen_questions(path: Path) -> List[Case]: + if not path.is_file(): + raise FileNotFoundError(f"OWASP_ADV_QUESTIONS_FILE not found: {path}") + cases: List[Case] = [] + for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"bad JSONL at {path}:{line_no}: {exc}") from exc + q = str(row.get("q") or "").strip() + if not q: + raise ValueError(f"missing q at {path}:{line_no}") + needles_raw = row.get("needles") or [] + if not isinstance(needles_raw, list): + raise ValueError(f"needles must be a list at {path}:{line_no}") + cases.append( + Case( + q=q, + expect=str(row.get("expect") or "must_not_refuse"), + needles=tuple(str(x) for x in needles_raw), + note=str(row.get("note") or "frozen"), + cat=str(row.get("cat") or "frozen"), + ) + ) + if not cases: + raise ValueError(f"empty questions file: {path}") + return cases + + def main() -> int: seed = int(os.environ.get("OWASP_ADV_SEED") or "14") rng = random.Random(seed) @@ -943,9 +1045,42 @@ def main() -> int: mode = (os.environ.get("OWASP_PROBE_COMPLETION_MODE") or "mcp_ask").strip().lower() os.environ.setdefault("OWASP_AGENT_ENABLED", "1") - bank = _bank(rng) + if (os.environ.get("OWASP_ADV_EXPORT_QUESTIONS") or "").strip() == "1": + export_path = Path( + os.environ.get("OWASP_ADV_QUESTIONS_OUT") or str(_DEFAULT_QUESTIONS_OUT) + ) + return _export_questions( + seed=seed, + max_turns=max_turns, + start_turn=start_turn, + out_path=export_path, + ) + + questions_file = (os.environ.get("OWASP_ADV_QUESTIONS_FILE") or "").strip() + frozen: Optional[List[Case]] = None + if questions_file: + frozen_all = _load_frozen_questions(Path(questions_file)) + # Allow batch windows over a frozen file (START_TURN / MAX_TURNS inclusive). + start_turn = max(1, start_turn) + max_turns = min(max_turns, len(frozen_all)) + if start_turn > max_turns: + print( + json.dumps( + { + "error": "empty frozen window", + "start_turn": start_turn, + "max_turns": max_turns, + "file_turns": len(frozen_all), + } + ), + flush=True, + ) + return 2 + frozen = frozen_all[start_turn - 1 : max_turns] + + bank = _bank(rng) if frozen is None else [] cats_raw = (os.environ.get("OWASP_ADV_CATS") or "").strip() - if cats_raw: + if cats_raw and frozen is None: allow = {c.strip() for c in cats_raw.split(",") if c.strip()} bank = [c for c in bank if c.cat in allow] if not bank: @@ -974,7 +1109,9 @@ def main() -> int: "max_turns": max_turns, "start_turn": start_turn, "append": append, - "bank": len(bank), + "bank": len(bank) if frozen is None else None, + "questions_file": questions_file or None, + "frozen_turns": len(frozen) if frozen is not None else None, "cats": cats_raw or None, "mode": mode, "mcp_transport": os.environ.get( @@ -990,8 +1127,12 @@ def main() -> int: try: with out_path.open("a" if append else "w") as fh: for turn in range(start_turn, max_turns + 1): - case = bank[(turn - 1) % len(bank)] - q = _mutate_question(case.q, turn, rng) + if frozen is not None: + case = frozen[turn - start_turn] + q = case.q + else: + case = bank[(turn - 1) % len(bank)] + q = _mutate_question(case.q, turn, rng) t0 = time.time() answer = ask(q) dt = round(time.time() - t0, 2) @@ -1065,6 +1206,7 @@ def main() -> int: "unique_substantive": len(promote), "failures_path": str(out_path), "mode": mode, + "questions_file": questions_file or None, "failure_ids": [r["id"] for r in failures], } summary_path.write_text(json.dumps(summary, indent=2)) diff --git a/scripts/owasp_agent_adversarial_batch_run.py b/scripts/owasp_agent_adversarial_batch_run.py index 44a3002e3..0c02eedb8 100644 --- a/scripts/owasp_agent_adversarial_batch_run.py +++ b/scripts/owasp_agent_adversarial_batch_run.py @@ -46,6 +46,9 @@ def main() -> int: "OWASP_AGENT_ENABLED": "1", "PYTHONUNBUFFERED": "1", } + qfile = (os.environ.get("OWASP_ADV_QUESTIONS_FILE") or "").strip() + if qfile: + env_base["OWASP_ADV_QUESTIONS_FILE"] = qfile with log_path.open("a") as log: t = start diff --git a/scripts/seed_node_facts.py b/scripts/seed_node_facts.py new file mode 100644 index 000000000..1717254b9 --- /dev/null +++ b/scripts/seed_node_facts.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +"""Surgically seed node_facts into document_metadata + re-embed evidence only. + +Applies only curated ASVS / MASVS / SAMM / OWASP AI Exchange / CRE facts from +``application.utils.owasp_agent.node_facts``, then re-embeds *only* evidence +URLs. Nest project meta rows are never re-embedded with a facts appendix +(structure answers come from evidence docs, not Nest project cards). + +Usage (local DB):: + + python scripts/seed_node_facts.py + python scripts/seed_node_facts.py --dry-run + python scripts/seed_node_facts.py --skip-embed + +Env: standard Flask/OpenCRE DB URL (same as the running app). +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Set + +_REPO = Path(__file__).resolve().parents[1] +if str(_REPO) not in sys.path: + sys.path.insert(0, str(_REPO)) + + +def _family_node_names() -> Dict[str, List[str]]: + """DB Node.name values to match per registry family.""" + return { + "ASVS": ["ASVS", "OWASP ASVS"], + "MASVS": ["MASVS", "OWASP MASVS"], + "SAMM": ["SAMM", "OWASP SAMM"], + "OWASP AI Exchange": [ + "OWASP AI Exchange", + "AI Exchange", + "AIX", + ], + } + + +def _seed_nodes(database: Any, *, dry_run: bool) -> Dict[str, int]: + from application.database import db as db_mod + from application.utils.owasp_agent.node_facts import ( + NODE_FACTS_BY_CRE, + NODE_FACTS_BY_FAMILY, + merge_facts_into_metadata, + ) + from sqlalchemy.orm.attributes import flag_modified + + updated_nodes = 0 + updated_cres = 0 + session = database.session + + for family, facts in NODE_FACTS_BY_FAMILY.items(): + names = _family_node_names().get(family, [family]) + seen_ids: Set[int] = set() + for name in names: + rows = session.query(db_mod.Node).filter(db_mod.Node.name == name).all() + # Also match case-insensitive exact when no exact hits. + if not rows: + rows = ( + session.query(db_mod.Node) + .filter(db_mod.Node.name.ilike(name)) + .all() + ) + for row in rows: + if row.id in seen_ids: + continue + seen_ids.add(row.id) + meta = dict(row.metadata_json or {}) + new_meta = merge_facts_into_metadata(meta, facts) + if new_meta == meta: + continue + print( + f" node id={row.id} name={row.name!r} " + f"facts_keys={list(facts.keys())}", + flush=True, + ) + if not dry_run: + row.metadata_json = new_meta + flag_modified(row, "metadata_json") + updated_nodes += 1 + + for cre_id, facts in NODE_FACTS_BY_CRE.items(): + # CRE.external_id is often NNN-NNN without prefix. + bare = cre_id.replace("CRE-", "") + rows = ( + session.query(db_mod.CRE) + .filter( + (db_mod.CRE.external_id == cre_id) | (db_mod.CRE.external_id == bare) + ) + .all() + ) + for row in rows: + meta = dict(row.metadata_json or {}) + new_meta = merge_facts_into_metadata(meta, facts) + if new_meta == meta: + continue + print( + f" cre id={row.id} external_id={row.external_id!r} " + f"facts_keys={list(facts.keys())}", + flush=True, + ) + if not dry_run: + row.metadata_json = new_meta + flag_modified(row, "metadata_json") + updated_cres += 1 + + if not dry_run: + session.commit() + return {"nodes": updated_nodes, "cres": updated_cres} + + +def _require_source_channel_column(database: Any) -> Optional[str]: + """Return None if embeddings.source_channel exists; else an error message. + + Refuses ad-hoc ALTER — require an Alembic migration / existing column. + """ + try: + bind = database.session.get_bind() + dialect = getattr(getattr(bind, "dialect", None), "name", "") or "" + if dialect != "postgresql": + return ( + f"dialect={dialect!r} (need postgresql + " + "embeddings.source_channel / pgvector)" + ) + from sqlalchemy import inspect + + cols = {c["name"] for c in inspect(bind).get_columns("embeddings")} + if "source_channel" not in cols: + return ( + "embeddings.source_channel is missing. " + "Run Alembic migrations (make migrate-upgrade) — " + "this script will not ALTER TABLE." + ) + except Exception as exc: + return f"cannot inspect bind ({exc})" + return None + + +def _reembed_evidence( + database: Any, *, dry_run: bool, skip_embed: bool +) -> Dict[str, int]: + if skip_embed: + return {"evidence": 0} + err = _require_source_channel_column(database) + if err: + print(f" skip embed: {err}", flush=True) + return {"evidence": 0, "skipped_reason": err} + + from application.prompt_client import prompt_client + from application.utils.owasp_agent.lane_similarity import CHANNEL_AGENT + from application.utils.owasp_agent.meta_embeddings import ( + DOC_TYPE_EVIDENCE, + evidence_documents, + evidence_url, + upsert_channel_embedding, + ) + + handler = prompt_client.PromptHandler(database, load_all_embeddings=False) + + def _embed(text: str): + return handler.get_text_embeddings(text) + + evidence_n = 0 + for slug, embed_text, citations in evidence_documents(): + url = evidence_url(slug) + body = embed_text + if citations: + body += "\n\nCitations: " + ", ".join(citations) + print(f" evidence {url}", flush=True) + if not dry_run: + vec = list(_embed(body)) + upsert_channel_embedding( + database, + source_channel=CHANNEL_AGENT, + doc_type=DOC_TYPE_EVIDENCE, + embeddings_url=url, + embedding_text=body, + embeddings=vec, + ) + evidence_n += 1 + + # Intentionally do not re-embed Nest project meta with facts appendix — + # that boosts Nest similarity for structure asks (Nest-steal). + return {"evidence": evidence_n} + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print actions without writing DB / embeddings", + ) + parser.add_argument( + "--skip-embed", + action="store_true", + help="Only merge metadata.facts; do not re-embed", + ) + args = parser.parse_args(argv) + + os.environ.setdefault("FLASK_ENV", "development") + from application import create_app + from application.database import db as db_mod + + app = create_app(mode="production") + with app.app_context(): + database = db_mod.Node_collection() + print("Seeding node metadata.facts …", flush=True) + counts = _seed_nodes(database, dry_run=args.dry_run) + print(f"metadata updated: {counts}", flush=True) + print("Re-embedding evidence for node_facts …", flush=True) + emb = _reembed_evidence( + database, dry_run=args.dry_run, skip_embed=args.skip_embed + ) + print(f"embeddings upserted: {emb}", flush=True) + print("done", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())