Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 19 additions & 5 deletions backend/routes/live.py
Original file line number Diff line number Diff line change
Expand Up @@ -646,6 +646,9 @@ async def set_kill_switch(data: KillSwitchRequest, current_user: User = Depends(
# ==================== GOPHISH CONFIG (admin) ====================


_SECRETS = ("gophish_api_key", "smtp_password", "imap_password", "report_imap_password")


class LiveConfig(BaseModel):
model_config = {"extra": "forbid"}
gophish_base_url: str = Field(min_length=1, max_length=300)
Expand All @@ -666,6 +669,10 @@ class LiveConfig(BaseModel):
imap_ssl: bool = True
imap_user: str = ""
imap_password: str = ""
# Report mailbox on the same IMAP server: staff forward suspicious mail here
# and reporters of our simulations are credited automatically.
report_imap_user: str = Field(default="", max_length=200)
report_imap_password: str = ""


@router.get("/config")
Expand All @@ -674,18 +681,19 @@ async def get_config(current_user: User = Depends(require_admin)):
if not config:
return None
# Never return stored secrets
for secret in ("gophish_api_key", "smtp_password", "imap_password"):
for secret in _SECRETS:
if config.get(secret):
config[secret] = "***"
return config


@router.post("/poll")
async def poll_inbox(current_user: User = Depends(require_admin)):
"""Fetch and route inbound replies now. The poller also runs on a timer."""
from services.live.inbound import poll_once
"""Fetch and route inbound replies and forwarded reports now. The poller
also runs on a timer."""
from services.live.inbound import poll_once, poll_reports_once

return await poll_once()
return {"replies": await poll_once(), "reports": await poll_reports_once()}


@router.put("/config")
Expand All @@ -694,9 +702,15 @@ async def set_config(data: LiveConfig, current_user: User = Depends(require_admi
# field (e.g. an older client without the IMAP-SSL toggle) does not reset it
# to the model default.
doc = {"id": "live_config", **data.model_dump(exclude_unset=True)}
report_box = (doc.get("report_imap_user") or "").strip().lower()
if report_box and report_box == (doc.get("imap_user") or "").strip().lower():
# Replies would then be counted as reports.
raise HTTPException(
status_code=400, detail="Report mailbox must differ from the reply mailbox"
)
# A resubmitted mask ("***") means "keep the stored secret", not overwrite it.
existing = await db.live_config.find_one({"id": "live_config"}, {"_id": 0})
for secret in ("gophish_api_key", "smtp_password", "imap_password"):
for secret in _SECRETS:
if doc.get(secret) in ("", "***") and existing:
doc[secret] = existing.get(secret, "")
await db.live_config.update_one({"id": "live_config"}, {"$set": doc}, upsert=True)
Expand Down
18 changes: 17 additions & 1 deletion backend/routes/report_inbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,22 @@ async def report_leaderboard(user: User = Depends(require_trainer)):
return await report_inbox.leaderboard(_scope(user))


@router.get("/mailbox")
async def report_mailbox(user: User = Depends(require_trainer)):
"""The address staff forward suspicious mail to, if a report mailbox is set up."""
config = await db.live_config.find_one({"id": "live_config"}, {"_id": 0}) or {}
address = (config.get("report_imap_user") or "").strip().lower()
return {"address": address or None, "configured": bool(address and config.get("imap_host"))}


@router.post("/poll")
async def poll_report_mailbox(user: User = Depends(require_trainer)):
"""Fetch forwarded reports now instead of waiting for the next poller tick."""
from services.live.inbound import poll_reports_once

return await poll_reports_once()


class ReportIn(BaseModel):
reporter_email: str = Field(min_length=3, max_length=320)
subject: str = Field(default="", max_length=400)
Expand All @@ -52,7 +68,7 @@ class ReportIn(BaseModel):

@router.post("/ingest", status_code=201)
async def ingest(body: ReportIn, user: User = Depends(require_trainer)):
"""Record a reported message (the integration point the mail poller also uses)."""
"""Record a reported message by hand (the report mailbox poller feeds the same path)."""
org_id = None if user.role == "admin" else user.organization_id
report = await report_inbox.ingest_report(
org_id, body.reporter_email, body.subject, body.from_address, body.body
Expand Down
132 changes: 124 additions & 8 deletions backend/services/live/inbound.py
Original file line number Diff line number Diff line change
@@ -1,23 +1,31 @@
"""
IMAP poller for live-campaign replies (P3 plumbing).
IMAP pollers for live campaigns: recipient replies (P3 plumbing) and the
report mailbox staff forward suspicious mail to.

A recipient's reply arrives at the Reply-To address
`<reply_local>+<campaign_token>@<reply_domain>`. The poller reads the mailbox,
maps the token to a campaign, matches the recipient by the reply's From address,
strips quoted history, and hands the fresh text to the conversation engine.

The report mailbox is a second inbox (e.g. lapor@acme.test). Every message in
it is a forwarded report: it is parsed for the original sender and subject and
handed to the report inbox, which credits the reporter when it is one of our
simulations — the behaviour report rate measures.

Pure parsing (token, body, sender) is separated from IMAP I/O so it can be
tested without a server.
"""

import asyncio
import html
import imaplib
import logging
import re
from email import message_from_bytes
from email.message import Message
from email.utils import parseaddr

from services import report_inbox
from services.ai_gateway import AIGateway
from services.database import db
from services.live import conversation
Expand All @@ -34,6 +42,13 @@
re.compile(r"^\s*_{5,}\s*$"),
)

# Header lines inside an inline-forwarded message (Gmail, Outlook, Thunderbird; EN/ID).
_FWD_FROM_RE = re.compile(r"^\s*\*?(?:From|Dari)\s*:\*?\s*(.+)$", re.IGNORECASE | re.MULTILINE)
_FWD_SUBJECT_RE = re.compile(
r"^\s*\*?(?:Subject|Subjek|Perihal)\s*:\*?\s*(.+)$", re.IGNORECASE | re.MULTILINE
)
_TAG_RE = re.compile(r"<[^>]+>")


def parse_token(*header_values: str | None) -> str | None:
"""Extract the campaign token from any plus-addressed recipient header."""
Expand Down Expand Up @@ -82,6 +97,56 @@ def extract_body(msg: Message) -> str:
return payload.decode(msg.get_content_charset() or "utf-8", "replace")


def _text_of(msg: Message) -> str:
"""Plain text of a message, falling back to tag-stripped HTML."""
html_text = ""
for part in msg.walk():
ctype = part.get_content_type()
if ctype not in ("text/plain", "text/html") or "attachment" in str(
part.get("Content-Disposition", "")
):
continue
payload = part.get_payload(decode=True)
if not payload:
continue
text = payload.decode(part.get_content_charset() or "utf-8", "replace")
if ctype == "text/plain":
return text
html_text = html_text or html.unescape(_TAG_RE.sub("\n", text))
return html_text


def parse_report(raw: bytes) -> dict | None:
"""Pull the reporter and the reported message out of one forwarded email.

Handles "forward as attachment" (a message/rfc822 part) and inline
forwards, where the original From/Subject appear as lines in the body.
"""
msg = message_from_bytes(raw)
reporter = sender_address(msg.get("From"))
if not reporter:
return None
for part in msg.walk():
if part.get_content_type() == "message/rfc822":
inner = part.get_payload(0) if part.is_multipart() else None
if isinstance(inner, Message):
return {
"reporter_email": reporter,
"subject": str(inner.get("Subject") or msg.get("Subject") or ""),
"from_address": sender_address(inner.get("From")) or "",
"body": _text_of(inner),
}
body = _text_of(msg)
fwd_from = _FWD_FROM_RE.search(body)
fwd_subject = _FWD_SUBJECT_RE.search(body)
return {
"reporter_email": reporter,
"subject": fwd_subject.group(1).strip() if fwd_subject else str(msg.get("Subject") or ""),
"from_address": sender_address(fwd_from.group(1)) if fwd_from else "",
"body": body,
}


async def route_message(raw: bytes, gateway=None) -> dict:
"""Parse one raw RFC822 message and dispatch it to the conversation engine."""
msg = message_from_bytes(raw)
Expand Down Expand Up @@ -125,7 +190,9 @@ async def route_message(raw: bytes, gateway=None) -> dict:
# ── IMAP I/O ─────────────────────────────────────────────────────────


def _fetch_new(config: dict, last_uid: int, last_uidvalidity: int) -> tuple[list[bytes], int, int]:
def _fetch_new(
config: dict, user: str, password: str, last_uid: int, last_uidvalidity: int
) -> tuple[list[bytes], int, int]:
"""Blocking IMAP fetch of messages with UID greater than the watermark.
Returns (raw messages, current UIDVALIDITY, highest UID fetched). Runs in a thread."""
host, _, port = config["imap_host"].partition(":")
Expand All @@ -135,7 +202,7 @@ def _fetch_new(config: dict, last_uid: int, last_uidvalidity: int) -> tuple[list
else imaplib.IMAP4(host, int(port or 143))
)
try:
conn.login(config["imap_user"], config.get("imap_password", ""))
conn.login(user, password)
conn.select("INBOX")
# Track by UID, not the \Seen flag: a human reading the mailbox (e.g. in
# Thunderbird) marks replies seen, which an UNSEEN search would then skip.
Expand Down Expand Up @@ -175,7 +242,12 @@ async def poll_once() -> dict:
state = await db.live_state.find_one({"id": "imap_watermark"}, {"_id": 0}) or {}
try:
raws, uidvalidity, max_uid = await asyncio.to_thread(
_fetch_new, config, int(state.get("last_uid", 0)), int(state.get("uidvalidity", 0))
_fetch_new,
config,
config["imap_user"],
config.get("imap_password", ""),
int(state.get("last_uid", 0)),
int(state.get("uidvalidity", 0)),
)
except Exception as exc:
logger.warning(f"IMAP poll failed: {exc}")
Expand Down Expand Up @@ -203,14 +275,58 @@ async def poll_once() -> dict:
return {"polled": len(raws), "outcomes": outcomes}


async def poll_reports_once() -> dict:
"""Fetch forwarded reports from the report mailbox and credit the reporters."""
config = await db.live_config.find_one({"id": "live_config"}, {"_id": 0})
mailbox = ((config or {}).get("report_imap_user") or "").strip().lower()
if not config or not config.get("imap_host") or not mailbox:
return {"polled": 0, "reason": "report_mailbox_not_configured"}

state = await db.live_state.find_one({"id": "report_imap_watermark"}, {"_id": 0}) or {}
try:
raws, uidvalidity, max_uid = await asyncio.to_thread(
_fetch_new,
config,
mailbox,
config.get("report_imap_password", ""),
int(state.get("last_uid", 0)),
int(state.get("uidvalidity", 0)),
)
except Exception as exc:
logger.warning(f"report mailbox poll failed: {exc}")
return {"polled": 0, "error": str(exc)}

outcomes: dict[str, int] = {}
for raw in raws:
try:
parsed = parse_report(raw)
if not parsed or parsed["reporter_email"] == mailbox:
status = "skipped"
else:
report = await report_inbox.ingest_report(None, **parsed)
status = "credited" if report["credited"] else report["verdict"]
except Exception as exc:
logger.warning(f"ingesting a forwarded report failed: {exc}")
status = "ingest_error"
outcomes[status] = outcomes.get(status, 0) + 1

await db.live_state.update_one(
{"id": "report_imap_watermark"},
{"$set": {"last_uid": max_uid, "uidvalidity": uidvalidity}},
upsert=True,
)
return {"polled": len(raws), "outcomes": outcomes}


async def run_poller(interval_seconds: int = 30) -> None:
"""Background loop, always running. poll_once is a no-op until IMAP is
configured, so saving the config in Settings takes effect on the next tick
without a restart."""
logger.info("Live reply poller started")
while True:
try:
await poll_once()
except Exception as exc:
logger.warning(f"poller iteration error: {exc}")
for poll in (poll_once, poll_reports_once):
try:
await poll()
except Exception as exc:
logger.warning(f"poller iteration error: {exc}")
await asyncio.sleep(interval_seconds)
54 changes: 44 additions & 10 deletions backend/services/report_inbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,28 +39,50 @@ def _clean_subject(subject: str) -> str:


async def classify(
org_scope: dict, subject: str, from_address: str, body: str
org_scope: dict, subject: str, from_address: str, body: str, reporter_email: str = ""
) -> tuple[str, str | None]:
"""Decide whether a reported message is one of our simulations.

Matches by the campaign's sender address (in the From or anywhere in the
forwarded body) or by an exact subject match. Returns (verdict, campaign_id).
forwarded body) or by an exact subject match. Campaigns often reuse a
sender, so among matches the newest one the reporter actually received
wins (preferring one they have not been credited for yet).
Returns (verdict, campaign_id).
"""
clean_subj = _clean_subject(subject)
from_addr = (from_address or "").strip().lower()
body_l = (body or "").lower()

matches: list[str] = []
async for c in db.live_campaigns.find(
{**org_scope, "status": {"$in": list(_SENT_STATUSES)}},
{"_id": 0, "id": 1, "sender_email": 1, "subject": 1},
):
).sort("created_at", -1):
sender = (c.get("sender_email") or "").strip().lower()
if sender and (sender == from_addr or sender in body_l):
return "simulation", c["id"]
csubj = (c.get("subject") or "").strip().lower()
if csubj and clean_subj and csubj == clean_subj:
return "simulation", c["id"]
return "needs_review", None
if (sender and (sender == from_addr or sender in body_l)) or (
csubj and clean_subj and csubj == clean_subj
):
matches.append(c["id"])
if not matches:
return "needs_review", None

email = (reporter_email or "").strip().lower()
if email:
received = {
r["campaign_id"]: r.get("reported", False)
async for r in db.live_recipients.find(
{"campaign_id": {"$in": matches}, "email": email},
{"_id": 0, "campaign_id": 1, "reported": 1},
)
}
for cid in matches:
if cid in received and not received[cid]:
return "simulation", cid
for cid in matches:
if cid in received:
return "simulation", cid
return "simulation", matches[0]


async def credit_reporter(campaign_id: str, reporter_email: str) -> bool:
Expand Down Expand Up @@ -103,13 +125,25 @@ async def ingest_report(
"""Store one reported message and, if it is ours, credit the reporter."""
org_scope = {} if org_id is None else {"organization_id": org_id}
redacted, _ = scrub(body or "")
verdict, campaign_id = await classify(org_scope, subject, from_address, body)
verdict, campaign_id = await classify(org_scope, subject, from_address, body, reporter_email)

credited = False
if verdict == "simulation" and campaign_id:
credited = await credit_reporter(campaign_id, reporter_email)

user = await db.users.find_one({"email": reporter_email.strip().lower()}, {"_id": 0, "id": 1})
user = await db.users.find_one(
{"email": reporter_email.strip().lower()}, {"_id": 0, "id": 1, "organization_id": 1}
)
if org_id is None:
# Arrived through the report mailbox, not a staff member's scope: file it
# under the matched campaign's organization, else the reporter's.
if campaign_id:
campaign = await db.live_campaigns.find_one(
{"id": campaign_id}, {"_id": 0, "organization_id": 1}
)
org_id = (campaign or {}).get("organization_id")
if org_id is None:
org_id = (user or {}).get("organization_id")
report = PhishReport(
organization_id=org_id,
reporter_email=reporter_email.strip().lower(),
Expand Down
Loading