diff --git a/TODO-minor-findings.md b/TODO-minor-findings.md new file mode 100644 index 0000000..46d7918 --- /dev/null +++ b/TODO-minor-findings.md @@ -0,0 +1,60 @@ +# TODO — minor findings + +Recorded: 2026-10-25 +Source: validation audit of commit `0b6ebe1` (per-target HTTP catch-all override) +Verdict: no functional bugs. Items below are polish/consistency, none block use. + +Status: items 1–3 fixed in the follow-up commit (see git log). Items 4–5 remain +open by design / pending a decision. + +## 1. List endpoints emit empty override strings (API consistency) + +`GET /targets` (handleListTargets) and the `POST /targets` create response +build `models.Target` without the new catchall fields, so JSON contains: + + "catchall_detection": "", + "catchall_effective": false + +The detail endpoint (`GET /targets/{id}`) always returns `inherit|on|off`. +The UI reads these fields only from the detail endpoint, so there is no UI +effect today. External API clients could misread the empty string. + +Fix options: +- Fill the fields in the list/create handlers (compute like handleGetTarget), or +- add `omitempty` to both JSON tags in `internal/models/models.go`. + +## 2. Doc comment misattached (cosmetic) + +In `internal/scanner/http_catchall.go`, the `runCatchallPass` doc comment now +sits above `catchallEnabledForTarget` because the helper was inserted between +the comment and its function. `runCatchallPass` has no doc comment of its own. + +Fix: move the comment down, or give each function its own short comment. + +## 3. Silent UI failure on catch-all save (UX) + +`saveCatchall` in `frontend/src/pages/TargetDetail.tsx` swallows errors +(`catch { /* ignore */ }`), same pattern as `saveMonitor`. On a failed PUT the +select snaps back to the previous value with no toast. + +Fix: add an error toast (the page already has `addToast`). + +## 4. DB error during override lookup falls back to global (edge) + +`catchallEnabledForTarget` returns the global config when the targets query +errors. An explicitly-Off target could therefore run one catch-all pass during +a transient DB error. The pass is best-effort by design; noted so the behavior +is not a surprise. + +Optional fix: fail closed for explicit overrides by caching the resolved value +per scan instead of re-querying. + +## 5. Turning the override Off does not unmark existing rows (by design) + +Marked rows keep `catchall = 1` and stay hidden from the Hosts tab, reports, +graph, stats and wordlist learning. This is the mark-never-delete design: the +override only controls FUTURE passes. + +Optional follow-up (needs a decision): a small "unmark" action or a cleanup +query (`UPDATE subdomains SET catchall = 0 WHERE target_id = ?`) exposed in +the UI. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c06a218..5b4e1ef 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,13 +1,14 @@ { "name": "reconner-frontend", - "version": "3.4.0", + "version": "4.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "reconner-frontend", - "version": "3.4.0", + "version": "4.7.0", "dependencies": { + "diff": "^9.0.0", "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^7.18.3", @@ -1425,6 +1426,15 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/diff": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-9.0.0.tgz", + "integrity": "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", diff --git a/frontend/package.json b/frontend/package.json index 08b659f..e1c01ff 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,6 +14,7 @@ "test:e2e": "playwright test" }, "dependencies": { + "diff": "^9.0.0", "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^7.18.3", diff --git a/frontend/src/components/layout/TopBar.tsx b/frontend/src/components/layout/TopBar.tsx index c77f14c..38d2665 100644 --- a/frontend/src/components/layout/TopBar.tsx +++ b/frontend/src/components/layout/TopBar.tsx @@ -182,7 +182,7 @@ export const TopBar = () => { return ( <> -
+
{/* Breadcrumbs */}
- setScanDraft(d => d && ({ ...d, depth: parseInt(e.target.value, 10) || 1 }))} - className="input font-mono text-xs mt-1 w-28" /> -

After the apex, each resolving subdomain is itself enumerated, down to this many levels (default 4). 1 ⇒ apex only.

+
+ setScanDraft(d => d && ({ ...d, depth: parseInt(e.target.value, 10) || 1 }))} + className="input font-mono text-xs w-28" /> + +
+

After the apex, each resolving subdomain is itself enumerated, down to this many levels (default 4). Off or depth 1 ⇒ apex only. This is the deployment default; the scan dialog can opt in per scan.

+ +

Optional, off by default. After HTTP probing, groups 2xx hosts that all serve one wildcard page — redirect-to-apex fleets or homepage clones (the "junk.git.target" catch-all case) — and hides them from the Hosts tab, reports, graph and web modules. Reads stored responses only, no extra requests. Seeds and vhosts are never touched.

@@ -330,6 +368,20 @@ export default function System() { className="input font-mono text-xs mt-1" />

Cloned on top of the official set. Empty ⇒ the two official ProjectDiscovery repos only. Community repos are a known false-positive source — add deliberately.

+
+ +
+ {MONITOR_MODULE_OPTIONS.map(m => ( + + ))} +
+

Modules the automatic monitor schedule runs; applies from its next cycle. The state monitor always snapshots first, before HTTP/JS refresh. Uncheck everything ⇒ the built-in set above applies.

+
)} diff --git a/frontend/src/pages/TargetDetail.monitor.test.ts b/frontend/src/pages/TargetDetail.monitor.test.ts new file mode 100644 index 0000000..ca95a13 --- /dev/null +++ b/frontend/src/pages/TargetDetail.monitor.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from 'vitest' +import { changeBadgeVariant, monitorRowKind } from './TargetDetail' +import type { MonitoringChange } from '../types' + +// Regression: clicking the Changes tab used to render the PREVIOUS tab's rows +// for one frame (data is replaced only after the fetch resolves). A subdomain +// row has no change_type, and the old helpers dereferenced it directly: +// TypeError: Cannot read properties of undefined (reading 'includes') +// The chips call monitorRowKind() for every loaded row, so the throw happened +// before React could commit the tab. Both helpers must classify any row shape. +describe('Changes tab row helpers tolerate foreign rows', () => { + it('classifies a stale non-monitor row instead of throwing', () => { + const stale = { id: 's1', target_id: 't1', subdomain: 'a.example.test' } as unknown as MonitoringChange + expect(monitorRowKind(stale)).toBe('http') + expect(changeBadgeVariant(stale)).toBe('info') + }) + + it('still classifies real monitor rows', () => { + const css = { change_type: 'css_change', severity: 'info' } as MonitoringChange + const js = { change_type: 'js_change', severity: 'medium' } as MonitoringChange + const sheet = { change_type: 'security:stylesheet_added', severity: 'info' } as MonitoringChange + const header = { change_type: 'security:header_removed', severity: 'high' } as MonitoringChange + expect(monitorRowKind(css)).toBe('css') + expect(monitorRowKind(js)).toBe('scripts') + expect(monitorRowKind(sheet)).toBe('css') + expect(monitorRowKind(header)).toBe('headers') + expect(changeBadgeVariant(css)).toBe('info') + expect(changeBadgeVariant(js)).toBe('medium') + expect(changeBadgeVariant(header)).toBe('high') + }) +}) diff --git a/frontend/src/pages/TargetDetail.tsx b/frontend/src/pages/TargetDetail.tsx index dbc8b3f..22dca73 100644 --- a/frontend/src/pages/TargetDetail.tsx +++ b/frontend/src/pages/TargetDetail.tsx @@ -5,6 +5,7 @@ import { useUIStore } from '../store/ui' import { Badge, Button, Spinner, Empty, CopyButton, ErrorBoundary, SkeletonRows } from '../components/ui' import { ScanModal } from '../components/targets/ScanModal' import { EvidenceViewer } from '../components/targets/EvidenceViewer' +import { ChangeDiffViewer } from '../components/targets/ChangeDiffViewer' import { LiveResults } from '../components/targets/LiveResults' import { ws } from '../lib/websocket' import { timeAgo, statusCodeColor, truncate, cn } from '../lib/utils' @@ -17,6 +18,50 @@ const isScannableProjectAsset = (asset: Asset) => (asset.kind === 'web' && ['domain', 'wildcard', 'url', 'page', 'js', 'api'].includes(asset.asset_type || 'domain')) || (asset.kind === 'network' && ['ip', 'cidr'].includes(asset.asset_type || 'ip')) +// Changes(/monitor) tab: filter buckets for the noise chips. The stylesheet +// content tracker (css_change + security:stylesheet_*) turned this tab into a +// steady stream; the chips let an operator pull one signal back out of it. +type ChangeFilter = 'all' | 'security' | 'scripts' | 'css' | 'http' | 'tech' | 'headers' +const CHANGE_FILTERS: { id: ChangeFilter; label: string }[] = [ + { id: 'all', label: 'All' }, + { id: 'security', label: 'Security' }, + { id: 'scripts', label: 'Scripts' }, + { id: 'css', label: 'Stylesheets' }, + { id: 'http', label: 'HTTP' }, + { id: 'tech', label: 'Tech' }, + { id: 'headers', label: 'Headers' }, +] + +// Bucket order matters: the stylesheet/script markers win before the generic +// "security:" prefix, so a security:stylesheet_* row lands in Stylesheets (not +// Security), a security:script_* row in Scripts, and header removals in Headers. +export const monitorRowKind = (c: MonitoringChange): ChangeFilter => { + // One render can hold rows from the PREVIOUS tab while the next tab's fetch + // is in flight (data is replaced only after the request resolves). Never + // assume the row really is a monitoring_changes row. + const ct = typeof c?.change_type === 'string' ? c.change_type : '' + if (ct === 'css_change' || ct.includes('stylesheet')) return 'css' + if (ct === 'js_change' || ct.includes('script')) return 'scripts' + if (ct.includes('header')) return 'headers' + if (ct.startsWith('security')) return 'security' + if (ct === 'tech_change') return 'tech' + return 'http' +} + +// Badge color follows the stored severity (rows written since the severity +// column carry one); older rows without it keep the old type-based estimate. +export const changeBadgeVariant = (c: { severity?: string; change_type?: string }): 'high' | 'medium' | 'low' | 'info' => { + const sev = (c.severity || '').toLowerCase() + if (sev === 'high' || sev === 'medium' || sev === 'low' || sev === 'info') return sev + // Same stale-row hazard as monitorRowKind: the badge must render even when a + // previous tab's row is briefly shown under the Changes tab. + const ct = typeof c.change_type === 'string' ? c.change_type : '' + return ct === 'js_change' ? 'medium' + : ct.startsWith('security') || ct.includes('header') ? 'high' + : ct === 'tech_change' ? 'low' + : 'info' +} + // Projects created from the WP Scanner page carry this tag (see WPScanner.tsx) — // the same signal used there to decide this is a WordPress project, reused here // to decide this project's detail page should show WordPress-specific tabs. @@ -472,8 +517,16 @@ export default function TargetDetail() { const [scopeEvents, setScopeEvents] = useState([]) const [eventBusy, setEventBusy] = useState(null) const [evidence, setEvidence] = useState<{ id: string; url: string; type: string } | null>(null) + // Changes-tab noise filter + the open content-diff drawer (css/js rows). + const [changeFilter, setChangeFilter] = useState('all') + const [diffView, setDiffView] = useState<{ url: string; kind: 'css' | 'js' } | null>(null) + // Changes tab: the endpoint pages 500 rows at a time. Track whether older + // history remains server-side and a load flag for the footer button. + const [monitorHasMore, setMonitorHasMore] = useState(false) + const [monitorLoadingMore, setMonitorLoadingMore] = useState(false) const [isScanning, setIsScanning] = useState(false) const [monitorSaving, setMonitorSaving] = useState(false) + const [catchallSaving, setCatchallSaving] = useState(false) const [paths, setPaths] = useState([]) const [lastFailedTask, setLastFailedTask] = useState(null) const [resuming, setResuming] = useState(false) @@ -661,6 +714,8 @@ export default function TargetDetail() { const code = Number((row as { status_code?: number })?.status_code || 0) return Math.floor(code / 100) === Number(statusClass) }) + : (tab === 'monitor' && changeFilter !== 'all') + ? (data as MonitoringChange[]).filter(row => monitorRowKind(row) === changeFilter) : data // Windowed view of the loaded rows — only the first `visibleCount` are mounted. const pageData = filteredData.slice(0, visibleCount) @@ -721,13 +776,29 @@ export default function TargetDetail() { r = all.filter(f => want.has(f.type)) } else if (t === 'cameras') r = await findingsApi.ingram(id) - else if (t === 'monitor') r = await findingsApi.monitoringChanges(id) + else if (t === 'monitor') { + const batch = await findingsApi.monitoringChanges(id) + r = batch + setMonitorHasMore((batch || []).length >= 500) + } setData(r || []) setVisibleCount(PAGE) } catch { setData([]) } setTabLoading(false) } + // Changes tab: pull the next 500-row page of older history. + const loadOlderChanges = async () => { + if (!id || monitorLoadingMore) return + setMonitorLoadingMore(true) + try { + const batch = await findingsApi.monitoringChanges(id, data.length) + setData(prev => [...prev, ...batch]) + setMonitorHasMore(batch.length >= 500) + } catch (e) { console.error(e) } + finally { setMonitorLoadingMore(false) } + } + if (loading) return
if (!target) return null @@ -779,6 +850,19 @@ export default function TargetDetail() { } catch { /* ignore */ } finally { setMonitorSaving(false) } } + const saveCatchall = async (value: 'inherit' | 'on' | 'off') => { + if (!id) return + setCatchallSaving(true) + try { + await targetsApi.update(id, { catchall_detection: value }) + // Refresh so catchall_effective reflects the global config on "inherit". + targetsApi.get(id).then(setTarget).catch(() => {}) + addToast('success', 'Catch-all detection updated') + } catch (e) { + addToast('error', e instanceof Error ? e.message : 'Failed to update catch-all detection') + } finally { setCatchallSaving(false) } + } + // Network targets' actual results (Nuclei/Vulns/Candidates) used to sit // BELOW monitoring settings, attack paths, the auth-header form and the // identities/authorization panels — none of which are commonly used for a @@ -974,6 +1058,31 @@ export default function TargetDetail() { )} + {/* HTTP catch-all detection override */} +
+
+ HTTP catch-all detection + + {catchallSaving && Saving…} + + Effective: {target.catchall_effective ? 'On' : 'Off'} + +
+

+ Marks fleets that redirect to the apex or clone the apex homepage as catch-all junk (hidden from Hosts, + reports and later modules). The global default is off; the per-target override applies from the next scan. + Seeds and vhost rows are never touched. +

+
+ {/* Attack paths (correlation / intelligence layer) */} {paths.length > 0 && (
@@ -1009,7 +1118,18 @@ export default function TargetDetail() { {gi > 0 && } {g.group} {g.tabs.map(t => ( -
)} - + {/* key={tab}: a caught render error belongs to one tab's row shape; do + not leave the fallback stuck when the operator switches tabs. */} +
{tabLoading ?
@@ -1105,6 +1227,26 @@ export default function TargetDetail() {
) : ( + <> + {tab === 'monitor' && ( +
+ {CHANGE_FILTERS.map(f => { + const n = f.id === 'all' + ? data.length + : (data as MonitoringChange[]).filter(row => monitorRowKind(row) === f.id).length + return ( + + ) + })} +
+ )}
@@ -1532,13 +1674,20 @@ export default function TargetDetail() {
{truncate(c.url, 50)} + {/* Content-capture diff for monitored stylesheets/scripts. + Legacy rows (recorded before capture) open too, but + show the not-captured note + hashes only. */} + {(c.change_type === 'css_change' || c.change_type === 'js_change') && ( + + )}
{urls.length > 1 && ( +{urls.length - 1} more URL{urls.length - 1 === 1 ? '' : 's'} )} {/* Old/new values were truncated to 50 chars with no way to read the rest. Now each value wraps, shows the full text on hover, and has a @@ -1562,6 +1711,7 @@ export default function TargetDetail() {
- {c.change_type} + {c.change_type}
+ )} {!tabLoading && footerHasMore && (
@@ -1577,6 +1727,14 @@ export default function TargetDetail() {
)} + {tab === 'monitor' && !tabLoading && monitorHasMore && !footerHasMore && ( +
+ Showing newest {data.length.toLocaleString()} changes + +
+ )} @@ -1608,6 +1766,9 @@ export default function TargetDetail() { onClose={() => setEvidence(null)} /> )} + {diffView && ( + setDiffView(null)} /> + )} ) diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index a9cf94e..de2c8a4 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -132,6 +132,8 @@ export interface Target { monitor_last_run: string | null scan_user_agent?: string scan_headers?: Record + catchall_detection?: 'inherit' | 'on' | 'off' + catchall_effective?: boolean } export interface Subdomain { @@ -311,9 +313,22 @@ export interface MonitoringChange { change_type: string old_value: string new_value: string + severity: string detected_at: string } +// Body of a monitored file (css_files / js_files) as captured by the monitor +// pass when the change was detected — see GET /targets/{id}/css-files/body. +export interface FileBodyResponse { + kind: string + url: string + side: string + hash: string + size: number + content: string + captured: boolean +} + export interface IngramCamera { id: string address: string // ip:port diff --git a/internal/api/finding_handlers.go b/internal/api/finding_handlers.go index 1defb40..37a6e33 100644 --- a/internal/api/finding_handlers.go +++ b/internal/api/finding_handlers.go @@ -6,6 +6,7 @@ import ( "net/http" "sort" "strconv" + "strings" "github.com/gorilla/mux" "github.com/recon-platform/internal/models" @@ -63,7 +64,7 @@ func (h *Handler) handleListSubdomains(w http.ResponseWriter, r *http.Request) { query := `SELECT id, target_id, subdomain, ip, is_alive, status_code, page_title, has_https, has_http, response_time_ms, technologies, server, framework, cdn, waf, COALESCE(source,'dns'), last_seen, created_at - FROM subdomains WHERE target_id = ?` + FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0) = 0` args := []any{id} if alive == "true" { @@ -78,7 +79,7 @@ func (h *Handler) handleListSubdomains(w http.ResponseWriter, r *http.Request) { } var total int - countQuery := "SELECT COUNT(*) FROM subdomains WHERE target_id = ?" + countQuery := "SELECT COUNT(*) FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0) = 0" countArgs := []any{id} if alive == "true" { countQuery += " AND is_alive = 1" @@ -560,13 +561,72 @@ func (h *Handler) handleNucleiAffected(w http.ResponseWriter, r *http.Request) { h.writeSuccess(w, out) } +// serveFileBody returns the capped content capture the MONITOR pass stored for +// a diffed asset (css_files / js_files): side=current (live capture) or +// side=old (previous capture rolled in at detection time). URL-keyed — the +// monitor records changes under the exact same URL string it tracks content +// for. Hard-scoped to the routed target: no cross-target reads. This is NOT a +// fetch proxy: it only ever returns content captured at detection time, so it +// has no request-driven SSRF surface. +func (h *Handler) serveFileBody(w http.ResponseWriter, r *http.Request, table, kind string) { + id := mux.Vars(r)["id"] + rawURL := strings.TrimSpace(r.URL.Query().Get("url")) + if rawURL == "" { + h.writeError(w, http.StatusBadRequest, "url query parameter required") + return + } + side := r.URL.Query().Get("side") + if side != "old" && side != "current" { + side = "current" + } + var storedURL, hash, content string + var size int64 + var rowErr error + if side == "old" { + rowErr = h.db.QueryRowContext(r.Context(), `SELECT url, COALESCE(old_hash,''), COALESCE(old_body,'') FROM `+table+` WHERE target_id = ? AND url = ?`, id, rawURL). + Scan(&storedURL, &hash, &content) + } else { + rowErr = h.db.QueryRowContext(r.Context(), `SELECT url, size, hash, COALESCE(body,'') FROM `+table+` WHERE target_id = ? AND url = ?`, id, rawURL). + Scan(&storedURL, &size, &hash, &content) + } + if rowErr != nil { + h.writeError(w, http.StatusNotFound, "no capture for this asset (the change predates content tracking, or the URL is not tracked)") + return + } + h.writeSuccess(w, map[string]any{ + "kind": kind, + "url": storedURL, + "side": side, + "hash": hash, + "size": size, + "content": content, + "captured": strings.TrimSpace(content) != "", + }) +} + +func (h *Handler) handleGetCSSFileBody(w http.ResponseWriter, r *http.Request) { + h.serveFileBody(w, r, "css_files", "css") +} + +func (h *Handler) handleGetJSFileBody(w http.ResponseWriter, r *http.Request) { + h.serveFileBody(w, r, "js_files", "js") +} + func (h *Handler) handleListMonitoringChanges(w http.ResponseWriter, r *http.Request) { id := mux.Vars(r)["id"] + // Newest first, 500 per page. `offset` lets the Changes tab walk further + // back than the newest page (the tab used to silently stop at 500 rows); + // id is the tiebreaker so rows sharing detected_at never shuffle between + // pages. + offset := 0 + if n, err := strconv.Atoi(r.URL.Query().Get("offset")); err == nil && n > 0 { + offset = n + } rows, err := h.db.QueryContext(r.Context(), ` - SELECT id, target_id, url, change_type, old_value, new_value, detected_at + SELECT id, target_id, url, change_type, old_value, new_value, COALESCE(severity,'info'), detected_at FROM monitoring_changes WHERE target_id = ? - ORDER BY detected_at DESC LIMIT 500 - `, id) + ORDER BY detected_at DESC, id DESC LIMIT 500 OFFSET ? + `, id, offset) if err != nil { h.writeError(w, http.StatusInternalServerError, "query failed") return @@ -580,12 +640,13 @@ func (h *Handler) handleListMonitoringChanges(w http.ResponseWriter, r *http.Req ChangeType string `json:"change_type"` OldValue string `json:"old_value"` NewValue string `json:"new_value"` + Severity string `json:"severity"` DetectedAt string `json:"detected_at"` } changes := make([]Change, 0) for rows.Next() { var c Change - if err := rows.Scan(&c.ID, &c.TargetID, &c.URL, &c.ChangeType, &c.OldValue, &c.NewValue, &c.DetectedAt); err == nil { + if err := rows.Scan(&c.ID, &c.TargetID, &c.URL, &c.ChangeType, &c.OldValue, &c.NewValue, &c.Severity, &c.DetectedAt); err == nil { changes = append(changes, c) } } diff --git a/internal/api/graph_handlers.go b/internal/api/graph_handlers.go index abdb488..dfe1389 100644 --- a/internal/api/graph_handlers.go +++ b/internal/api/graph_handlers.go @@ -67,7 +67,7 @@ func (h *Handler) buildGraph(r *http.Request, targetID string) ([]graphNode, []g // Subdomains ↔ IPs (co-location reveals pivot opportunities). ipHosts := map[string][]string{} subTech := map[string]string{} - rows, err := h.db.QueryContext(ctx, `SELECT subdomain, ip, technologies FROM subdomains WHERE target_id = ? AND is_alive = 1`, targetID) + rows, err := h.db.QueryContext(ctx, `SELECT subdomain, ip, technologies FROM subdomains WHERE target_id = ? AND is_alive = 1 AND COALESCE(catchall,0) = 0`, targetID) if err == nil { for rows.Next() { var sub, ip, techs string diff --git a/internal/api/htmlreport.go b/internal/api/htmlreport.go index ff26e7a..463c929 100644 --- a/internal/api/htmlreport.go +++ b/internal/api/htmlreport.go @@ -46,8 +46,8 @@ func (h *Handler) handleGenerateReportHTML(w http.ResponseWriter, r *http.Reques h.db.QueryRowContext(ctx, q, id).Scan(&n) counts[key] = n } - countInto("subs", "SELECT COUNT(*) FROM subdomains WHERE target_id=?") - countInto("alive", "SELECT COUNT(*) FROM subdomains WHERE target_id=? AND is_alive=1") + countInto("subs", "SELECT COUNT(*) FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0") + countInto("alive", "SELECT COUNT(*) FROM subdomains WHERE target_id=? AND is_alive=1 AND COALESCE(catchall,0)=0") countInto("hosts", "SELECT COUNT(*) FROM http_services WHERE target_id=? AND COALESCE(source,'probe')='probe'") countInto("js", "SELECT COUNT(*) FROM js_findings WHERE target_id=?") countInto("params", "SELECT COUNT(*) FROM parameters WHERE target_id=? AND is_reflected=1") @@ -126,7 +126,7 @@ func (h *Handler) handleGenerateReportHTML(w http.ResponseWriter, r *http.Reques h.section(ctx, &b, id, "subs", "All Subdomains", ` SELECT subdomain, CASE is_alive WHEN 1 THEN 'alive' ELSE '' END, COALESCE(ip,''), COALESCE(server,'') - FROM subdomains WHERE target_id=? ORDER BY is_alive DESC, subdomain LIMIT 5000`, + FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0 ORDER BY is_alive DESC, subdomain LIMIT 5000`, []string{"Subdomain", "Alive", "IP", "Server"}, -1) b.WriteString(reportFoot()) diff --git a/internal/api/pdfreport.go b/internal/api/pdfreport.go index 2954834..cfa6672 100644 --- a/internal/api/pdfreport.go +++ b/internal/api/pdfreport.go @@ -398,7 +398,7 @@ func (h *Handler) pdfReconSections(r *http.Request, p *pdfBuilder, id string) { p.rule() } - section("Subdomains", `SELECT subdomain, CASE is_alive WHEN 1 THEN 'alive' ELSE '' END, COALESCE(ip,'') FROM subdomains WHERE target_id=? ORDER BY subdomain LIMIT 1500`, 3) + section("Subdomains", `SELECT subdomain, CASE is_alive WHEN 1 THEN 'alive' ELSE '' END, COALESCE(ip,'') FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0 ORDER BY subdomain LIMIT 1500`, 3) section("Live HTTP Hosts", `SELECT url, status_code, COALESCE(title,'') FROM http_services WHERE target_id=? AND COALESCE(source,'probe')='probe' ORDER BY url LIMIT 1000`, 3) section("Reflected Parameters", `SELECT url, parameter FROM parameters WHERE target_id=? AND is_reflected=1 ORDER BY url LIMIT 800`, 2) section("Open Redirects", `SELECT url, redirect_to FROM open_redirect_findings WHERE target_id=? AND COALESCE(status,'finding')='finding' ORDER BY url LIMIT 500`, 2) diff --git a/internal/api/router.go b/internal/api/router.go index f41d8f2..c6949f0 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -204,6 +204,8 @@ func (h *Handler) Router() http.Handler { // Global findings aggregation (top-level Findings page) across all the caller's targets. api.HandleFunc("/findings", h.requireAuth(h.handleListAllFindings)).Methods("GET") api.HandleFunc("/targets/{id}/monitoring-changes", h.requireAuth(h.handleListMonitoringChanges)).Methods("GET") + api.HandleFunc("/targets/{id}/css-files/body", h.requireAuth(h.handleGetCSSFileBody)).Methods("GET") + api.HandleFunc("/targets/{id}/js-files/body", h.requireAuth(h.handleGetJSFileBody)).Methods("GET") // Tasks api.HandleFunc("/tasks", h.requireAuth(h.handleListTasks)).Methods("GET") diff --git a/internal/api/system_handlers.go b/internal/api/system_handlers.go index 92a6e5c..6cec19e 100644 --- a/internal/api/system_handlers.go +++ b/internal/api/system_handlers.go @@ -98,10 +98,13 @@ func (h *Handler) handleUpdateSettings(w http.ResponseWriter, r *http.Request) { // reflecting over the whole Config) means only these deliberately-safe tunables // are reachable from the API — never secrets, paths, or auth settings. type scanConfigPayload struct { - DNSResolvers []string `json:"dns_resolvers"` - SubdomainRecursionDepth int `json:"subdomain_recursion_depth"` - NucleiExcludeTemplateIDs []string `json:"nuclei_exclude_template_ids"` - NucleiExtraTemplateRepos []string `json:"nuclei_extra_template_repos"` + DNSResolvers []string `json:"dns_resolvers"` + SubdomainRecursionDepth int `json:"subdomain_recursion_depth"` + SubdomainRecursionEnabled bool `json:"subdomain_recursion_enabled"` + HTTPCatchallDetection bool `json:"http_catchall_detection"` + NucleiExcludeTemplateIDs []string `json:"nuclei_exclude_template_ids"` + NucleiExtraTemplateRepos []string `json:"nuclei_extra_template_repos"` + MonitorModules []string `json:"monitor_modules"` // DefaultDNSResolvers / DefaultNucleiRepos are read-only hints so the UI can // show what the built-in defaults are when the operator has set nothing. DefaultDNSResolvers []string `json:"default_dns_resolvers"` @@ -115,12 +118,15 @@ type scanConfigPayload struct { // and resolvers" surface. func (h *Handler) handleGetScanConfig(w http.ResponseWriter, _ *http.Request) { h.writeSuccess(w, scanConfigPayload{ - DNSResolvers: h.cfg.DNSResolvers, - SubdomainRecursionDepth: h.cfg.SubdomainRecursionDepth, - NucleiExcludeTemplateIDs: h.cfg.NucleiExcludeTemplateIDs, - NucleiExtraTemplateRepos: h.cfg.NucleiExtraTemplateRepos, - DefaultDNSResolvers: config.DefaultDNSResolvers, - DefaultNucleiRepos: scanner.DefaultNucleiExtraRepos(), + DNSResolvers: h.cfg.DNSResolvers, + SubdomainRecursionDepth: h.cfg.SubdomainRecursionDepth, + SubdomainRecursionEnabled: h.cfg.SubdomainRecursionEnabled, + HTTPCatchallDetection: h.cfg.HTTPCatchallDetection, + NucleiExcludeTemplateIDs: h.cfg.NucleiExcludeTemplateIDs, + NucleiExtraTemplateRepos: h.cfg.NucleiExtraTemplateRepos, + MonitorModules: h.cfg.MonitorModules, + DefaultDNSResolvers: config.DefaultDNSResolvers, + DefaultNucleiRepos: scanner.DefaultNucleiExtraRepos(), }) } @@ -130,10 +136,13 @@ func (h *Handler) handleGetScanConfig(w http.ResponseWriter, _ *http.Request) { // which third-party template repos are cloned onto the box. func (h *Handler) handleUpdateScanConfig(w http.ResponseWriter, r *http.Request) { var body struct { - DNSResolvers *[]string `json:"dns_resolvers"` - SubdomainRecursionDepth *int `json:"subdomain_recursion_depth"` - NucleiExcludeTemplateIDs *[]string `json:"nuclei_exclude_template_ids"` - NucleiExtraTemplateRepos *[]string `json:"nuclei_extra_template_repos"` + DNSResolvers *[]string `json:"dns_resolvers"` + SubdomainRecursionDepth *int `json:"subdomain_recursion_depth"` + SubdomainRecursionEnabled *bool `json:"subdomain_recursion_enabled"` + HTTPCatchallDetection *bool `json:"http_catchall_detection"` + NucleiExcludeTemplateIDs *[]string `json:"nuclei_exclude_template_ids"` + NucleiExtraTemplateRepos *[]string `json:"nuclei_extra_template_repos"` + MonitorModules *[]string `json:"monitor_modules"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { h.writeError(w, http.StatusBadRequest, "invalid body") @@ -164,6 +173,14 @@ func (h *Handler) handleUpdateScanConfig(w http.ResponseWriter, r *http.Request) } h.cfg.SubdomainRecursionDepth = d } + if body.SubdomainRecursionEnabled != nil { + // No clamping — a bool is a bool. Omitted ⇒ unchanged, like the other knobs. + h.cfg.SubdomainRecursionEnabled = *body.SubdomainRecursionEnabled + } + if body.HTTPCatchallDetection != nil { + // Opt-in-by-default HTTP catch-all grouping; omitted ⇒ unchanged. + h.cfg.HTTPCatchallDetection = *body.HTTPCatchallDetection + } if body.NucleiExcludeTemplateIDs != nil { h.cfg.NucleiExcludeTemplateIDs = clean(*body.NucleiExcludeTemplateIDs) } @@ -178,6 +195,12 @@ func (h *Handler) handleUpdateScanConfig(w http.ResponseWriter, r *http.Request) } h.cfg.NucleiExtraTemplateRepos = repos } + if body.MonitorModules != nil { + // Trimmed/deduped here; NAME validation happens in the scheduler + // (monitorWatchModules validates against AllModules and logs unknown + // tokens), so the API/UI never needs to know the module catalog. + h.cfg.MonitorModules = clean(*body.MonitorModules) + } if err := h.cfg.Save(); err != nil { h.writeError(w, http.StatusInternalServerError, "saved in memory but could not persist to config.json: "+err.Error()) return @@ -736,7 +759,7 @@ func (h *Handler) handleSearch(w http.ResponseWriter, r *http.Request) { subRows, err := h.db.QueryContext(r.Context(), ` SELECT s.id, s.subdomain, s.ip, t.domain as target_domain FROM subdomains s JOIN targets t ON t.id = s.target_id - WHERE s.subdomain LIKE ? LIMIT 20 + WHERE s.subdomain LIKE ? AND COALESCE(s.catchall,0)=0 LIMIT 20 `, like) if err == nil { defer subRows.Close() diff --git a/internal/api/target_catchall_test.go b/internal/api/target_catchall_test.go new file mode 100644 index 0000000..2c5734e --- /dev/null +++ b/internal/api/target_catchall_test.go @@ -0,0 +1,108 @@ +package api + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" + "github.com/recon-platform/internal/config" + "github.com/recon-platform/internal/websocket" +) + +// TestTargetCatchallOverrideRoundTrip proves the per-target tri-state override +// end to end through the real handlers: GET exposes override + effective state, +// PUT persists 0/1/NULL, invalid values are rejected, and "inherit" tracks the +// global config without touching the column. +func TestTargetCatchallOverrideRoundTrip(t *testing.T) { + h := newTestHandler(t) + h.hub = websocket.NewHub() + h.cfg = &config.Config{HTTPCatchallDetection: true} + + if _, err := h.db.Exec(`INSERT INTO targets (id, domain) VALUES ('ct-target','catchall.example.com')`); err != nil { + t.Fatal(err) + } + + getTarget := func() map[string]any { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/api/targets/ct-target", nil) + req = mux.SetURLVars(req, map[string]string{"id": "ct-target"}) + rec := httptest.NewRecorder() + h.handleGetTarget(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("GET target status = %d, body = %s", rec.Code, rec.Body.String()) + } + var resp struct { + Data map[string]any `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode GET response: %v", err) + } + return resp.Data + } + putTarget := func(body string) int { + t.Helper() + req := httptest.NewRequest(http.MethodPut, "/api/targets/ct-target", bytes.NewBufferString(body)) + req = mux.SetURLVars(req, map[string]string{"id": "ct-target"}) + rec := httptest.NewRecorder() + h.handleUpdateTarget(rec, req) + return rec.Code + } + + // A new target inherits the global (on). + data := getTarget() + if data["catchall_detection"] != "inherit" { + t.Fatalf("default override = %v, want inherit", data["catchall_detection"]) + } + if data["catchall_effective"] != true { + t.Fatalf("inherit with global on effective = %v, want true", data["catchall_effective"]) + } + + // Force off: the override wins over the global. + if code := putTarget(`{"catchall_detection":"off"}`); code != http.StatusOK { + t.Fatalf("PUT off status = %d", code) + } + data = getTarget() + if data["catchall_detection"] != "off" || data["catchall_effective"] != false { + t.Fatalf("after off: %v/%v, want off/false", data["catchall_detection"], data["catchall_effective"]) + } + + // Force on. + if code := putTarget(`{"catchall_detection":"on"}`); code != http.StatusOK { + t.Fatalf("PUT on status = %d", code) + } + data = getTarget() + if data["catchall_detection"] != "on" || data["catchall_effective"] != true { + t.Fatalf("after on: %v/%v, want on/true", data["catchall_detection"], data["catchall_effective"]) + } + + // Clear back to inherit: the column must be NULL again. + if code := putTarget(`{"catchall_detection":"inherit"}`); code != http.StatusOK { + t.Fatalf("PUT inherit status = %d", code) + } + var stored *int + if err := h.db.QueryRow(`SELECT catchall_detection FROM targets WHERE id='ct-target'`).Scan(&stored); err != nil { + t.Fatalf("read catchall_detection column: %v", err) + } + if stored != nil { + t.Fatalf("inherit must store NULL, got %d", *stored) + } + data = getTarget() + if data["catchall_detection"] != "inherit" || data["catchall_effective"] != true { + t.Fatalf("after inherit: %v/%v, want inherit/true", data["catchall_detection"], data["catchall_effective"]) + } + + // Invalid values are rejected before any write. + if code := putTarget(`{"catchall_detection":"maybe"}`); code != http.StatusBadRequest { + t.Fatalf("invalid override status = %d, want 400", code) + } + + // Flipping the global config changes "inherit" effectiveness without a write. + h.cfg.HTTPCatchallDetection = false + data = getTarget() + if data["catchall_effective"] != false { + t.Fatalf("inherit with global off effective = %v, want false", data["catchall_effective"]) + } +} diff --git a/internal/api/target_handlers.go b/internal/api/target_handlers.go index 43c1ab3..3424a33 100644 --- a/internal/api/target_handlers.go +++ b/internal/api/target_handlers.go @@ -371,20 +371,20 @@ func (h *Handler) handleGetTarget(w http.ResponseWriter, r *http.Request) { var t models.Target var lastScan, monitorLastRun *string var tagsJSON, modulesJSON, scanHeadersJSON string - var monitorEnabled int + var monitorEnabled, catchallOverride int err := h.db.QueryRowContext(r.Context(), ` SELECT id, domain, COALESCE(kind,'web'), COALESCE(description,''), COALESCE(tags,'[]'), COALESCE(priority,'medium'), COALESCE(notes,''), COALESCE(status,'idle'), COALESCE(scan_status,'idle'), COALESCE(enabled_modules,'[]'), last_scan_at, created_at, updated_at, COALESCE(subdomain_count,0), COALESCE(alive_host_count,0), COALESCE(finding_count,0), COALESCE(monitor_enabled,0), COALESCE(monitor_interval_hours,2), COALESCE(name,''), COALESCE(exclude_scope,''), monitor_last_run, - COALESCE(scan_user_agent,''), COALESCE(scan_headers,'{}') + COALESCE(scan_user_agent,''), COALESCE(scan_headers,'{}'), COALESCE(catchall_detection,-1) FROM targets WHERE id = ? `, id).Scan(&t.ID, &t.Domain, &t.Kind, &t.Description, &tagsJSON, &t.Priority, &t.Notes, &t.Status, &t.ScanStatus, &modulesJSON, &lastScan, &t.CreatedAt, &t.UpdatedAt, &t.SubdomainCount, &t.AliveHostCount, &t.FindingCount, - &monitorEnabled, &t.MonitorIntervalHours, &t.Name, &t.ExcludeScope, &monitorLastRun, &t.ScanUserAgent, &scanHeadersJSON) + &monitorEnabled, &t.MonitorIntervalHours, &t.Name, &t.ExcludeScope, &monitorLastRun, &t.ScanUserAgent, &scanHeadersJSON, &catchallOverride) if err != nil { h.writeError(w, http.StatusNotFound, "target not found") return @@ -393,6 +393,20 @@ func (h *Handler) handleGetTarget(w http.ResponseWriter, r *http.Request) { t.Tags = models.JSONToStringSlice(tagsJSON) t.EnabledModules = models.JSONToStringSlice(modulesJSON) t.MonitorEnabled = monitorEnabled == 1 + switch catchallOverride { + case 1: + t.CatchallDetection = "on" + on := true + t.CatchallEffective = &on + case 0: + t.CatchallDetection = "off" + off := false + t.CatchallEffective = &off + default: + t.CatchallDetection = "inherit" + effective := h.cfg != nil && h.cfg.HTTPCatchallDetection + t.CatchallEffective = &effective + } t.ScanHeaders = map[string]string{} _ = json.Unmarshal([]byte(scanHeadersJSON), &t.ScanHeaders) if lastScan != nil { @@ -421,6 +435,9 @@ func (h *Handler) handleUpdateTarget(w http.ResponseWriter, r *http.Request) { EnabledModules *[]string `json:"enabled_modules"` ScanUserAgent *string `json:"scan_user_agent"` ScanHeaders *map[string]string `json:"scan_headers"` + // Tri-state per-target override for the HTTP catch-all pass: + // "inherit" (NULL column, follow the global config), "on" (1), "off" (0). + CatchallDetection *string `json:"catchall_detection"` } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -432,10 +449,11 @@ func (h *Handler) handleUpdateTarget(w http.ResponseWriter, r *http.Request) { // omitted fields stay untouched; the old zero-value decoding silently erased // enabled_modules (and, for API clients, metadata/identity) on unrelated edits. var name, description, tagsJSON, priority, notes, excludeScope, modulesJSON, scanUserAgent, scanHeadersJSON string + var catchallOverride int err := h.db.QueryRowContext(r.Context(), `SELECT COALESCE(name,''),COALESCE(description,''),COALESCE(tags,'[]'), COALESCE(priority,'medium'),COALESCE(notes,''),COALESCE(exclude_scope,''),COALESCE(enabled_modules,'[]'), - COALESCE(scan_user_agent,''),COALESCE(scan_headers,'{}') FROM targets WHERE id=?`, id). - Scan(&name, &description, &tagsJSON, &priority, ¬es, &excludeScope, &modulesJSON, &scanUserAgent, &scanHeadersJSON) + COALESCE(scan_user_agent,''),COALESCE(scan_headers,'{}'),COALESCE(catchall_detection,-1) FROM targets WHERE id=?`, id). + Scan(&name, &description, &tagsJSON, &priority, ¬es, &excludeScope, &modulesJSON, &scanUserAgent, &scanHeadersJSON, &catchallOverride) if err == sql.ErrNoRows { h.writeError(w, http.StatusNotFound, "target not found") return @@ -480,6 +498,25 @@ func (h *Handler) handleUpdateTarget(w http.ResponseWriter, r *http.Request) { h.writeError(w, http.StatusBadRequest, err.Error()) return } + if req.CatchallDetection != nil { + // Tri-state override; "inherit" clears the column back to NULL. + switch strings.ToLower(strings.TrimSpace(*req.CatchallDetection)) { + case "inherit": + catchallOverride = -1 + case "on": + catchallOverride = 1 + case "off": + catchallOverride = 0 + default: + h.writeError(w, http.StatusBadRequest, "catchall_detection must be inherit, on, or off") + return + } + } + // nil ⇒ SQL NULL (inherit), otherwise the 0/1 override. + var catchallDB any + if catchallOverride >= 0 { + catchallDB = catchallOverride + } rawScope := "" if req.Domain != nil { @@ -512,9 +549,10 @@ func (h *Handler) handleUpdateTarget(w http.ResponseWriter, r *http.Request) { res, err := tx.ExecContext(r.Context(), ` UPDATE targets SET name = ?, description = ?, tags = ?, priority = ?, notes = ?, - exclude_scope = ?, enabled_modules = ?, scan_user_agent = ?, scan_headers = ?, updated_at = CURRENT_TIMESTAMP + exclude_scope = ?, enabled_modules = ?, scan_user_agent = ?, scan_headers = ?, + catchall_detection = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? - `, name, description, tagsJSON, priority, notes, excludeScope, modulesJSON, scanUserAgent, scanHeadersJSON, id) + `, name, description, tagsJSON, priority, notes, excludeScope, modulesJSON, scanUserAgent, scanHeadersJSON, catchallDB, id) if err != nil { h.writeError(w, http.StatusInternalServerError, "failed to update target") return @@ -1561,8 +1599,8 @@ func (h *Handler) handleTargetStats(w http.ResponseWriter, r *http.Request) { query string args []any }{ - {&stats.Subdomains, "SELECT COUNT(*) FROM subdomains WHERE target_id = ?", []any{id}}, - {&stats.AliveHosts, "SELECT COUNT(*) FROM subdomains WHERE target_id = ? AND is_alive = 1", []any{id}}, + {&stats.Subdomains, "SELECT COUNT(*) FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0) = 0", []any{id}}, + {&stats.AliveHosts, "SELECT COUNT(*) FROM subdomains WHERE target_id = ? AND is_alive = 1 AND COALESCE(catchall,0) = 0", []any{id}}, {&stats.HTTPServices, "SELECT COUNT(*) FROM http_services WHERE target_id = ?", []any{id}}, {&stats.JSFiles, "SELECT COUNT(*) FROM js_files WHERE target_id = ?", []any{id}}, {&stats.JSFindings, "SELECT COUNT(*) FROM js_findings WHERE target_id = ?", []any{id}}, @@ -1974,7 +2012,7 @@ func (h *Handler) mdReconSections(r *http.Request, b *strings.Builder, id string } } - rowsList("Subdomains", `SELECT subdomain, CASE is_alive WHEN 1 THEN 'alive' ELSE '' END, COALESCE(ip,'') FROM subdomains WHERE target_id=? ORDER BY subdomain LIMIT 5000`, 3) + rowsList("Subdomains", `SELECT subdomain, CASE is_alive WHEN 1 THEN 'alive' ELSE '' END, COALESCE(ip,'') FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0 ORDER BY subdomain LIMIT 5000`, 3) rowsList("Live HTTP Hosts", `SELECT url, status_code, COALESCE(title,'') FROM http_services WHERE target_id=? AND COALESCE(source,'probe')='probe' ORDER BY url LIMIT 3000`, 3) rowsList("Reflected Parameters", `SELECT url, parameter FROM parameters WHERE target_id=? AND is_reflected=1 ORDER BY url LIMIT 2000`, 2) rowsList("Open Redirects", `SELECT url, redirect_to FROM open_redirect_findings WHERE target_id=? ORDER BY url`, 2) @@ -2014,7 +2052,7 @@ func (h *Handler) handleExportTarget(w http.ResponseWriter, r *http.Request) { writer := csv.NewWriter(w) writer.Write([]string{"type", "url", "details"}) - rows, _ := h.db.QueryContext(r.Context(), "SELECT url, status_code FROM subdomains WHERE target_id = ?", id) + rows, _ := h.db.QueryContext(r.Context(), "SELECT url, status_code FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0) = 0", id) if rows != nil { for rows.Next() { var url string @@ -2029,7 +2067,7 @@ func (h *Handler) handleExportTarget(w http.ResponseWriter, r *http.Request) { default: export := map[string]any{"target": domain} - rows, _ := h.db.QueryContext(r.Context(), "SELECT subdomain, ip FROM subdomains WHERE target_id = ?", id) + rows, _ := h.db.QueryContext(r.Context(), "SELECT subdomain, ip FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0) = 0", id) if rows != nil { var subs []map[string]string for rows.Next() { diff --git a/internal/config/config.go b/internal/config/config.go index 3f11ce6..446bdd1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -212,6 +212,20 @@ type Config struct { // (written to config.json) or the RECON_DNS_RESOLVERS env var (comma-separated, // env wins). Set it to a single internal resolver to force the old behavior. DNSResolvers []string `json:"dns_resolvers"` + // MonitorModules overrides which pipeline modules the automatic periodic + // monitor watch task runs (the scheduled pass creating monitor_watch tasks + // in scheduler.runDueMonitors). Names are validated against the scheduler's + // AllModules catalog by the scheduler; unknown names are logged and skipped + // (never rejected) so neither the API layer nor the UI needs to know the + // catalog. If nothing valid remains, the built-in default watch set applies. + // The snapshot module ("monitor") is always hoisted ahead of refresh-capable + // modules (http_probe/js_analysis) server-side — snapshot-before-refresh is + // an invariant, not a form checkbox. Empty + // ⇒ the built-in watch set (subdomain_enum, monitor, http_probe, takeover, + // cors, backup_discovery). Set via the System → Scanning page (written to + // config.json; scheduler picks the list up on its next due check — no + // restart needed). + MonitorModules []string `json:"monitor_modules"` // SubdomainRecursionDepth controls recursive subdomain enumeration: after the // apex domain is enumerated, each newly-discovered RESOLVING subdomain is itself // enumerated (passive sources + resolution) as a sub-apex, and so on, until this @@ -220,6 +234,23 @@ type Config struct { // raise it to go deeper on a sprawling target. Bounded per level // (SubdomainRecursionMaxPerLevel) so it can't explode on a huge zone. SubdomainRecursionDepth int `json:"subdomain_recursion_depth"` + // SubdomainRecursionEnabled is the deployment default for recursive + // enumeration. Off ⇒ apex only (the pre-recursion behavior); the depth value + // stays stored for when it is switched back on. Default false: recursion + // multiplies DNS query volume and invites public-resolver throttling. A scan + // can still opt in through the scan dialog's recursion toggle. + // Set via the System → Scanning panel, next to the depth field. + SubdomainRecursionEnabled bool `json:"subdomain_recursion_enabled"` + // HTTPCatchallDetection is OPT-IN (default false). When enabled, the end of + // the HTTP-probe module groups stored 2xx http_services fingerprints and + // marks subdomains that serve a wildcard catch-all page — either a redirect + // fleet aimed at the apex, or >=5 live hosts byte-identical to the apex + // homepage. Marked hosts are excluded from the Hosts tab, reports, graph and + // wordlist learning; their rows stay stored (marked, not deleted: seeds, + // vhosts and anything the operator vouched for are never touched). The pass + // reads already-stored responses and sends NO extra network requests; it + // costs under a second. Toggle via the System → Scanning panel. + HTTPCatchallDetection bool `json:"http_catchall_detection"` // ScanWatchdogHours bounds one scan PHASE so a wedged module can never hold // its concurrency slot forever (the actual purpose — see scheduler.go). It // is deliberately reset after every completed phase, so the combined runtime @@ -505,15 +536,16 @@ func defaultConfig() *Config { // (see the field comments) instead of baking a fixed numeric ceiling into // every config.json ever saved — an operator who wants a REAL cap sets one // explicitly; a fresh install gets the generous built-in default. - ScanWatchdogHours: 24, // per-phase watchdog floor; reset on progress and scaled for large targets - EnableDAST: true, // native context-aware DAST (XSS/SQLi) over all insertion points - AIModel: "claude-opus-4-8", // latest Opus; used only when AIEnabled and a key is set - AIMaxIterations: 40, // hard cap on the agent loop - AuthzMode: "balanced", // two-identity BOLA: deep auth crawl + read/write differential - AuthzDestructive: false, // cross-user DELETE stays opt-in - SubdomainRecursionDepth: 4, // recurse passive+resolve into each resolving subdomain, up to 4 levels - Workers: workers, // auto-scaled to host CPU (floors = previous fixed defaults) - Limits: limits, // auto-scaled to host CPU/RAM + ScanWatchdogHours: 24, // per-phase watchdog floor; reset on progress and scaled for large targets + EnableDAST: true, // native context-aware DAST (XSS/SQLi) over all insertion points + AIModel: "claude-opus-4-8", // latest Opus; used only when AIEnabled and a key is set + AIMaxIterations: 40, // hard cap on the agent loop + AuthzMode: "balanced", // two-identity BOLA: deep auth crawl + read/write differential + AuthzDestructive: false, // cross-user DELETE stays opt-in + SubdomainRecursionEnabled: false, // apex-only by default; System panel sets the deployment default, per-scan toggle overrides + SubdomainRecursionDepth: 4, // recurse passive+resolve into each resolving subdomain, up to 4 levels + Workers: workers, // auto-scaled to host CPU (floors = previous fixed defaults) + Limits: limits, // auto-scaled to host CPU/RAM } } diff --git a/internal/database/migrations.go b/internal/database/migrations.go index 6005414..ffa84b4 100644 --- a/internal/database/migrations.go +++ b/internal/database/migrations.go @@ -100,6 +100,7 @@ func RunMigrations(db *DB) error { alterNucleiFindingsAddVerification, alterNucleiFindingsAddConfidence, alterSubdomainsAddSource, + alterSubdomainsAddMissCount, alterCandidatesAddStateVersion, alterVulnFindingsAddCandidateID, alterVulnFindingsAddLifecycle, @@ -156,7 +157,13 @@ func RunMigrations(db *DB) error { alterTasksAddAssetsTotal, alterAdminPanelFindingsAddScreenshotID, alterVulnFindingsAddScreenshotID, + createCSSFilesTable, + alterJSFilesAddBody, + alterJSFilesAddOldHash, + alterJSFilesAddOldBody, createWPSitesTable, + alterSubdomainsAddCatchall, + alterTargetsAddCatchallDetection, } for i, m := range migrations { @@ -1071,6 +1078,26 @@ CREATE TABLE IF NOT EXISTS subdomains ( // hosts, which are often internal/forgotten and high-value. const alterSubdomainsAddSource = `ALTER TABLE subdomains ADD COLUMN source TEXT DEFAULT 'dns';` +// A name is not dead after ONE missed resolve pass: a single resolver timeout +// or dnsx hiccup must not delete a previously verified row (and, with it, the +// recursion frontier seed). pruneRejectedSubdomains increments this counter on +// every rejected pass and only deletes at 2 consecutive misses; a pass that +// resolves the name resets it to 0 via the admission upsert. +const alterSubdomainsAddMissCount = `ALTER TABLE subdomains ADD COLUMN miss_count INTEGER DEFAULT 0;` + +// catchall marks subdomains whose HTTP responses match the target's wildcard +// catch-all fleet (redirect-to-apex groups or homepage-content clones). Rows are +// marked, never deleted: operator seeds, vhost findings and previously-alive +// evidence stay untouched, and resurfaces of the same page are idempotent. +// Human-facing surfaces (Hosts tab, reports, graph, adaptive wordlist) filter +// COALESCE(catchall,0)=0. Written only when the opt-in config flag is set. +const alterSubdomainsAddCatchall = `ALTER TABLE subdomains ADD COLUMN catchall INTEGER NOT NULL DEFAULT 0;` + +// alterTargetsAddCatchallDetection adds the per-target tri-state override of the +// HTTP catch-all pass: NULL = inherit the global http_catchall_detection config, +// 0 = force off for this target, 1 = force on. +const alterTargetsAddCatchallDetection = `ALTER TABLE targets ADD COLUMN catchall_detection INTEGER DEFAULT NULL;` + // ── P1 canonical lifecycle bridge ──────────────────────────────────────────── // candidates is the AUTHORITATIVE lifecycle state source. state_version is a // monotonic counter used for optimistic, concurrency-safe transitions. @@ -1150,6 +1177,31 @@ CREATE TABLE IF NOT EXISTS js_findings ( FOREIGN KEY (js_file_id) REFERENCES js_files(id) ON DELETE CASCADE );` +const createCSSFilesTable = ` +CREATE TABLE IF NOT EXISTS css_files ( + id TEXT PRIMARY KEY, + target_id TEXT NOT NULL, + url TEXT NOT NULL, + size INTEGER DEFAULT 0, + hash TEXT DEFAULT '', + body TEXT DEFAULT '', + old_hash TEXT DEFAULT '', + old_body TEXT DEFAULT '', + last_seen DATETIME DEFAULT CURRENT_TIMESTAMP, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (target_id) REFERENCES targets(id) ON DELETE CASCADE, + UNIQUE(target_id, url) +);` + +// Body capture for diffing (monitor pass). The monitor keeps the previous +// content next to the current one so a `css_change`/`js_change` row carries a +// real line-level diff in the dashboard instead of two opaque SHA-256 hashes. +// Bodies are capped (monitor body cap) — truncated captures still diff the +// first N lines, which is where most real changes live. +const alterJSFilesAddBody = `ALTER TABLE js_files ADD COLUMN body TEXT DEFAULT '';` +const alterJSFilesAddOldHash = `ALTER TABLE js_files ADD COLUMN old_hash TEXT DEFAULT '';` +const alterJSFilesAddOldBody = `ALTER TABLE js_files ADD COLUMN old_body TEXT DEFAULT '';` + const createParametersTable = ` CREATE TABLE IF NOT EXISTS parameters ( id TEXT PRIMARY KEY, diff --git a/internal/models/models.go b/internal/models/models.go index 916644f..ee91f8a 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -29,6 +29,14 @@ type Target struct { MonitorLastRun *time.Time `json:"monitor_last_run" db:"monitor_last_run"` ScanUserAgent string `json:"scan_user_agent" db:"scan_user_agent"` ScanHeaders map[string]string `json:"scan_headers" db:"scan_headers"` + // CatchallDetection is the per-target override of the HTTP catch-all pass: + // "inherit" (follow the global http_catchall_detection setting), "on" or + // "off". CatchallEffective is the resolved state the next scan will use. + CatchallDetection string `json:"catchall_detection,omitempty"` + // CatchallEffective is a pointer so list/create responses (which do not + // resolve the override) omit the field entirely, while the detail endpoint + // always sends an explicit true/false. + CatchallEffective *bool `json:"catchall_effective,omitempty"` } // Asset is one individually-scannable entry of a target (a domain, IP, CIDR, or diff --git a/internal/scanner/adaptive_wordlist.go b/internal/scanner/adaptive_wordlist.go index 53064df..9e1f9a0 100644 --- a/internal/scanner/adaptive_wordlist.go +++ b/internal/scanner/adaptive_wordlist.go @@ -148,7 +148,7 @@ func buildAdaptiveWordlist(ctx context.Context, db *database.DB, cfg *config.Con } rows.Close() } - rows, err = db.QueryContext(ctx, `SELECT subdomain FROM subdomains WHERE target_id=? LIMIT 10000`, targetID) + rows, err = db.QueryContext(ctx, `SELECT subdomain FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0 LIMIT 10000`, targetID) if err == nil { for rows.Next() { var host string diff --git a/internal/scanner/asset_priority.go b/internal/scanner/asset_priority.go index 0d0448b..da037ce 100644 --- a/internal/scanner/asset_priority.go +++ b/internal/scanner/asset_priority.go @@ -171,7 +171,7 @@ func loadAssetRows(ctx context.Context, db *database.DB, targetID string) (map[s srows, err := db.QueryContext(ctx, ` SELECT subdomain, is_alive, status_code, COALESCE(page_title,''), COALESCE(server,''), COALESCE(technologies,'[]'), COALESCE(waf,''), COALESCE(source,'dns') - FROM subdomains WHERE target_id=?`, targetID) + FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0`, targetID) if err != nil { return nil, err } diff --git a/internal/scanner/dnsx_parse_test.go b/internal/scanner/dnsx_parse_test.go new file mode 100644 index 0000000..8103e32 --- /dev/null +++ b/internal/scanner/dnsx_parse_test.go @@ -0,0 +1,46 @@ +package scanner + +import ( + "context" + "testing" + + "github.com/recon-platform/internal/config" +) + +func TestParseDNSXResponseLine(t *testing.T) { + cases := []struct { + line string + name string + want []string + }{ + {"www.example.com [34.117.59.50, 34.149.123.216]", "www.example.com", []string{"34.117.59.50", "34.149.123.216"}}, + {"www.example.com [157.240.1.35 2a03:2880:f142:83:face:b00c:0:25de]", "www.example.com", []string{"157.240.1.35", "2a03:2880:f142:83:face:b00c:0:25de"}}, + {"www.example.com [Example.Edge.Cdn.Net.]", "www.example.com", []string{"example.edge.cdn.net"}}, + {"mixed.example.com [1.2.3.4, something.example.net]", "mixed.example.com", []string{"1.2.3.4", "something.example.net"}}, + {"trailing.example.com. [ 1.2.3.4 ]", "trailing.example.com", []string{"1.2.3.4"}}, + {"no brackets here", "", nil}, + {"[1.2.3.4] alone", "", nil}, + } + for i, tc := range cases { + gotName, gotToks := parseDNSXResponseLine(tc.line) + if gotName != tc.name { + t.Errorf("case %d: name = %q, want %q", i, gotName, tc.name) + } + if len(gotToks) != len(tc.want) { + t.Errorf("case %d: tokens = %v, want %v", i, gotToks, tc.want) + continue + } + for j := range gotToks { + if gotToks[j] != tc.want[j] { + t.Errorf("case %d: token %d = %q, want %q", i, j, gotToks[j], tc.want[j]) + } + } + } +} + +func TestDNSXResolveIPsNilWithoutExecutor(t *testing.T) { + s := &SubdomainScanner{cfg: &config.Config{}} + if got := s.dnsxResolveIPs(context.Background(), "target", []string{"a.example.com"}); got != nil { + t.Fatalf("nil-executor scanner must signal fallback (nil map), got %v", got) + } +} diff --git a/internal/scanner/http.go b/internal/scanner/http.go index a59f925..49d9700 100644 --- a/internal/scanner/http.go +++ b/internal/scanner/http.go @@ -53,9 +53,10 @@ func (s *HTTPScanner) Run(ctx context.Context, targetID string, logFn LogFunc) e logFn("info", "http_probe", "Loading subdomains for HTTP probing...") rows, err := s.db.QueryContext(ctx, ` - SELECT subdomain FROM subdomains + SELECT subdomain FROM subdomains WHERE target_id = ? AND (COALESCE(ip,'') != '' OR COALESCE(source,'dns') IN ('seed','vhost')) + AND COALESCE(catchall,0) = 0 ORDER BY subdomain `, targetID) if err != nil { @@ -203,6 +204,17 @@ func (s *HTTPScanner) Run(ctx context.Context, targetID string, logFn LogFunc) e // false positives on it. s.fingerprintHosts(ctx, targetID, logFn) + // Opt-in HTTP catch-all pass (global config http_catchall_detection, default + // OFF, overridable per target): groups the 2xx fingerprints gathered above and + // marks wildcard catch-all fleets (redirect-to-apex / homepage clones) so they + // stop polluting the Hosts tab, reports and web modules. Reads stored data + // only — no requests. + if s.catchallEnabledForTarget(ctx, targetID) { + if err := s.runCatchallPass(ctx, targetID, logFn); err != nil && ctx.Err() == nil { + logFn("warn", "http_probe", fmt.Sprintf("HTTP catch-all pass failed: %v", err)) + } + } + // Authenticated-session preflight: validate configured identities NOW, at the // front of the scan, so an expired login is surfaced before every downstream // injection/authz module wastes itself against the logged-out app. diff --git a/internal/scanner/http_catchall.go b/internal/scanner/http_catchall.go new file mode 100644 index 0000000..8da48c7 --- /dev/null +++ b/internal/scanner/http_catchall.go @@ -0,0 +1,314 @@ +package scanner + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "net/url" + "sort" + "strings" +) + +// The HTTP catch-all pass closes the hole the DNS wildcard gate cannot see: +// a zone with NO DNS wildcard can still serve one shared homepage or one +// redirect for EVERY hostname under it (parking-style hosting, SaaS +// platforms). Those names arrive as fully "verified" subdomains — the DNS gate +// happily passes them — and flood the Hosts tab, reports and every web module +// downstream. +// +// The pass runs at the END of the HTTP-probe module (only when the operator +// enabled it in config; default is OFF) and groups the http_services rows that +// the probing ALREADY collected: +// +// - Redirect groups: 3+ hosts that share one fingerprint AND redirect to the +// apex (or www-apex). This is the classic wildcard redirect the operator +// reported ("asdf-garbage.git." style junk pointing at the homepage). +// - Homepage clones: 5+ hosts byte-identical (status), title-, server-, +// content-type/length-, tech-, WAF/CMS-fingerprint-identical to the apex +// homepage that was probed in the same run. +// +// Grouping is 2xx-only by design: 401/403 auth walls legitimately serve the +// same shell for every host and must never be marked; per-host SSO redirects +// carry state that makes their fingerprints differ. The residual risk is a +// legit byte-identical 200-fleet that is NOT apex-related — rare, and +// reversible since rows are marked, never deleted, and the whole pass is a +// config toggle. +// +// Zero network cost: no DNS, no extra HTTP requests — the pass reads the +// fingerprints stored seconds earlier. + +const ( + catchallMinRedirectGroup = 3 // redirect-to-apex fleets need less agreement: pointing at the apex is the wildcard's signature + catchallMinContentGroup = 5 // homepage cloning is only "catch-all-shaped" with more members; protects legitimate twins (staging/prod mirrors) +) + +// hostFromServiceURL extracts and normalizes the hostname of an http_services +// URL. Probing stores scheme://host[:port][/path]; the subdomain matching only +// cares about the registered hostname part. +func hostFromServiceURL(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + if u, err := url.Parse(raw); err == nil && u.Hostname() != "" { + return strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) + } + // Fallback for scheme-less or malformed rows: cut the authority. + rest := raw + rest = strings.TrimPrefix(rest, "https://") + rest = strings.TrimPrefix(rest, "http://") + if idx := strings.IndexAny(rest, "/?#"); idx >= 0 { + rest = rest[:idx] + } + if idx := strings.LastIndex(rest, "@"); idx >= 0 { // strip userinfo + rest = rest[idx+1:] + } + if idx := strings.Index(rest, ":"); idx >= 0 { // strip port + rest = rest[:idx] + } + return strings.ToLower(strings.TrimSuffix(rest, ".")) +} + +// sortedTechnologies normalizes the stored technologies JSON so two hosts with +// the same tech stack in different order share one fingerprint. +func sortedTechnologies(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" || raw == "[]" { + return "" + } + var techs []string + if err := json.Unmarshal([]byte(raw), &techs); err != nil { + return strings.ToLower(raw) // tolerate non-JSON rows; still comparable + } + sort.Strings(techs) + return strings.ToLower(strings.Join(techs, ",")) +} + +type catchallRow struct { + host string + contentFP string // status|title|server|content-type|length|techs|waf|cms + redirectHost string // lowercase target hostname, "" when no redirect +} + +// buildCatchallRow computes the grouping fields of one http_services row. +func buildCatchallRow(rawURL string, statusCode int, title, server, contentType string, contentLength int, redirectURL, technologies, waf, cms string) catchallRow { + var fp strings.Builder + fmt.Fprintf(&fp, "%d|%s|%s|%s|%d|%s|%s|%s", + statusCode, strings.ToLower(strings.TrimSpace(title)), strings.ToLower(strings.TrimSpace(server)), + strings.ToLower(strings.TrimSpace(contentType)), contentLength, + sortedTechnologies(technologies), strings.ToLower(strings.TrimSpace(waf)), strings.ToLower(strings.TrimSpace(cms))) + return catchallRow{ + host: hostFromServiceURL(rawURL), + contentFP: fp.String(), + redirectHost: normalizeSubdomainName(hostFromServiceURL(redirectURL)), + } +} + +// catchallApexTargets reports the hostnames a wildcard redirect aims at: the +// apex itself or the canonical www host. +func catchallApexTargets(apex string) map[string]bool { + targets := map[string]bool{apex: true, "www." + apex: true} + return targets +} + +// groupCatchallFingerprints groups rows by content fingerprint. +func groupCatchallFingerprints(rows []catchallRow) map[string][]catchallRow { + groups := map[string][]catchallRow{} + for _, row := range rows { + groups[row.contentFP] = append(groups[row.contentFP], row) + } + return groups +} + +// qualifiesCatchallGroup decides whether a fingerprint group is a wildcard +// fleet and returns the members to mark. rows must exclude non-2xx entries. +// The apex and www hosts are never in the returned set: they ARE the origin +// the wildcard copies, so marking them would hide the real site. +func qualifiesCatchallGroup(rows []catchallRow, apexTargets map[string]bool) (marked []catchallRow, kind string) { + if len(rows) == 0 { + return nil, "" + } + redirecting := 0 + hasApexMember := false + hasForeignRedirect := false + for _, row := range rows { + if row.redirectHost != "" { + if apexTargets[row.redirectHost] { + redirecting++ + } else { + // One member redirects somewhere else entirely — a mixed + // group of distinct apps, not a wildcard funnel. + hasForeignRedirect = true + } + } + if apexTargets[row.host] { + hasApexMember = true + } + } + if redirecting >= catchallMinRedirectGroup && !hasForeignRedirect { + // At least three identical pages funnel into the apex and NOTHING in + // the group redirects elsewhere: a wildcard redirect. The apex/www + // rows stay unmarked (they ARE the origin). + for _, row := range rows { + if !apexTargets[row.host] { + marked = append(marked, row) + } + } + return marked, "redirect" + } + if hasApexMember && len(rows) >= catchallMinContentGroup { + // The apex homepage itself sits inside a large identical-content + // group: every member serves the apex's homepage — a content + // catch-all. The apex/www rows stay unmarked (they ARE the origin); + // strictly-deeper members get marked. + for _, row := range rows { + if !apexTargets[row.host] { + marked = append(marked, row) + } + } + return marked, "homepage" + } + return nil, "" +} + +// catchallEnabledForTarget resolves the per-target tri-state override over the +// global config default: NULL/absent column inherits http_catchall_detection, +// 0 forces the pass off for this target, 1 forces it on. An unknown target or +// query error falls back to the global default (the pass is best-effort). +func (s *HTTPScanner) catchallEnabledForTarget(ctx context.Context, targetID string) bool { + global := s.cfg != nil && s.cfg.HTTPCatchallDetection + var override sql.NullInt64 + if err := s.db.QueryRowContext(ctx, `SELECT catchall_detection FROM targets WHERE id=?`, targetID).Scan(&override); err != nil || !override.Valid { + return global + } + return override.Int64 != 0 +} + +// runCatchallPass implements the opt-in step-2 pass. It resolves the target's +// apex from the targets row, fingerprints all stored 2xx services, marks the +// wildcard fleet members (DNS-source rows only) and drops their +// http_services rows so web modules never fetch them again. +func (s *HTTPScanner) runCatchallPass(ctx context.Context, targetID string, logFn LogFunc) error { + var apex string + if err := s.db.QueryRowContext(ctx, `SELECT domain FROM targets WHERE id=?`, targetID).Scan(&apex); err != nil || strings.TrimSpace(apex) == "" { + // Unknown target — the pass is best-effort; probing already logged context. + return nil + } + apex = normalizeSubdomainName(apex) + + rows, err := s.db.QueryContext(ctx, ` + SELECT url, status_code, title, COALESCE(server,''), COALESCE(content_type,''), + COALESCE(content_length,0), COALESCE(redirect_url,''), COALESCE(technologies,'[]'), + COALESCE(waf,''), COALESCE(cms,'') + FROM http_services + WHERE target_id=? AND status_code BETWEEN 200 AND 299`, targetID) + if err != nil { + return nil + } + defer rows.Close() + + var candidateRows []catchallRow + var total2xx int + for rows.Next() { + var rawURL, title, server, contentType, redirectURL, technologies, waf, cms string + var statusCode, contentLength int + if rows.Scan(&rawURL, &statusCode, &title, &server, &contentType, &contentLength, &redirectURL, &technologies, &waf, &cms) != nil { + continue + } + total2xx++ + row := buildCatchallRow(rawURL, statusCode, title, server, contentType, contentLength, redirectURL, technologies, waf, cms) + if row.host == "" { + continue + } + // Apex/www rows stay in the candidate pool (they anchor the homepage + // group) but can only ever be marked by group logic, never as strays. + candidateRows = append(candidateRows, row) + } + rows.Close() + + if len(candidateRows) == 0 { + logFn("info", "http_probe", fmt.Sprintf("HTTP catch-all pass: %d stored 2xx service(s) — nothing to group", total2xx)) + return nil + } + + apexTargets := catchallApexTargets(apex) + groups := groupCatchallFingerprints(candidateRows) + + markedSet := map[string]string{} // host -> pattern kind (deduped across schemes/groups) + redirectGroups, homepageGroups := 0, 0 + for _, members := range groups { + marked, kind := qualifiesCatchallGroup(members, apexTargets) + if kind == "" { + continue + } + if kind == "redirect" { + redirectGroups++ + } else { + homepageGroups++ + } + for _, row := range marked { + // Defense in depth: only hosts strictly under the target apex are + // ever marked (qualifies already excludes apex/www; this also + // rejects any foreign host a stray service row might carry). + if row.host == apex || !strings.HasSuffix(row.host, "."+apex) { + continue + } + if _, seen := markedSet[row.host]; !seen { + markedSet[row.host] = kind + } + } + } + + if len(markedSet) == 0 { + logFn("info", "http_probe", fmt.Sprintf("HTTP catch-all pass: %d live 2xx host(s) analysed — no catch-all groups", total2xx)) + return nil + } + + markedHosts := make([]string, 0, len(markedSet)) + for host := range markedSet { + markedHosts = append(markedHosts, host) + } + sort.Strings(markedHosts) + for _, host := range markedHosts { + logFn("info", "http_probe", fmt.Sprintf("HTTP catch-all: %s serves the wildcard group (%s pattern)", host, markedSet[host])) + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + stmt, err := tx.PrepareContext(ctx, `UPDATE subdomains SET catchall=1 + WHERE target_id=? AND subdomain=? AND COALESCE(source,'dns')='dns' AND COALESCE(catchall,0)=0`) + if err != nil { + _ = tx.Rollback() + return err + } + urlStmt, err := tx.PrepareContext(ctx, `DELETE FROM http_services + WHERE target_id=? AND (url LIKE 'https://'||?||'/%' OR url LIKE 'https://'||?||':%' + OR url LIKE 'http://'||?||'/%' OR url LIKE 'http://'||?||':%' OR url='https://'||? OR url='http://'||?)`) + if err != nil { + _ = tx.Rollback() + return err + } + marked := 0 + for _, host := range markedHosts { + res, _ := stmt.ExecContext(ctx, targetID, host) + if affRows, aerr := res.RowsAffected(); aerr == nil && affRows > 0 { + marked++ + // Drop the junk services so JS/params/DAST/nuclei never target + // them again. The subdomain row keeps the evidence of WHY. + _, _ = urlStmt.ExecContext(ctx, targetID, host, host, host, host, host, host) + } + } + _ = stmt.Close() + _ = urlStmt.Close() + if err := tx.Commit(); err != nil { + return err + } + + logFn("info", "http_probe", fmt.Sprintf( + "HTTP catch-all pass: %d live 2xx host(s) analysed — %d wildcard group(s) (%d redirect, %d homepage), %d host(s) marked and hidden", + total2xx, redirectGroups+homepageGroups, redirectGroups, homepageGroups, marked)) + return nil +} diff --git a/internal/scanner/http_catchall_test.go b/internal/scanner/http_catchall_test.go new file mode 100644 index 0000000..a19980b --- /dev/null +++ b/internal/scanner/http_catchall_test.go @@ -0,0 +1,232 @@ +package scanner + +import ( + "context" + "fmt" + "testing" + + "github.com/recon-platform/internal/config" +) + +func TestHostFromServiceURL(t *testing.T) { + cases := map[string]string{ + "https://junk.realestate.com.au": "junk.realestate.com.au", + "http://junk.realestate.com.au/login": "junk.realestate.com.au", + "https://junk.realestate.com.au:8443/": "junk.realestate.com.au", + "https://user:pass@junk.realestate.com.au/": "junk.realestate.com.au", + "JUNK.REALESTATE.com.au.": "junk.realestate.com.au", + "": "", + "/only/path": "", + } + for raw, want := range cases { + if got := hostFromServiceURL(raw); got != want { + t.Fatalf("hostFromServiceURL(%q) = %q, want %q", raw, got, want) + } + } +} + +func TestSortedTechnologies(t *testing.T) { + if got := sortedTechnologies(`["nginx","WordPress"]`); got != "wordpress,nginx" { + t.Fatalf("mixed order/case must normalize (sort is byte-order, lowercase after): %q", got) + } + if got := sortedTechnologies(`[]`); got != "" { + t.Fatalf("empty list must be an empty string, got %q", got) + } + if got := sortedTechnologies(`not-json`); got != "not-json" { + t.Fatalf("non-JSON rows must stay comparable: %q", got) + } +} + +func TestQualifiesCatchallGroups(t *testing.T) { + apexTargets := catchallApexTargets("realestate.com.au") + + // Redirect fleet: 3 junk members, all funneling into the apex → marked; + // an apex/www member must never be marked. + redirect := []catchallRow{ + {host: "a.realestate.com.au", contentFP: "FP", redirectHost: "realestate.com.au"}, + {host: "b.realestate.com.au", contentFP: "FP", redirectHost: "www.realestate.com.au"}, + {host: "asdf-garbage.git.realestate.com.au", contentFP: "FP", redirectHost: "realestate.com.au"}, + } + marked, kind := qualifiesCatchallGroup(redirect, apexTargets) + if kind != "redirect" || len(marked) != 3 { + t.Fatalf("3-member redirect fleet must be marked (kind=%q n=%d)", kind, len(marked)) + } + mixed := append(append([]catchallRow{}, redirect...), catchallRow{host: "realestate.com.au", contentFP: "FP"}) + marked, kind = qualifiesCatchallGroup(mixed, apexTargets) + if kind != "redirect" || len(marked) != 3 { + t.Fatalf("fleet+apex group must mark only junk members (kind=%q n=%d)", kind, len(marked)) + } + for _, row := range marked { + if apexTargets[row.host] { + t.Fatalf("apex/www host marked as catch-all: %q", row.host) + } + } + + // Same fingerprint with members split across redirect targets → not a fleet. + half := []catchallRow{ + {host: "a.realestate.com.au", contentFP: "FP", redirectHost: "realestate.com.au"}, + {host: "b.realestate.com.au", contentFP: "FP", redirectHost: "realestate.com.au"}, + {host: "c.realestate.com.au", contentFP: "FP", redirectHost: "elsewhere.example"}, + } + if _, kind = qualifiesCatchallGroup(half, apexTargets); kind != "" { + t.Fatalf("partial redirect agreement must not mark (kind=%q)", kind) + } + + // Homepage clones: apex row anchors a group of >=5 identical fingerprints. + home := []catchallRow{{host: "realestate.com.au", contentFP: "HOMEFP"}} + for i := 0; i < 5; i++ { + home = append(home, catchallRow{host: fmt.Sprintf("clone%d.git.realestate.com.au", i), contentFP: "HOMEFP"}) + } + marked, kind = qualifiesCatchallGroup(home, apexTargets) + if kind != "homepage" || len(marked) != 5 { + t.Fatalf("apex-anchored clone group must mark the 5 junk rows (kind=%q n=%d)", kind, len(marked)) + } + + // 4 identical members without the apex row must NOT mark (a legit twin + // pair fleet is not catch-all evidence on its own). + twins := make([]catchallRow, 0, 4) + for i := 0; i < 4; i++ { + twins = append(twins, catchallRow{host: fmt.Sprintf("twin%d.realestate.com.au", i), contentFP: "TW"}) + } + if _, kind = qualifiesCatchallGroup(twins, apexTargets); kind != "" { + t.Fatalf("4 identical hosts without the apex must stay unmarked (kind=%q)", kind) + } + + // With the apex but only 3 members → below the content threshold (4 total). + short := []catchallRow{ + catchallRow{host: "realestate.com.au", contentFP: "TW"}, + catchallRow{host: "twin0.realestate.com.au", contentFP: "TW"}, + catchallRow{host: "twin1.realestate.com.au", contentFP: "TW"}, + catchallRow{host: "twin2.realestate.com.au", contentFP: "TW"}, + } + if _, kind = qualifiesCatchallGroup(short, apexTargets); kind != "" { + t.Fatalf("3 members + apex is below the content threshold (kind=%q)", kind) + } +} + +func TestRunCatchallPassEndToEnd(t *testing.T) { + db, tid := testDB(t) + ctx := context.Background() + + seed := func(host string, statusCode int, title string, redirect string, source string) { + if _, err := db.Exec(`INSERT INTO subdomains (id, target_id, subdomain, ip, source, catchall, is_alive) + VALUES (?, ?, ?, '10.0.0.1', ?, 0, 1)`, "ct-"+host, tid, host, source); err != nil { + t.Fatalf("seed subdomain %s: %v", host, err) + } + if _, err := db.Exec(`INSERT INTO http_services (id, target_id, url, status_code, title, server, redirect_url, technologies) + VALUES (?, ?, ?, ?, ?, 'nginx', ?, '[]')`, + "ct-svc-"+host, tid, "https://"+host, statusCode, title, redirect); err != nil { + t.Fatalf("seed service %s: %v", host, err) + } + } + + // The wildcard homepage fleet: apex + 5 junk clones with identical title. + seed("m.local", 200, "Real Estate Portal", "", "dns") + for i := 0; i < 5; i++ { + seed(fmt.Sprintf("junk%d.git.m.local", i), 200, "Real Estate Portal", "", "dns") + } + // An auth wall that legitimately shares a shell across hosts: 401s are + // outside the 2xx window and must never group. + seed("portal.m.local", 401, "Sign in", "", "dns") + // An operator-vouched seed that shares the homepage content: seed/vhost + // rows are never re-marked by the pass. + seed("operator-host.m.local", 200, "Real Estate Portal", "", "seed") + + scanner := &HTTPScanner{db: db} + logFn := func(_ string, _, _ string) {} + if err := scanner.runCatchallPass(ctx, tid, logFn); err != nil { + t.Fatalf("runCatchallPass: %v", err) + } + + var junk int + if err := db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE catchall=1 AND target_id=?`, tid).Scan(&junk); err != nil { + t.Fatalf("count marked: %v", err) + } + if junk != 5 { + t.Fatalf("expected the 5 junk clones marked, got %d", junk) + } + var apex int + if err := db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE target_id=? AND subdomain='m.local' AND catchall=1`, tid).Scan(&apex); err != nil { + t.Fatalf("count apex: %v", err) + } + if apex != 0 { + t.Fatalf("the apex itself must never be marked") + } + var services int + if err := db.QueryRow(`SELECT COUNT(*) FROM http_services WHERE target_id=? AND url LIKE 'https://junk%'`, tid).Scan(&services); err != nil { + t.Fatalf("count junk services: %v", err) + } + if services != 0 { + t.Fatalf("junk http_services rows must be dropped so web modules skip them, got %d", services) + } + var left int + if err := db.QueryRow(`SELECT COUNT(*) FROM http_services WHERE target_id=?`, tid).Scan(&left); err != nil { + t.Fatalf("count left services: %v", err) + } + if left != 3 { // apex + operator seed stay; the 401 auth wall is never marked, so it stays too + t.Fatalf("apex + seed + 401 services must survive, got %d", left) + } + + // Idempotency: a second pass (next scan) must not change anything — the + // junk services are already gone and the marks already exist. + if err := scanner.runCatchallPass(ctx, tid, logFn); err != nil { + t.Fatalf("second runCatchallPass: %v", err) + } + var junkAgain int + if err := db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE catchall=1 AND target_id=?`, tid).Scan(&junkAgain); err != nil { + t.Fatalf("count marked after rerun: %v", err) + } + if junkAgain != 5 { + t.Fatalf("second pass must be idempotent, marked=%d", junkAgain) + } + var leftAgain int + if err := db.QueryRow(`SELECT COUNT(*) FROM http_services WHERE target_id=?`, tid).Scan(&leftAgain); err != nil { + t.Fatalf("count left services after rerun: %v", err) + } + if leftAgain != 3 { + t.Fatalf("second pass must not delete surviving services, got %d", leftAgain) + } +} + +// TestCatchallEnabledForTargetMatrix proves the tri-state resolution order: +// a per-target override always wins; NULL (inherit) follows the global config; +// a missing target falls back to the global config too. +func TestCatchallEnabledForTargetMatrix(t *testing.T) { + db, tid := testDB(t) + ctx := context.Background() + + globalOn := &HTTPScanner{db: db, cfg: &config.Config{HTTPCatchallDetection: true}} + globalOff := &HTTPScanner{db: db, cfg: &config.Config{HTTPCatchallDetection: false}} + + setOverride := func(value any) { + t.Helper() + if _, err := db.Exec(`UPDATE targets SET catchall_detection=? WHERE id=?`, value, tid); err != nil { + t.Fatalf("set catchall_detection=%v: %v", value, err) + } + } + + setOverride(nil) // inherit + if !globalOn.catchallEnabledForTarget(ctx, tid) { + t.Fatal("inherit must follow global=on") + } + if globalOff.catchallEnabledForTarget(ctx, tid) { + t.Fatal("inherit must follow global=off") + } + + setOverride(1) // force on, even though the global is off + if !globalOff.catchallEnabledForTarget(ctx, tid) { + t.Fatal("override=on must beat global=off") + } + + setOverride(0) // force off, even though the global is on + if globalOn.catchallEnabledForTarget(ctx, tid) { + t.Fatal("override=off must beat global=on") + } + + if !globalOn.catchallEnabledForTarget(ctx, "missing-target") { + t.Fatal("missing target must fall back to the global config") + } + if globalOff.catchallEnabledForTarget(ctx, "missing-target") { + t.Fatal("missing target must fall back to the global config (off)") + } +} diff --git a/internal/scanner/monitor.go b/internal/scanner/monitor.go index 6b929f7..45dca0c 100644 --- a/internal/scanner/monitor.go +++ b/internal/scanner/monitor.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "net/url" + "regexp" "strings" "time" @@ -105,8 +106,20 @@ func (s *MonitorScanner) Run(ctx context.Context, targetID string, logFn LogFunc logFn("info", "monitor", fmt.Sprintf("Monitoring %d HTTP services for changes...", len(services))) changed := 0 + harvested := 0 knownHTTP := map[string]bool{} knownJS := map[string]bool{} + knownCSS := map[string]bool{} + cssKeyRows, err := s.db.QueryContext(ctx, `SELECT url FROM css_files WHERE target_id = ?`, targetID) + if err == nil { + for cssKeyRows.Next() { + var u string + if cssKeyRows.Scan(&u) == nil { + knownCSS[monitorURLKey(u)] = true + } + } + cssKeyRows.Close() + } for _, svc := range services { knownHTTP[monitorURLKey(svc.URL)] = true @@ -120,6 +133,7 @@ func (s *MonitorScanner) Run(ctx context.Context, targetID string, logFn LogFunc // NORMALIZED hash: volatile values (UUIDs/timestamps/CSRF/high-entropy) // are stripped first, so a dynamic page doesn't false-positive. newNormHash := normalizedHash(body) + harvested += s.harvestStylesheets(ctx, targetID, svc.URL, body, knownCSS) // ── Security-attribute change detection (supply-chain / form-hijack) ── curSnap := extractSecuritySnapshot(body, hostOf(svc.URL)) @@ -217,14 +231,19 @@ func (s *MonitorScanner) Run(ctx context.Context, targetID string, logFn LogFunc `, newStatus, newTitle, len(body), newNormHash, curSnap.toJSON(), storeServer, svc.ID) } - // Monitor JS file changes - jsRows, err := s.db.QueryContext(ctx, `SELECT id, url, hash FROM js_files WHERE target_id = ?`, targetID) + if harvested > 0 { + logFn("info", "monitor", fmt.Sprintf("Tracking %d new stylesheet URL(s) for content changes", harvested)) + } + + // Monitor JS file changes — hash AND the capped content body, so a js_change + // row can render a real line-diff in the dashboard instead of two hashes. + jsRows, err := s.db.QueryContext(ctx, `SELECT id, url, hash, COALESCE(body,'') FROM js_files WHERE target_id = ?`, targetID) if err == nil { - type jsRecord struct{ ID, URL, Hash string } + type jsRecord struct{ ID, URL, Hash, Body string } var jsFiles []jsRecord for jsRows.Next() { var r jsRecord - if err := jsRows.Scan(&r.ID, &r.URL, &r.Hash); err != nil { + if err := jsRows.Scan(&r.ID, &r.URL, &r.Hash, &r.Body); err != nil { jsRows.Close() return err } @@ -243,23 +262,95 @@ func (s *MonitorScanner) Run(ctx context.Context, targetID string, logFn LogFunc if ctx.Err() != nil { break } - newHash, newSize, err := s.fetchHashWithHeaders(ctx, jf.URL, headersFor(jf.URL)) + newHash, newSize, newBody, err := s.fetchFileWithBody(ctx, jf.URL, headersFor(jf.URL)) if err != nil || newHash == "" { continue } if jf.Hash != "" && newHash != jf.Hash { - confirmHash, _, confirmErr := s.fetchHashWithHeaders(ctx, jf.URL, headersFor(jf.URL)) + confirmHash, _, _, confirmErr := s.fetchFileWithBody(ctx, jf.URL, headersFor(jf.URL)) if confirmErr != nil || confirmHash != newHash { logFn("info", "monitor", fmt.Sprintf("Ignoring unstable one-off JavaScript response from %s", jf.URL)) continue } logFn("warn", "monitor", fmt.Sprintf("[JS CHANGE] %s hash changed", jf.URL)) s.recordChange(targetID, jf.URL, "js_change", jf.Hash, newHash, classifyChangeSeverity("js_change")) + // Roll the previous content into old_* for the diff drawer, then + // publish the new baseline in one go. + _, _ = s.db.ExecContext(ctx, `UPDATE js_files SET hash=?, size=?, body=?, old_hash=?, old_body=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, + newHash, newSize, newBody, jf.Hash, jf.Body, jf.ID) changed++ + continue } // Always establish/refresh the baseline. Previously an empty initial hash // was never written, so that JS file could change forever undetected. - _, _ = s.db.ExecContext(ctx, `UPDATE js_files SET hash=?, size=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, newHash, newSize, jf.ID) + // The content capture is written alongside it on the first pass only — + // rewriting unchanged 256KB bodies every pass is pure write churn. + if jf.Body == "" { + _, _ = s.db.ExecContext(ctx, `UPDATE js_files SET hash=?, size=?, body=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, newHash, newSize, newBody, jf.ID) + } else { + _, _ = s.db.ExecContext(ctx, `UPDATE js_files SET hash=?, size=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, newHash, newSize, jf.ID) + } + } + } + + // Monitor stylesheet content changes. css_files tracks every + // harvested from page bodies (same-origin AND + // cross-origin), so a version bump is judged by CONTENT, not by the link + // attribute: changed bytes → one css_change row (with a real line-diff in + // the drawer); a cache-buster rename with unchanged bytes → silence. + cssRows, err := s.db.QueryContext(ctx, `SELECT id, url, hash, COALESCE(body,''), COALESCE(size,0) FROM css_files WHERE target_id = ?`, targetID) + if err == nil { + type cssRecord struct { + ID, URL, Hash, Body string + Size int + } + var cssFiles []cssRecord + for cssRows.Next() { + var r cssRecord + if err := cssRows.Scan(&r.ID, &r.URL, &r.Hash, &r.Body, &r.Size); err != nil { + cssRows.Close() + return err + } + cssFiles = append(cssFiles, r) + } + if err := cssRows.Err(); err != nil { + cssRows.Close() + return err + } + cssRows.Close() + + for _, cf := range cssFiles { + knownCSS[monitorURLKey(cf.URL)] = true + if ctx.Err() != nil { + break + } + newHash, newSize, newBody, err := s.fetchFileWithBody(ctx, cf.URL, headersFor(cf.URL)) + if err != nil || newHash == "" { + continue + } + if cf.Hash != "" && newHash != cf.Hash { + confirmHash, _, _, confirmErr := s.fetchFileWithBody(ctx, cf.URL, headersFor(cf.URL)) + if confirmErr != nil || confirmHash != newHash { + logFn("info", "monitor", fmt.Sprintf("Ignoring unstable one-off stylesheet response from %s", cf.URL)) + continue + } + logFn("warn", "monitor", fmt.Sprintf("[CSS CHANGE] %s hash changed", cf.URL)) + s.recordChange(targetID, cf.URL, "css_change", cf.Hash, newHash, classifyChangeSeverity("css_change")) + _, _ = s.db.ExecContext(ctx, `UPDATE css_files SET hash=?, size=?, body=?, old_hash=?, old_body=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, + newHash, newSize, newBody, cf.Hash, cf.Body, cf.ID) + changed++ + continue + } + if cf.Hash == "" { + // Establish the baseline without ringing the bell (same as JS). + _, _ = s.db.ExecContext(ctx, `UPDATE css_files SET hash=?, size=?, body=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, newHash, newSize, newBody, cf.ID) + } else if cf.Body == "" { + // Legacy row that predates capture: backfill the content snapshot + // so the next real change has a "before" side to diff against. + _, _ = s.db.ExecContext(ctx, `UPDATE css_files SET body=?, size=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, newBody, newSize, cf.ID) + } else { + _, _ = s.db.ExecContext(ctx, `UPDATE css_files SET size=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, newSize, cf.ID) + } } } @@ -348,8 +439,12 @@ func monitorAssetCandidates(value, assetType, lastURL string) []string { } func (s *MonitorScanner) monitorProjectAssets(ctx context.Context, targetID, targetScope string, authHeaders map[string]string, knownHTTP, knownJS map[string]bool, logFn LogFunc) (int, error) { + // Only web assets are monitorable. A network asset (ip/cidr) has no page to + // fetch: building https://10.0.0.0/24 from it would just burn a request and + // record a meaningless "change" on the range string itself. rows, err := s.db.QueryContext(ctx, `SELECT id,value,COALESCE(asset_type,'domain'),COALESCE(metadata,'{}') - FROM assets WHERE target_id=? AND COALESCE(approval_status,'approved')='approved' AND COALESCE(monitor_enabled,1)=1`, targetID) + FROM assets WHERE target_id=? AND COALESCE(approval_status,'approved')='approved' AND COALESCE(monitor_enabled,1)=1 + AND COALESCE(kind,'web')='web'`, targetID) if err != nil { return 0, err } @@ -559,6 +654,12 @@ func notificationTitle(changeType string) string { switch changeType { case "js_change": return "JavaScript changed" + case "css_change": + return "Stylesheet content changed" + case "security:stylesheet_added": + return "External stylesheet added" + case "security:stylesheet_removed": + return "External stylesheet removed" case "http_change", "status_change": return "HTTP response changed" case "page_change": @@ -652,6 +753,115 @@ func (s *MonitorScanner) fetchHashWithHeaders(ctx context.Context, url string, h return h, len(body), nil } +// monitorBodyCaptureCap bounds the per-file content stored beside the hash +// (see the "Body capture for diffing" migration comment). The HASH always +// covers the full download (up to the 5MB reader cap), so a real change deep +// inside a large bundle is still detected; only the stored diff material is +// truncated to the first N bytes. +const monitorBodyCaptureCap = 256 * 1024 + +// Stylesheet `…` harvesting, attribute-order agnostic. +var ( + reLinkTag = regexp.MustCompile(`(?i)]*>`) + reLinkAttrRel = regexp.MustCompile(`(?i)\brel\s*=\s*["']?([^"'\s>]{2,20})`) + reLinkAttrHref = regexp.MustCompile(`(?i)\bhref\s*=\s*["']([^"']+)["']`) +) + +// fetchFileWithBody fetches a static asset (JS/CSS) for content tracking: +// full-content SHA-256, size, and the capped body the diff drawer renders. +// The HTML-document rejection mirrors fetchHashWithHeaders — an SPA fallback +// must never be captured as an asset's content. +func (s *MonitorScanner) fetchFileWithBody(ctx context.Context, rawURL string, headers map[string]string) (hash string, size int, body string, err error) { + req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil) + if err != nil { + return "", 0, "", err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36") + for k, v := range headers { + if strings.EqualFold(k, "Host") || strings.EqualFold(k, "Content-Length") { + continue + } + req.Header.Set(k, v) + } + + resp, err := monitorClient.Do(req) + if err != nil { + return "", 0, "", err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", 0, "", fmt.Errorf("unexpected asset response status %d", resp.StatusCode) + } + + raw, err := io.ReadAll(io.LimitReader(resp.Body, 5*1024*1024)) + if err != nil { + return "", 0, "", err + } + trimmed := strings.ToLower(strings.TrimSpace(string(raw))) + if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/html") && + (strings.HasPrefix(trimmed, " monitorBodyCaptureCap { + captured = captured[:monitorBodyCaptureCap] + } + h := fmt.Sprintf("%x", sha256.Sum256(raw)) + return h, len(raw), string(captured), nil +} + +// harvestStylesheets pulls URLs out of fresh page +// HTML and tracks them in css_files (same-origin AND cross-origin — that is +// what the browser actually loads), so the monitor can diff stylesheet +// CONTENT — the layer behind the Changes tab's css_change rows — rather than +// only the attribute-level "link appeared" check. Relative hrefs resolve +// against the page URL. Returns how many NEW URLs were registered. +func (s *MonitorScanner) harvestStylesheets(ctx context.Context, targetID, pageURL, body string, known map[string]bool) int { + base, err := url.Parse(strings.TrimSpace(pageURL)) + if err != nil || base.Host == "" { + return 0 + } + inserted := 0 + for _, tag := range reLinkTag.FindAllString(body, -1) { + relMatch := reLinkAttrRel.FindStringSubmatch(tag) + if relMatch == nil || !strings.Contains(strings.ToLower(relMatch[1]), "stylesheet") { + continue + } + hrefMatch := reLinkAttrHref.FindStringSubmatch(tag) + if hrefMatch == nil { + continue + } + href := strings.TrimSpace(hrefMatch[1]) + if href == "" || strings.HasPrefix(href, "#") { + continue + } + ref, err := url.Parse(href) + if err != nil { + continue + } + if ref.Scheme != "" && ref.Scheme != "http" && ref.Scheme != "https" { + continue // data:/javascript:/mailto:/tel: and friends + } + resolved := base.ResolveReference(ref) + if resolved.Scheme != "http" && resolved.Scheme != "https" { + continue + } + // Register the query-stripped URL: a `?ver=` churn maps to ONE tracked + // row (content re-fetched from the stable path each pass) instead of a + // new css_files row per cache-buster. + trackedURL := stripResourceQuery(resolved.String()) + key := monitorURLKey(trackedURL) + if key == "" || known[key] { + continue + } + known[key] = true + inserted++ + _, _ = s.db.ExecContext(ctx, `INSERT INTO css_files (id, target_id, url) VALUES (?, ?, ?) + ON CONFLICT(target_id, url) DO NOTHING`, uuid.New().String(), targetID, trackedURL) + } + return inserted +} + func (s *MonitorScanner) describeChange(oldStatus, newStatus int, oldTitle, newTitle, oldHash, newHash string) string { var parts []string if oldStatus != newStatus { diff --git a/internal/scanner/monitor_assets_test.go b/internal/scanner/monitor_assets_test.go new file mode 100644 index 0000000..f75defe --- /dev/null +++ b/internal/scanner/monitor_assets_test.go @@ -0,0 +1,93 @@ +package scanner + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/google/uuid" +) + +// Network assets (ip/cidr) are not web pages. The page monitor must never build +// a pseudo-URL from one and fetch it — the counting server proves no request is +// made. A regression here means every monitored network range gets a bogus +// https:// GET on each watch pass. +func TestMonitorProjectAssetsSkipsNetworkAssets(t *testing.T) { + var mu sync.Mutex + hits := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + hits++ + mu.Unlock() + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(`okok`)) + })) + defer srv.Close() + + db := newMonitorTestDB(t) + targetID := uuid.NewString() + if _, err := db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, "example.test"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO assets(id,target_id,value,kind,asset_type,approval_status,monitor_enabled) + VALUES(?,?,?,?,?,?,1)`, + uuid.NewString(), targetID, strings.TrimPrefix(srv.URL, "http://"), "network", "ip", "approved"); err != nil { + t.Fatal(err) + } + + s := &MonitorScanner{db: db} + logFn := func(string, string, string) {} + changed, err := s.monitorProjectAssets(context.Background(), targetID, "", nil, + map[string]bool{}, map[string]bool{}, logFn) + if err != nil { + t.Fatal(err) + } + mu.Lock() + got := hits + mu.Unlock() + if changed != 0 || got != 0 { + t.Fatalf("network asset was monitored: changed=%d requests=%d", changed, got) + } +} + +// The counterpart: a web asset IS monitored (so the kind filter above cannot +// silently starve the whole project-asset monitor path). +func TestMonitorProjectAssetsStillMonitorsWebAssets(t *testing.T) { + var mu sync.Mutex + hits := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + hits++ + mu.Unlock() + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(`okok`)) + })) + defer srv.Close() + + db := newMonitorTestDB(t) + targetID := uuid.NewString() + if _, err := db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, "example.test"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO assets(id,target_id,value,kind,asset_type,approval_status,monitor_enabled) + VALUES(?,?,?,?,?,?,1)`, + uuid.NewString(), targetID, srv.URL, "web", "url", "approved"); err != nil { + t.Fatal(err) + } + + s := &MonitorScanner{db: db} + logFn := func(string, string, string) {} + if _, err := s.monitorProjectAssets(context.Background(), targetID, "", nil, + map[string]bool{}, map[string]bool{}, logFn); err != nil { + t.Fatal(err) + } + mu.Lock() + got := hits + mu.Unlock() + if got == 0 { + t.Fatal("web asset was never fetched — the kind filter is too broad") + } +} diff --git a/internal/scanner/monitor_css_test.go b/internal/scanner/monitor_css_test.go new file mode 100644 index 0000000..963d329 --- /dev/null +++ b/internal/scanner/monitor_css_test.go @@ -0,0 +1,331 @@ +package scanner + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/google/uuid" +) + +// Stylesheet change tracking: the monitor harvests +// hrefs from every fetched page into css_files, then diffs CONTENT. A version +// bump with unchanged bytes is silence; a real change yields ONE css_change +// row whose old_body rolls the previous capture — the material the Changes +// tab's diff drawer renders. +func TestStylesheetBaselineThenContentChange(t *testing.T) { + var mu sync.Mutex + state := struct { + css string + pageVer string + }{css: "body{color:blue}", pageVer: "0"} + renderPage := func(ver string) string { + return `hi` + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + pv, cs := state.pageVer, state.css + mu.Unlock() + switch { + case strings.HasSuffix(r.URL.Path, "/style.css"): + w.Header().Set("Content-Type", "text/css") + _, _ = w.Write([]byte(cs)) + default: + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(renderPage(pv))) + } + })) + defer srv.Close() + + db := newMonitorTestDB(t) + targetID := uuid.NewString() + _, _ = db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, srv.URL) + _, _ = db.Exec(`INSERT INTO http_services(id,target_id,url,status_code,title,security_snapshot,server) VALUES(?,?,?,?,?,'','')`, + uuid.NewString(), targetID, srv.URL+"/", 200, "") + + s := &MonitorScanner{db: db} + logFn := func(string, string, string) {} + + // Pass 1 establishes the baseline: URL harvested, content captured, zero rows. + mu.Lock() + state.pageVer = "1" + mu.Unlock() + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + var baselineHash, baselineBody string + if err := db.QueryRow(`SELECT hash, COALESCE(body,'') FROM css_files WHERE target_id=?`, targetID).Scan(&baselineHash, &baselineBody); err != nil { + t.Fatalf("stylesheet was not harvested into css_files: %v", err) + } + if baselineHash == "" || baselineBody == "" { + t.Fatal("first pass did not capture stylesheet content baseline") + } + var rows int + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=?`, targetID).Scan(&rows) + if rows != 0 { + t.Fatalf("baseline pass raised %d change rows, want 0", rows) + } + + // Pass 2 with a REAL content change → exactly one css_change row with the + // previous capture rolled into old_*. + mu.Lock() + state.pageVer = "2" + state.css = "body{color:red}" + mu.Unlock() + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + var oldHash, oldBody, curBody string + if err := db.QueryRow(`SELECT old_hash, old_body, body FROM css_files WHERE target_id=?`, targetID).Scan(&oldHash, &oldBody, &curBody); err != nil { + t.Fatal(err) + } + if oldHash != baselineHash || oldBody != "body{color:blue}" || !strings.Contains(curBody, "color:red") { + t.Fatalf("diff material not rolled: old_hash=%q old_body=%q body=%q", oldHash, oldBody, curBody) + } + var cssTotal int + if err := db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type='css_change'`, targetID).Scan(&cssTotal); err != nil { + t.Fatal(err) + } + if cssTotal != 1 { + t.Fatalf("content-change pass produced %d css_change rows, want 1 (a relative link never raises security: rows)", cssTotal) + } + + // Pass 3: cache-buster bump with UNCHANGED bytes → silence (no extra rows). + mu.Lock() + state.pageVer = "3" + mu.Unlock() + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type='css_change'`, targetID).Scan(&cssTotal) + if cssTotal != 1 { + t.Fatalf("no-op version bump produced extra css rows: %d css_change rows, want still 1", cssTotal) + } + var attrRows int + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type LIKE 'security:stylesheet%'`, targetID).Scan(&attrRows) + if attrRows != 0 { + t.Fatalf("relative stylesheets raised %d security: rows, want 0", attrRows) + } +} + +// A cache-busted bump on an EXTERNAL stylesheet must not spam the Changes tab +// with add/remove pairs when the bytes are identical — snapshot keys are +// path-based (stripResourceQuery), and unchanged content yields no css_change. +func TestExternalStylesheetVerBumpIsSilent(t *testing.T) { + pageVer := "1" + renderPage := func(ver string) string { + return `x` + } + + pageSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(renderPage(pageVer))) + })) + defer pageSrv.Close() + + db := newMonitorTestDB(t) + targetID := uuid.NewString() + _, _ = db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, pageSrv.URL) + _, _ = db.Exec(`INSERT INTO http_services(id,target_id,url,status_code,title,security_snapshot) VALUES(?,?,?,?,?,'')`, + uuid.NewString(), targetID, pageSrv.URL+"/", 200, "x") + + s := &MonitorScanner{db: db} + logFn := func(string, string, string) {} + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + pageVer = "2" + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + + var attrRows, cssRows int + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type LIKE 'security:stylesheet%'`, targetID).Scan(&attrRows) + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type='css_change'`, targetID).Scan(&cssRows) + if attrRows != 0 || cssRows != 0 { + t.Fatalf("ver-bump with unchanged bytes is not silent: %d security: rows, %d css_change rows", attrRows, cssRows) + } +} + +// Adding a genuinely NEW stylesheet path still alerts, at info severity. +func TestNewExternalStylesheetPathStillAlerts(t *testing.T) { + extraEnabled := false + renderPage := func(withExtra bool) string { + base := `x` + if withExtra { + return `` + + `x` + } + return base + } + + pageSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(renderPage(extraEnabled))) + })) + defer pageSrv.Close() + + db := newMonitorTestDB(t) + targetID := uuid.NewString() + _, _ = db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, pageSrv.URL) + _, _ = db.Exec(`INSERT INTO http_services(id,target_id,url,status_code,title,security_snapshot) VALUES(?,?,?,?,?,'')`, + uuid.NewString(), targetID, pageSrv.URL+"/", 200, "x") + + s := &MonitorScanner{db: db} + logFn := func(string, string, string) {} + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + extraEnabled = true + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + + var addedRows int + if err := db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type='security:stylesheet_added'`, targetID).Scan(&addedRows); err != nil { + t.Fatal(err) + } + if addedRows != 1 { + t.Fatalf("new stylesheet path produced %d security:stylesheet_added rows, want exactly 1", addedRows) + } + var sev string + _ = db.QueryRow(`SELECT severity FROM monitoring_changes WHERE target_id=? AND change_type='security:stylesheet_added'`, targetID).Scan(&sev) + if sev != "info" { + t.Fatalf("new stylesheet severity = %q, want info", sev) + } + var removedRows int + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type='security:stylesheet_removed'`, targetID).Scan(&removedRows) + if removedRows != 0 { + t.Fatalf("adding a path produced %d removed rows, want 0", removedRows) + } +} + +// js_change rows carry the same diff material: the previous capture rolls into +// old_body at detection time. +func TestJSChangeCapturesBodies(t *testing.T) { + var mu sync.Mutex + js := "console.log('v1')" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + body := js + mu.Unlock() + w.Header().Set("Content-Type", "application/javascript") + _, _ = w.Write([]byte(body)) + })) + defer srv.Close() + + db := newMonitorTestDB(t) + targetID := uuid.NewString() + _, _ = db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, srv.URL) + _, _ = db.Exec(`INSERT INTO js_files(id,target_id,url,hash) VALUES(?,?,?,'')`, uuid.NewString(), targetID, srv.URL+"/app.js") + + s := &MonitorScanner{db: db} + logFn := func(string, string, string) {} + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + + mu.Lock() + js = "console.log('v2')" + mu.Unlock() + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + + var oldBody, curBody string + if err := db.QueryRow(`SELECT old_body, body FROM js_files WHERE target_id=?`, targetID).Scan(&oldBody, &curBody); err != nil { + t.Fatal(err) + } + if oldBody != "console.log('v1')" || curBody != "console.log('v2')" { + t.Fatalf("js bodies not rolled: old_body=%q body=%q", oldBody, curBody) + } + var changes int + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type='js_change'`, targetID).Scan(&changes) + if changes != 1 { + t.Fatalf("changed javascript produced %d js_change rows, want 1", changes) + } +} + +// ── pure-unit tests ────────────────────────────────────────────────────────── + +func TestStripResourceQuery(t *testing.T) { + cases := map[string]string{ + "https://cdn.t/app.css?ver=8&x=1": "https://cdn.t/app.css", + "https://cdn.t/app.css": "https://cdn.t/app.css", + "/app.css?ver=1": "/app.css?ver=1", // no host: leave untouched + "": "", + } + for in, want := range cases { + if got := stripResourceQuery(in); got != want { + t.Errorf("stripResourceQuery(%q) = %q, want %q", in, got, want) + } + } +} + +func TestSecuritySnapshotSuppressesVerBumpButNotNewPath(t *testing.T) { + page := func(path string) string { + return `` + } + base := extractSecuritySnapshot(page("/x.css?ver=1"), "app.test") + bumped := extractSecuritySnapshot(page("/x.css?ver=2"), "app.test") + if base.toJSON() != bumped.toJSON() { + t.Fatal("ver bump on the same stylesheet path must not alter the security snapshot") + } + newPath := extractSecuritySnapshot(page("/y.css"), "app.test") + if newPath.toJSON() == bumped.toJSON() { + t.Fatal("a genuinely new stylesheet path must alter the security snapshot") + } + withScript := extractSecuritySnapshot(page("/x.css?ver=2\")>"), "app.test") + if withScript.toJSON() == bumped.toJSON() { + t.Fatal("a new external script must alter the security snapshot") + } +} + +func TestSecurityChangeSeverityStylesheetIsInfo(t *testing.T) { + cases := []struct { + change securityChange + want string + }{ + {securityChange{action: "added", attr: securityAttr{Kind: "script", Value: "https://x/e.js"}}, "high"}, + {securityChange{action: "added", attr: securityAttr{Kind: "stylesheet", Value: "https://x/e.css"}}, "info"}, + {securityChange{action: "added", attr: securityAttr{Kind: "hidden", Value: "csrf_seen"}}, "medium"}, + {securityChange{action: "removed", attr: securityAttr{Kind: "form", Value: "/login"}}, "medium"}, + } + for _, tc := range cases { + if got := securityChangeSeverity(tc.change); got != tc.want { + t.Errorf("severity(%s %s) = %q, want %q", tc.change.action, tc.change.attr.Kind, got, tc.want) + } + } +} + +// Legacy snapshots (recorded before extraction started stripping ?ver=) must +// not unleash a one-time add+remove pair per stylesheet on the first diff +// after an upgrade: the diff key normalizes stylesheet values on both sides. +// Non-stylesheet kinds keep exact-match keys (script bump → added + removed). +func TestSecuritySnapshotLegacyQueryKeysDiffSilently(t *testing.T) { + old := securitySnapshot{Attrs: []securityAttr{ + {Kind: "stylesheet", Value: "https://cdn.example.test/s/a.css?ver=1"}, + {Kind: "script", Value: "https://x.example.test/y.js?h=1"}, + }} + cur := securitySnapshot{Attrs: []securityAttr{ + {Kind: "stylesheet", Value: "https://cdn.example.test/s/a.css?ver=2"}, + {Kind: "script", Value: "https://x.example.test/y.js?h=2"}, + }} + got := diffSecuritySnapshots(old, cur) + sheetChanges := 0 + for _, c := range got { + if c.attr.Kind == "stylesheet" { + sheetChanges++ + } + } + if sheetChanges != 0 { + t.Fatalf("stylesheet ver-bump diffed as %d change(s), want 0", sheetChanges) + } + if len(got) != 2 { + t.Fatalf("script hash bump produced %d change(s), want exactly 2 (added+removed)", len(got)) + } +} diff --git a/internal/scanner/monitor_engine.go b/internal/scanner/monitor_engine.go index 18bd56a..72e5705 100644 --- a/internal/scanner/monitor_engine.go +++ b/internal/scanner/monitor_engine.go @@ -185,7 +185,11 @@ func extractSecuritySnapshot(body, pageHost string) securitySnapshot { } for _, m := range reLinkHref.FindAllStringSubmatch(body, -1) { if ext(m[1]) { - add("stylesheet", m[1]) + // Cache-bust churn (`?ver=`, hashed filenames) flickers the href on + // every deploy. Key the diff by path so a version bump on the SAME + // resource stops emitting add/remove pairs; the CONTENT truth lives + // in css_change rows from css_files content tracking. + add("stylesheet", stripResourceQuery(m[1])) } } for _, m := range reAHref.FindAllStringSubmatch(body, -1) { @@ -228,6 +232,20 @@ func hostOf(rawURL string) string { return strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) } +// stripResourceQuery drops the query string and fragment from an asset URL so +// cache-bust churn (`/app.css?ver=8`) keys as the SAME resource across +// security-snapshot diffs. The versioned URL is still captured and diffed by +// content via css_files, so no real change is hidden. +func stripResourceQuery(raw string) string { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Hostname() == "" { + return raw + } + u.RawQuery = "" + u.Fragment = "" + return u.String() +} + func (s securitySnapshot) toJSON() string { b, _ := json.Marshal(s) return string(b) @@ -246,15 +264,27 @@ type securityChange struct { attr securityAttr } +// securityKey is the diff-set key for one snapshot attribute. Stylesheets key +// on the query-stripped path so a LEGACY stored snapshot (recorded before +// extraction started stripping prompts) and a fresh extraction of the same +// resource share one key — an upgrade to this behavior must not unleash one +// add+remove pair per stylesheet on the first pass after it ships. +func securityKey(a securityAttr) string { + if a.Kind == "stylesheet" { + return a.Kind + "|" + stripResourceQuery(a.Value) + } + return a.Kind + "|" + a.Value +} + // diffSecuritySnapshots returns what security attributes were added/removed. func diffSecuritySnapshots(old, cur securitySnapshot) []securityChange { oldSet := map[string]securityAttr{} for _, a := range old.Attrs { - oldSet[a.Kind+"|"+a.Value] = a + oldSet[securityKey(a)] = a } curSet := map[string]securityAttr{} for _, a := range cur.Attrs { - curSet[a.Kind+"|"+a.Value] = a + curSet[securityKey(a)] = a } var out []securityChange for k, a := range curSet { @@ -287,8 +317,13 @@ func securityChangeSeverity(c securityChange) string { switch c.attr.Kind { case "script", "iframe", "form": return "high" - case "stylesheet", "hidden": + case "hidden": return "medium" + case "stylesheet": + // Path-keyed now (stripResourceQuery), so stylesheet rows are the + // rare genuinely-new/deleted-path case. Content-level truth lives + // in css_change rows. + return "info" default: return "low" } diff --git a/internal/scanner/recon_stack.go b/internal/scanner/recon_stack.go index b37e066..7ec8f01 100644 --- a/internal/scanner/recon_stack.go +++ b/internal/scanner/recon_stack.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "fmt" + "net" "os" "strings" "sync" @@ -54,8 +55,8 @@ func (s *SubdomainScanner) toolResolverList() []string { // deepDNSEnum runs the heavy DNS discovery chain (alterx permutations + // puredns/massdns brute) on top of the built-in active enum. It only adds // names that actually resolve and aren't wildcard answers. -func (s *SubdomainScanner) deepDNSEnum(ctx context.Context, targetID, domain string, found map[string]bool, mu *sync.Mutex, wildcardIPs map[string]bool, logFn LogFunc) { - if len(wildcardIPs) > 0 { +func (s *SubdomainScanner) deepDNSEnum(ctx context.Context, targetID, domain string, found map[string]bool, mu *sync.Mutex, wildcardProfile *wildcardProfile, logFn LogFunc) { + if wildcardProfile.apexWildcard() { return // wildcard domains: brute is meaningless, skip } @@ -155,7 +156,7 @@ func (s *SubdomainScanner) deepDNSEnum(ctx context.Context, targetID, domain str return } for _, n := range resolved { - ip := firstNonWildcardIP(resolveHostIPs(ctx, n, s.resolver), wildcardIPs) + ip := wildcardProfile.firstNonWildcardIP(n, resolveHostIPs(ctx, n, s.resolver)) if ip == "" { continue // wildcard-only or unresolved on recheck — not an asset } @@ -168,7 +169,7 @@ func (s *SubdomainScanner) deepDNSEnum(ctx context.Context, targetID, domain str return } - s.resolveCandidates(ctx, targetID, names, found, mu, wildcardIPs, logFn) + s.resolveCandidates(ctx, targetID, names, found, mu, wildcardProfile, logFn) } // dnsxResolve performs only DNS resolution; every returned name is subsequently @@ -216,6 +217,92 @@ func (s *SubdomainScanner) dnsxResolve(ctx context.Context, targetID string, nam return out } +// dnsxResolveIPs batch-resolves the A/AAAA record sets of a large candidate +// list in a single dnsx run and returns name -> every answer IP as canonical +// net.ParseIP strings, so comparisons against Go-resolver wildcard-probe IPs +// stay string-identical. A nil map means dnsx was unavailable or failed while +// the context was still live — the caller must run the legacy per-name path +// and must never treat nil as "everything unresolved". +func (s *SubdomainScanner) dnsxResolveIPs(ctx context.Context, targetID string, names []string) map[string][]string { + if s.exec == nil || len(names) == 0 || !s.exec.IsToolAvailable("dnsx") { + return nil + } + // Concurrency floor, mirroring dnsxResolve: a 20k-name admission pass at + // the generic low -t crawls; dnsx handles a high thread count comfortably. + workers := s.cfg.Workers.SubdomainEnumeration + if workers < 150 { + workers = 150 + } + if workers > 400 { + workers = 400 + } + args := []string{"-silent", "-a", "-aaaa", "-resp", "-retry", "2", "-t", fmt.Sprint(workers)} + // Same resolver file contract as every other dnsx pass: the operator list + // (or the public default) — the SAME list the in-process s.resolver + // round-robins, so answers and wildcard probes share one resolver source. + if resolverFile, err := writeTempLines("recon-dnsx-aresp-resolvers-", s.toolResolverList()); err == nil { + defer os.Remove(resolverFile) + args = append(args, "-r", resolverFile) + } + out := map[string][]string{} + err := s.exec.RunWithInputCallback(ctx, strings.NewReader(strings.Join(names, "\n")), targetID, + func(line string) { + name, toks := parseDNSXResponseLine(line) + if name == "" { + return + } + ips := make([]string, 0, len(toks)) + for _, tok := range toks { + if ip := net.ParseIP(tok); ip != nil { + ips = append(ips, ip.String()) + } + } + // dnsx can emit one -resp line per record type for the same name; + // merge so an A-only or AAAA-only second line is never lost. + if len(ips) > 0 { + out[name] = append(out[name], ips...) + } + }, "dnsx", args...) + if err != nil && ctx.Err() == nil { + return nil + } + return out +} + +// parseDNSXResponseLine splits one dnsx `-resp` output line into the hostname +// and the bracket-group tokens — separators and trailing dots stripped, lower +// cased: +// +// sub.example.com [93.184.216.34, 2606:2800:220:1:248:1893:25c8:1946] +// www.example.com [example.edge.cdn.net] +// +// Tokens come back as raw text; IP-vs-CNAME validation is the caller's +// concern. Lines without a usable bracket group parse as ("", nil). +func parseDNSXResponseLine(line string) (string, []string) { + line = strings.TrimSpace(line) + openBr := strings.Index(line, "[") + closeBr := strings.LastIndex(line, "]") + if openBr <= 0 || closeBr <= openBr { + return "", nil + } + fields := strings.Fields(line[:openBr]) + if len(fields) == 0 { + return "", nil + } + name := strings.TrimSuffix(strings.ToLower(fields[0]), ".") + if name == "" { + return "", nil + } + var toks []string + for _, tok := range strings.FieldsFunc(line[openBr+1:closeBr], func(r rune) bool { return r == ',' || r == ' ' || r == '\t' }) { + tok = strings.TrimSuffix(strings.ToLower(strings.Trim(tok, "[] \t")), ".") + if tok != "" { + toks = append(toks, tok) + } + } + return name, toks +} + // purednsBruteforce runs `puredns bruteforce ` with a // generated resolvers file, returning the resolved hostnames. func (s *SubdomainScanner) purednsBruteforce(ctx context.Context, targetID, domain string, logFn LogFunc) ([]string, error) { @@ -270,7 +357,7 @@ func (s *SubdomainScanner) purednsResolve(ctx context.Context, targetID string, } // resolveCandidates is the built-in fallback resolver pool for a name list. -func (s *SubdomainScanner) resolveCandidates(ctx context.Context, targetID string, names []string, found map[string]bool, mu *sync.Mutex, wildcardIPs map[string]bool, logFn LogFunc) { +func (s *SubdomainScanner) resolveCandidates(ctx context.Context, targetID string, names []string, found map[string]bool, mu *sync.Mutex, wildcardProfile *wildcardProfile, logFn LogFunc) { workers := s.cfg.Workers.SubdomainEnumeration if workers <= 0 { workers = 20 @@ -287,7 +374,7 @@ func (s *SubdomainScanner) resolveCandidates(ctx context.Context, targetID strin if ctx.Err() != nil { return } - ip := firstNonWildcardIP(resolveHostIPs(ctx, name, s.resolver), wildcardIPs) + ip := wildcardProfile.firstNonWildcardIP(name, resolveHostIPs(ctx, name, s.resolver)) if ip == "" { continue } diff --git a/internal/scanner/shodan.go b/internal/scanner/shodan.go index cdedc2c..e3e99dd 100644 --- a/internal/scanner/shodan.go +++ b/internal/scanner/shodan.go @@ -104,7 +104,7 @@ func (s *ShodanScanner) Run(ctx context.Context, targetID string, logFn LogFunc) // gatherIPs returns a map of IP → representative hostname for resolved subdomains. func (s *ShodanScanner) gatherIPs(ctx context.Context, targetID string) map[string]string { out := map[string]string{} - rows, err := s.db.QueryContext(ctx, `SELECT subdomain, ip FROM subdomains WHERE target_id=? AND ip != ''`, targetID) + rows, err := s.db.QueryContext(ctx, `SELECT subdomain, ip FROM subdomains WHERE target_id=? AND ip != '' AND COALESCE(catchall,0)=0`, targetID) if err != nil { return out } diff --git a/internal/scanner/subdomain.go b/internal/scanner/subdomain.go index e738b5d..4046f65 100644 --- a/internal/scanner/subdomain.go +++ b/internal/scanner/subdomain.go @@ -91,10 +91,9 @@ func (s *SubdomainScanner) Run(ctx context.Context, targetID, domain string, log } // Detect wildcard DNS up front so we can flag fake passive subdomains. - wildcardIPs, wildcardCNAMEs := detectWildcard(domain, s.resolver) - if len(wildcardIPs) > 0 || len(wildcardCNAMEs) > 0 { - logFn("warn", "subdomain_enum", fmt.Sprintf("Wildcard DNS detected (%d IPs) — passive results will be verified by resolution", len(wildcardIPs))) - } + // Probes retry across the configured public resolvers and distinguish a + // definitive NXDOMAIN from SERVFAIL/flake — see wildcard.go. + wildcardProfile := detectWildcard(domain, s.resolver, s.wildcardServers(), logFn) // Discovery and admission are deliberately separate. Passive tools are allowed // to be noisy, but their strings remain in-memory candidates until the batched @@ -326,11 +325,11 @@ func (s *SubdomainScanner) Run(ctx context.Context, targetID, domain string, log // operator disables it in the scan menu, only passive discovery + resolution + // ASN/vhost run, which is dramatically faster. if subdomainBruteEnabled(ctx) { - s.activeEnum(ctx, targetID, domain, found, &mu, wildcardIPs, logFn) + s.activeEnum(ctx, targetID, domain, found, &mu, wildcardProfile, logFn) // Deeper DNS discovery (optional, graceful): alterx permutations + // puredns/massdns brute at scale. No-op when those tools aren't installed. - s.deepDNSEnum(ctx, targetID, domain, found, &mu, wildcardIPs, logFn) + s.deepDNSEnum(ctx, targetID, domain, found, &mu, wildcardProfile, logFn) } else { logFn("info", "subdomain_enum", "Permutation brute-force disabled for this scan — passive discovery + resolution only.") } @@ -351,16 +350,29 @@ func (s *SubdomainScanner) Run(ctx context.Context, targetID, domain string, log } mu.Unlock() + // Branch wildcard detection (F3): a junk fleet can hide under a wildcard + // deeper than the apex — *.git.example.com was the live example, where + // hundreds of "asdf-garbage.git." names were admitted wholesale. Every + // discovery source has merged by now, so clustered candidate suffixes are + // known: probe the clusters before admission runs. + candidateSnapshot := make([]string, 0, len(found)) + mu.Lock() + for name := range found { + candidateSnapshot = append(candidateSnapshot, name) + } + mu.Unlock() + wildcardProfile.detectBranches(ctx, domain, candidateSnapshot, s.resolver, s.wildcardServers(), branchMaxProbes, logFn) + // Parallel admission pass. It returns candidates that had neither a genuine // A/AAAA answer nor an explicit CNAME; wildcard-only answers remain eligible // for vhost proof but are not assets yet. - rejected := s.resolveAll(ctx, targetID, found, &mu, wildcardIPs, wildcardCNAMEs, logFn) + rejected := s.resolveAll(ctx, targetID, found, &mu, wildcardProfile, logFn) // Virtual-host scan: probe candidate hostnames (that do NOT resolve in DNS) // against the IPs we already know, via the Host header, and record any that // the server actually serves — internal/forgotten vhosts DNS enumeration // can't see. Stored with source='vhost' so the UI can flag them. - s.vhostScan(ctx, targetID, domain, found, &mu, wildcardIPs, logFn) + s.vhostScan(ctx, targetID, domain, found, &mu, wildcardProfile, logFn) // Vhost verification had the final chance to promote wildcard/unresolved names. // Remove only old eager-DNS rows which still lack evidence; explicit seeds, @@ -369,10 +381,12 @@ func (s *SubdomainScanner) Run(ctx context.Context, targetID, domain string, log // Recursive enumeration: run passive discovery + resolution AGAINST each newly // discovered resolving subdomain as its own sub-apex, descending until the - // configured depth or until a level finds nothing new. OPT-IN per scan: it - // multiplies DNS volume and is the main driver of public-resolver throttling, - // so an ordinary scan leaves it off and stays fast. Cancelling the scan stops - // it (recursiveEnum checks ctx between every sub-apex and level). + // configured depth or until a level finds nothing new. Per-scan opt-in via + // the scan dialog's recursion toggle, with the System → Scanning switch as + // the deployment default. Recursion multiplies DNS volume and is the main + // driver of public-resolver throttling, so the built-in default stays off. + // Cancelling the scan stops it (recursiveEnum checks ctx between every + // sub-apex and level). if subdomainRecursionEnabled(ctx) { s.recursiveEnum(ctx, targetID, domain, logFn) } @@ -536,18 +550,23 @@ func (s *SubdomainScanner) vhostProbeConcurrency() int { return c } -func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain string, found map[string]bool, mu *sync.Mutex, wildcardIPs map[string]bool, logFn LogFunc) { +func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain string, found map[string]bool, mu *sync.Mutex, wildcardZones *wildcardProfile, logFn LogFunc) { if ctx.Err() != nil { return } + // A nil profile (test paths, "no wildcard evidence") behaves like an empty + // one; field accesses below must not trip on it. + if wildcardZones == nil { + wildcardZones = &wildcardProfile{} + } // Distinct known IPs (cap to keep the pass bounded; raised from 40 — a huge // multi-asset scan can legitimately have far more than 40 distinct hosting // IPs, and silently dropping the rest means whole IP ranges never get // Host-header probed at all). rows, err := s.db.QueryContext(ctx, `SELECT DISTINCT ip FROM subdomains - WHERE target_id=? AND ip!='' AND (subdomain=? OR subdomain LIKE ?) - LIMIT 100`, targetID, domain, "%."+domain) + WHERE target_id=? AND ip!='' AND (subdomain=? OR subdomain LIKE ? ESCAPE '\') + LIMIT 100`, targetID, domain, "%."+escapeLike(domain)) if err != nil { return } @@ -565,12 +584,21 @@ func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain strin for _, ip := range ips { ipSeen[ip] = true } - for ip := range wildcardIPs { + for ip := range wildcardZones.apex.ips { if !ipSeen[ip] { ips = append(ips, ip) ipSeen[ip] = true } } + // Branch wildcard answer IPs are equally valuable vhost-probe targets. + for _, zone := range wildcardZones.branches { + for ip := range zone.ips { + if !ipSeen[ip] { + ips = append(ips, ip) + ipSeen[ip] = true + } + } + } if len(ips) == 0 { return } @@ -596,7 +624,7 @@ func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain strin for _, p := range devToolWords { add(p + "." + domain) } - if len(wildcardIPs) > 0 { + if wildcardZones.hasAnyWildcard() { // DNS brute-force is meaningless on a wildcard zone, but the same adaptive // names are valuable Host-header candidates. Feed a bounded slice into the // differential vhost verifier instead of accepting their wildcard DNS answer. @@ -633,7 +661,7 @@ func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain strin // Probe names DNS cannot distinguish: NXDOMAIN/no-address candidates and names // whose only address is the wildcard answer. Ordinary independently-resolving // hosts are already admitted and do not need expensive Host-header probing. - probeHosts := classifyVhostCandidates(ctx, candidates, wildcardIPs, s.resolver) + probeHosts := classifyVhostCandidates(ctx, candidates, wildcardZones, s.resolver) logFn("info", "vhost", fmt.Sprintf("Virtual-host scan: %d IP(s) × %d candidate host(s)...", len(ips), len(probeHosts))) @@ -759,7 +787,7 @@ func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain strin logFn("info", "vhost", fmt.Sprintf("Virtual-host scan done. %d vhost(s) discovered.", discovered)) } -func classifyVhostCandidates(ctx context.Context, candidates []string, wildcardIPs map[string]bool, r *net.Resolver) []string { +func classifyVhostCandidates(ctx context.Context, candidates []string, wildcardProfile *wildcardProfile, r *net.Resolver) []string { if len(candidates) == 0 { return nil } @@ -776,7 +804,7 @@ func classifyVhostCandidates(ctx context.Context, candidates []string, wildcardI defer wg.Done() for host := range jobs { ips := resolveHostIPs(ctx, host, r) - if len(ips) == 0 || overlapsWildcardIPs(ips, wildcardIPs) { + if len(ips) == 0 || wildcardProfile.overlapsWildcardIPs(host, ips) { hits <- host } } @@ -972,9 +1000,12 @@ var bruteWords = []string{ var permPrefixes = []string{"dev", "staging", "test", "qa", "uat", "old", "new", "internal", "api", "admin"} var permSuffixes = []string{"dev", "staging", "test", "v1", "v2", "v3", "old", "new", "internal", "api"} -func (s *SubdomainScanner) activeEnum(ctx context.Context, targetID, domain string, found map[string]bool, mu *sync.Mutex, wildcardIPs map[string]bool, logFn LogFunc) { +func (s *SubdomainScanner) activeEnum(ctx context.Context, targetID, domain string, found map[string]bool, mu *sync.Mutex, wildcardProfile *wildcardProfile, logFn LogFunc) { // Skip active brute on wildcard domains (every name would "resolve"). - if len(wildcardIPs) > 0 { + // Only an APEX wildcard justifies the skip: a branch wildcard leaves the + // rest of the zone worth enumerating, and branch hits are filtered + // per-name by the admission gate. + if wildcardProfile.apexWildcard() { logFn("info", "subdomain_enum", "Wildcard domain — skipping active brute-force to avoid fake results") return } @@ -1042,7 +1073,7 @@ func (s *SubdomainScanner) activeEnum(ctx context.Context, targetID, domain stri if ctx.Err() != nil { return } - ip := firstNonWildcardIP(resolveHostIPs(ctx, name, s.resolver), wildcardIPs) + ip := wildcardProfile.firstNonWildcardIP(name, resolveHostIPs(ctx, name, s.resolver)) if ip == "" { continue } @@ -1070,7 +1101,8 @@ func (s *SubdomainScanner) activeEnum(ctx context.Context, targetID, domain stri func (s *SubdomainScanner) admittedSubdomainNames(ctx context.Context, targetID, domain string) []string { rows, err := s.db.QueryContext(ctx, `SELECT subdomain FROM subdomains - WHERE target_id=? AND (COALESCE(ip,'')!='' OR COALESCE(source,'dns') IN ('seed','vhost')) + WHERE target_id=? AND COALESCE(catchall,0)=0 + AND (COALESCE(ip,'')!='' OR COALESCE(source,'dns') IN ('seed','vhost')) ORDER BY subdomain`, targetID) if err != nil { return nil @@ -1089,12 +1121,7 @@ func (s *SubdomainScanner) admittedSubdomainNames(ctx context.Context, targetID, // resolveAll resolves every discovered subdomain concurrently and updates its // IP. Names that resolve to a wildcard IP are kept but logged so HTTP probing // can later dedupe them by content. -func (s *SubdomainScanner) resolveAll(ctx context.Context, targetID string, found map[string]bool, mu *sync.Mutex, wildcardIPs, wildcardCNAMEs map[string]bool, logFn LogFunc) map[string]bool { - workers := s.cfg.Workers.SubdomainEnumeration - if workers <= 0 { - workers = 20 - } - +func (s *SubdomainScanner) resolveAll(ctx context.Context, targetID string, found map[string]bool, mu *sync.Mutex, wildcardProfile *wildcardProfile, logFn LogFunc) map[string]bool { mu.Lock() names := make([]string, 0, len(found)) for n := range found { @@ -1109,8 +1136,6 @@ func (s *SubdomainScanner) resolveAll(ctx context.Context, targetID string, foun // are announced live. preexisting := s.existingSubdomainSet(ctx, targetID) - jobs := make(chan string, len(names)) - var wg sync.WaitGroup var resolved, cnameOnly, wildcardHits, unresolved, transient int var cmu sync.Mutex rejected := make(map[string]bool) @@ -1121,77 +1146,153 @@ func (s *SubdomainScanner) resolveAll(ctx context.Context, targetID string, foun // streams discoveries; only the DB write is batched. type resolvedRow struct{ name, ip, source string } var rows []resolvedRow - cnameCandidates, cnamePrefilter := s.dnsxCNAMECandidates(ctx, targetID, names) + cnameCandidates, cnameTargets, cnamePrefilter := s.dnsxCNAMECandidates(ctx, targetID, names) + + // admit appends one classified name to the batched write set. Only names + // absent from `preexisting` stream the live "new_subdomain" event, and only + // when they carry an A/AAAA answer (CNAME-only rows stay silent, ip == ""). + admit := func(name, ip, source string) { + cmu.Lock() + rows = append(rows, resolvedRow{name, ip, source}) + isNew := !preexisting[name] + cmu.Unlock() + if isNew && ip != "" && s.broadcast != nil { + s.broadcast("new_subdomain", map[string]any{ + "target_id": targetID, + "subdomain": name, + "ip": ip, + }) + } + } + + // cnameFor answers the explicit-CNAME question for an unresolved name. The + // dnsx -cname -resp prefilter already captured the target through the same + // resolver file the A/AAAA answers use; the in-process 3s lookup is only + // the fallback (tool-free build, dnsx unavailable, or a parse hiccup). + cnameFor := func(name string) string { + if cname, ok := cnameTargets[name]; ok { + return cname + } + if !cnamePrefilter || cnameCandidates[name] { + return lookupExplicitCNAME(ctx, name, s.resolver) + } + return "" + } - for i := 0; i < workers; i++ { - wg.Add(1) + // classify applies the identical wildcard-aware admission rules to one + // name's A/AAAA record set no matter which resolver path produced them — + // the batched dnsx pass or per-name Go lookups — so both admit/reject + // name-for-name identically (the gate counts below must match across paths). + // definitive distinguishes an authoritative NXDOMAIN (safe to reject and + // prune) from a timeout/SERVFAIL/rate-limited miss (must NOT reject: doing + // so deletes real hosts and starves every downstream detector). + classify := func(name string, ips []string, definitive bool) { + ip := wildcardProfile.firstNonWildcardIP(name, ips) + if ip == "" { + if len(ips) > 0 && wildcardProfile.overlapsWildcardIPs(name, ips) { + cmu.Lock() + rejected[name] = true + wildcardHits++ + cmu.Unlock() + return + } + // A dangling but explicit CNAME is a real DNS asset and must remain + // available to takeover checks, but it is not sent to the web pipeline. + if cname := cnameFor(name); cname != "" && !wildcardProfile.isWildcardCNAME(name, cname) { + cmu.Lock() + cnameOnly++ + cmu.Unlock() + admit(name, "", "dns-cname") + return + } + cmu.Lock() + if definitive { + rejected[name] = true + unresolved++ + } else { + transient++ + } + cmu.Unlock() + return + } + cmu.Lock() + resolved++ + cmu.Unlock() + admit(name, ip, "dns") + } + + // FAST PATH: one dnsx process resolves the A/AAAA record sets for the whole + // candidate list through the fast public/configured resolvers — one standard + // query per name, high in-flight concurrency, the same pattern the deep-DNS + // brute already uses — instead of paying a per-worker connect/timeout budget + // per name. + passStart := time.Now() + ipmap := s.dnsxResolveIPs(ctx, targetID, names) + if ipmap != nil { + logFn("info", "subdomain_enum", fmt.Sprintf("Batched DNS resolution finished in %.1fs — screening %d candidate(s)", time.Since(passStart).Seconds(), len(names))) + for _, name := range names { + if ctx.Err() != nil { + break + } + // dnsx -resp reports only answered names, so an absent name is + // ambiguous (NXDOMAIN or a throttled/timed-out query). Treat it as + // transient: never reject or prune on this path. Only the per-name + // fallback below can prove an authoritative NXDOMAIN. + classify(name, ipmap[name], false) + } + } else { + logFn("info", "subdomain_enum", "Batched dnsx resolution unavailable — per-name resolver fallback") + } + + if ipmap == nil { + // FALLBACK PATH (tool-free builds, missing dnsx, or a failed tool run): + // the legacy per-name Go-resolver pass, admission logic unchanged. + workers := s.cfg.Workers.SubdomainEnumeration + if workers <= 0 { + workers = 20 + } + jobs := make(chan string, len(names)) + var wg sync.WaitGroup + var checked atomic.Int64 + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for name := range jobs { + if ctx.Err() != nil { + return + } + checked.Add(1) + ips, definitive := lookupIPs(ctx, name, s.resolver) + classify(name, ips, definitive) + } + }() + } + // Periodic heartbeat so a long pass over tens of thousands of names no + // longer looks frozen between the start and the final gate line. + progressDone := make(chan struct{}) go func() { - defer wg.Done() - for name := range jobs { - if ctx.Err() != nil { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for { + select { + case <-progressDone: return - } - ips, definitive := lookupIPs(ctx, name, s.resolver) - ip := firstNonWildcardIP(ips, wildcardIPs) - if ip == "" { - if len(ips) > 0 && overlapsWildcardIPs(ips, wildcardIPs) { - cmu.Lock() - rejected[name] = true - wildcardHits++ - cmu.Unlock() - continue - } - // A dangling but explicit CNAME is a real DNS asset and must remain - // available to takeover checks, but it is not sent to the web pipeline. - cname := "" - if !cnamePrefilter || cnameCandidates[name] { - cname = lookupExplicitCNAME(ctx, name, s.resolver) - } - if cname != "" && !wildcardCNAMEs[cname] { - cmu.Lock() - cnameOnly++ - rows = append(rows, resolvedRow{name, "", "dns-cname"}) - cmu.Unlock() - continue - } - // Only an AUTHORITATIVE "no such host" is a real negative that may be - // rejected (and later pruned). A non-definitive miss — timeout, - // SERVFAIL, or a rate-limited public resolver — must NOT reject the - // name: doing so silently deleted real, resolvable hosts and starved - // every downstream detector. Leave such names untouched so an existing - // row survives and the next pass can re-resolve them. + case <-ticker.C: cmu.Lock() - if definitive { - rejected[name] = true - unresolved++ - } else { - transient++ - } + logFn("info", "subdomain_enum", fmt.Sprintf("Resolution progress: %d/%d checked — %d verified, %d wildcard, %d CNAME-only, %d unresolved", checked.Load(), len(names), resolved, cnameOnly, wildcardHits, unresolved)) cmu.Unlock() - continue - } - cmu.Lock() - resolved++ - rows = append(rows, resolvedRow{name, ip, "dns"}) - isNew := !preexisting[name] - cmu.Unlock() - if isNew && s.broadcast != nil { - s.broadcast("new_subdomain", map[string]any{ - "target_id": targetID, - "subdomain": name, - "ip": ip, - }) } } }() + for _, n := range names { + jobs <- n + } + close(jobs) + wg.Wait() + close(progressDone) } - for _, n := range names { - jobs <- n - } - close(jobs) - wg.Wait() - // Single-transaction flush of all admission writes. if len(rows) > 0 { if tx, err := s.db.BeginTx(ctx, nil); err == nil { @@ -1202,7 +1303,8 @@ func (s *SubdomainScanner) resolveAll(ctx context.Context, targetID string, foun ip = excluded.ip, source = CASE WHEN COALESCE(subdomains.source,'dns') IN ('seed','vhost') THEN subdomains.source ELSE excluded.source END, - last_seen = CURRENT_TIMESTAMP`) + last_seen = CURRENT_TIMESTAMP, + miss_count = 0`) if perr == nil { for _, r := range rows { _, _ = stmt.ExecContext(ctx, uuid.New().String(), targetID, r.name, r.ip, r.source) @@ -1302,9 +1404,12 @@ func (s *SubdomainScanner) recursiveEnum(ctx context.Context, targetID, apex str if len(cands) == 0 { continue } - wIPs, wCNAMEs := detectWildcard(sub, s.resolver) + wildcardProfile := detectWildcard(sub, s.resolver, s.wildcardServers(), logFn) + // Recursion probes at most 3 branch suffixes per sub-apex — a small + // budget, since up to 50 sub-apexes can run per level. + wildcardProfile.detectBranches(ctx, sub, mapKeys(cands), s.resolver, s.wildcardServers(), 3, logFn) var mu sync.Mutex - rejected := s.resolveAll(ctx, targetID, cands, &mu, wIPs, wCNAMEs, logFn) + rejected := s.resolveAll(ctx, targetID, cands, &mu, wildcardProfile, logFn) // Names that resolved (A/AAAA or CNAME) are the ones worth descending // into: everything in cands that the admission gate did NOT reject. for n := range cands { @@ -1321,9 +1426,21 @@ func (s *SubdomainScanner) recursiveEnum(ctx context.Context, targetID, apex str // resolvingSubdomainsUnder returns the stored, resolving (has an IP) subdomains // strictly under apex, excluding anything already visited — the seed frontier // for recursive enumeration. +// escapeLike neutralises SQLite LIKE metacharacters ('%', '_' and the escape +// character itself) in a value folded into a LIKE pattern. Apex names are +// validated at intake, but escaping keeps a malformed stored domain from +// widening the pattern beyond its own zone. +func escapeLike(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `%`, `\%`) + s = strings.ReplaceAll(s, `_`, `\_`) + return s +} + func (s *SubdomainScanner) resolvingSubdomainsUnder(ctx context.Context, targetID, apex string, visited map[string]bool) []string { rows, err := s.db.QueryContext(ctx, `SELECT subdomain FROM subdomains - WHERE target_id=? AND COALESCE(ip,'')!='' AND subdomain LIKE ?`, targetID, "%."+strings.ToLower(apex)) + WHERE target_id=? AND COALESCE(catchall,0)=0 AND COALESCE(ip,'')!='' + AND subdomain LIKE ? ESCAPE '\'`, targetID, "%."+escapeLike(strings.ToLower(apex))) if err != nil { return nil } @@ -1382,12 +1499,19 @@ func (s *SubdomainScanner) gatherPassiveSubdomains(ctx context.Context, domain s return out } -func (s *SubdomainScanner) dnsxCNAMECandidates(ctx context.Context, targetID string, names []string) (map[string]bool, bool) { +// dnsxCNAMECandidates runs one dnsx `-cname -resp` pass over the whole +// candidate list through the configured fast public resolvers. It returns the +// set of names carrying an explicit CNAME plus, where parseable, the exact +// CNAME target — so the admission gate usually needs no second in-process +// lookup. The third return value is false when dnsx was unavailable or the +// pass failed; the caller then falls back to per-name CNAME lookups. +func (s *SubdomainScanner) dnsxCNAMECandidates(ctx context.Context, targetID string, names []string) (map[string]bool, map[string]string, bool) { if s.exec == nil || len(names) == 0 || !s.exec.IsToolAvailable("dnsx") { - return nil, false + return nil, nil, false } out := map[string]bool{} - args := []string{"-silent", "-cname", "-retry", "1"} + targets := map[string]string{} + args := []string{"-silent", "-cname", "-resp", "-retry", "1"} // Resolve the CNAME prefilter through the configured fast public resolvers, // not the (often slow/single-flight) system resolver — this was a full extra // serial DNS pass over the whole candidate list with no -r before. @@ -1397,79 +1521,32 @@ func (s *SubdomainScanner) dnsxCNAMECandidates(ctx context.Context, targetID str } err := s.exec.RunWithInputCallback(ctx, strings.NewReader(strings.Join(names, "\n")), targetID, func(line string) { - fields := strings.Fields(strings.ToLower(strings.TrimSpace(line))) - if len(fields) > 0 { - out[strings.TrimSuffix(fields[0], ".")] = true + name, toks := parseDNSXResponseLine(line) + if name == "" { + // Defensive: a response shape without a usable bracket group + // still marks the name as CNAME-bearing (legacy behavior). + fields := strings.Fields(strings.ToLower(strings.TrimSpace(line))) + if len(fields) > 0 { + name = strings.TrimSuffix(fields[0], ".") + } + } + if name == "" { + return + } + out[name] = true + if len(toks) > 0 { + targets[name] = toks[0] } }, "dnsx", args...) if err != nil && ctx.Err() == nil { - return nil, false + return nil, nil, false } - return out, true + return out, targets, true } -// detectWildcard probes a few random hostnames; if they resolve, the shared -// IPs are wildcard answers we should treat with suspicion. -func detectWildcard(domain string, r *net.Resolver) (map[string]bool, map[string]bool) { - wildcardIPs := make(map[string]bool) - wildcardCNAMEs := make(map[string]bool) - probes := []string{ - "zz9z3k7q1wildcard." + domain, - "qx84jd02noexist." + domain, - "a1b2c3d4e5random." + domain, - "rcn7f4a9neverexists." + domain, - "nohost2c8e1b6d." + domain, - } - resolvedProbes := 0 - allIPs := map[string]bool{} - cnameCounts := map[string]int{} - type wildcardProbeResult struct { - ips []net.IPAddr - cname string - } - results := make(chan wildcardProbeResult, len(probes)) - var wg sync.WaitGroup - for _, p := range probes { - wg.Add(1) - go func(name string) { - defer wg.Done() - ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) - ips, _ := resolverOrDefault(r).LookupIPAddr(ctx, name) - cancel() - results <- wildcardProbeResult{ips: ips, cname: lookupExplicitCNAME(context.Background(), name, r)} - }(p) - } - wg.Wait() - close(results) - for result := range results { - if result.cname != "" { - cnameCounts[result.cname]++ - } - if len(result.ips) > 0 { - resolvedProbes++ - } - for _, ip := range result.ips { - value := ip.IP.String() - if value != "" { - allIPs[value] = true - } - } - } - // Require at least three independent random labels to resolve, then retain the - // union of their addresses. This catches rotating wildcard/CDN pools where no - // single IP necessarily appears twice, without trusting one poisoned answer. - if resolvedProbes >= 3 { - for ip := range allIPs { - wildcardIPs[ip] = true - } - } - for cname, n := range cnameCounts { - if n >= 3 { - wildcardCNAMEs[cname] = true - } - } - return wildcardIPs, wildcardCNAMEs -} +// detectWildcard in subdomain.go has been replaced by the zone-aware engine +// in wildcard.go (wildcardProfile + cross-resolver probe retry + branch +// detection). See wildcard.go. func (s *SubdomainScanner) upsertSubdomain(targetID, subdomain, ip string) error { return s.upsertSubdomainSource(targetID, subdomain, ip, "dns") @@ -1492,6 +1569,11 @@ func (s *SubdomainScanner) upsertSubdomainSource(targetID, subdomain, ip, source // pruneRejectedSubdomains repairs rows created by older eager-admission builds. // The delete is intentionally narrow: never touch explicit seeds, CNAME records, // verified vhosts or anything that was previously observed alive. +// pruneRejectedSubdomains removes DNS-sourced rows for names that stopped +// resolving. ONE missed pass is not proof of death — a resolver timeout or a +// dnsx flake must not wipe a previously verified row (and, with it, its +// recursion-frontier seed) — so deletion requires two CONSECUTIVE misses. A +// pass that resolves the name resets the counter in resolveAll's upsert. func (s *SubdomainScanner) pruneRejectedSubdomains(ctx context.Context, targetID string, rejected map[string]bool) { if len(rejected) == 0 { return @@ -1500,20 +1582,31 @@ func (s *SubdomainScanner) pruneRejectedSubdomains(ctx context.Context, targetID if err != nil { return } - stmt, err := tx.PrepareContext(ctx, `DELETE FROM subdomains + upd, err := tx.PrepareContext(ctx, `UPDATE subdomains + SET miss_count = COALESCE(miss_count,0) + 1 WHERE target_id=? AND subdomain=? AND COALESCE(source,'dns')='dns' AND COALESCE(is_alive,0)=0`) if err != nil { _ = tx.Rollback() return } + del, err := tx.PrepareContext(ctx, `DELETE FROM subdomains + WHERE target_id=? AND subdomain=? AND COALESCE(source,'dns')='dns' + AND COALESCE(is_alive,0)=0 AND COALESCE(miss_count,0)>=2`) + if err != nil { + _ = upd.Close() + _ = tx.Rollback() + return + } for name := range rejected { if ctx.Err() != nil { break } - _, _ = stmt.ExecContext(ctx, targetID, name) + _, _ = upd.ExecContext(ctx, targetID, name) + _, _ = del.ExecContext(ctx, targetID, name) } - _ = stmt.Close() + _ = upd.Close() + _ = del.Close() _ = tx.Commit() } @@ -1551,6 +1644,16 @@ func resolverOrDefault(r *net.Resolver) *net.Resolver { return r } +// wildcardServers returns the upstream resolver list the wildcard probes use. +// It is intentionally the SAME list the dnsx batched resolution passes supply +// via -r, so probe verdicts are drawn from the resolver pool whose answers the +// admission gate actually compares against. The first probe attempt still uses +// the shared in-process resolver, so a split-horizon zone seen from inside is +// represented too. +func (s *SubdomainScanner) wildcardServers() []string { + return s.toolResolverList() +} + func resolveHostIPs(parent context.Context, host string, r *net.Resolver) []string { ips, _ := lookupIPs(parent, host, r) return ips @@ -1624,33 +1727,6 @@ func dedupeIPs(rows []net.IPAddr) []string { return out } -func firstNonWildcardIP(ips []string, wildcard map[string]bool) string { - // If an answer overlaps the random-label wildcard profile, treat the complete - // RRset as wildcard. Rotating pools often return one familiar and one new IP; - // selecting the new address would incorrectly admit the fake hostname. - for _, ip := range ips { - if wildcard[ip] { - return "" - } - } - for _, ip := range ips { - return ip - } - return "" -} - -func overlapsWildcardIPs(ips []string, wildcard map[string]bool) bool { - if len(ips) == 0 || len(wildcard) == 0 { - return false - } - for _, ip := range ips { - if wildcard[ip] { - return true - } - } - return false -} - func lookupExplicitCNAME(parent context.Context, host string, r *net.Resolver) string { ctx, cancel := context.WithTimeout(parent, 3*time.Second) defer cancel() diff --git a/internal/scanner/subdomain_admission_test.go b/internal/scanner/subdomain_admission_test.go index 0ca5eed..5e23c39 100644 --- a/internal/scanner/subdomain_admission_test.go +++ b/internal/scanner/subdomain_admission_test.go @@ -66,18 +66,27 @@ func TestRejectedAdmissionPrunesOnlyUnverifiedDNSRows(t *testing.T) { insert("vhost.m.local", "vhost", 0) s := &SubdomainScanner{db: db} - s.pruneRejectedSubdomains(context.Background(), tid, map[string]bool{ + rejected := map[string]bool{ "junk.m.local": true, "historically-live.m.local": true, "seed.m.local": true, "cname.m.local": true, "vhost.m.local": true, - }) + } + // One missed resolve pass must never delete: transient resolver failures are + // common. The never-live eager DNS row survives the first pass at miss_count=1. + s.pruneRejectedSubdomains(context.Background(), tid, rejected) var n int _ = db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE target_id=?`, tid).Scan(&n) + if n != 5 { + t.Fatalf("a single missed pass pruned rows early: rows=%d", n) + } + // The second CONSECUTIVE miss prunes only that never-live eager DNS row. + s.pruneRejectedSubdomains(context.Background(), tid, rejected) + _ = db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE target_id=?`, tid).Scan(&n) if n != 4 { t.Fatalf("expected only the never-live eager DNS row to be pruned, rows=%d", n) } _ = db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE target_id=? AND subdomain='junk.m.local'`, tid).Scan(&n) if n != 0 { - t.Fatal("unverified junk DNS row survived admission cleanup") + t.Fatal("unverified junk DNS row survived two consecutive missed passes") } } diff --git a/internal/scanner/subdomain_prune_test.go b/internal/scanner/subdomain_prune_test.go new file mode 100644 index 0000000..160ddce --- /dev/null +++ b/internal/scanner/subdomain_prune_test.go @@ -0,0 +1,109 @@ +package scanner + +import ( + "context" + "testing" + + "github.com/google/uuid" +) + +// A single missed resolve pass must not delete a previously known row. The +// second CONSECUTIVE miss does. This guards against one dnsx/resolver hiccup +// wiping a target's DNS history and its recursion-frontier seeds. +func TestPruneRejectedSubdomainsRequiresTwoConsecutiveMisses(t *testing.T) { + db := newMonitorTestDB(t) + targetID := uuid.NewString() + if _, err := db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, "example.test"); err != nil { + t.Fatal(err) + } + ins := func(name, ip, source string, alive int) { + if _, err := db.Exec(`INSERT INTO subdomains(id,target_id,subdomain,ip,source,is_alive) VALUES(?,?,?,?,?,?)`, + uuid.NewString(), targetID, name, ip, source, alive); err != nil { + t.Fatal(err) + } + } + ins("dns.example.test", "1.2.3.4", "dns", 0) + ins("alive.example.test", "1.2.3.5", "dns", 1) + ins("seed.example.test", "1.2.3.6", "seed", 0) + + s := &SubdomainScanner{db: db} + rejected := map[string]bool{ + "dns.example.test": true, + "alive.example.test": true, + "seed.example.test": true, + } + count := func(name string) int { + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE target_id=? AND subdomain=?`, targetID, name).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + misses := func(name string) int { + var n int + if err := db.QueryRow(`SELECT COALESCE(miss_count,0) FROM subdomains WHERE target_id=? AND subdomain=?`, targetID, name).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + + s.pruneRejectedSubdomains(context.Background(), targetID, rejected) + if count("dns.example.test") != 1 || misses("dns.example.test") != 1 { + t.Fatalf("first miss deleted the row (count=%d misses=%d)", count("dns.example.test"), misses("dns.example.test")) + } + s.pruneRejectedSubdomains(context.Background(), targetID, rejected) + if count("dns.example.test") != 0 { + t.Fatal("second consecutive miss did not prune the stale DNS row") + } + if count("alive.example.test") != 1 || count("seed.example.test") != 1 { + t.Fatal("prune must never touch alive or operator-seeded rows") + } +} + +// A pass that resolves the name resets the miss counter, so two flakes years +// apart can never accumulate into a deletion without two CONSECUTIVE misses. +func TestResolvingUpsertResetsMissCount(t *testing.T) { + db := newMonitorTestDB(t) + targetID := uuid.NewString() + if _, err := db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, "example.test"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO subdomains(id,target_id,subdomain,ip,source,is_alive,miss_count) VALUES(?,?,?,?,?,0,1)`, + uuid.NewString(), targetID, "back.example.test", "1.2.3.4", "dns"); err != nil { + t.Fatal(err) + } + // The exact upsert resolveAll runs when a name is admitted this pass. + if _, err := db.Exec(` + INSERT INTO subdomains (id, target_id, subdomain, ip, source, last_seen) + VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(target_id, subdomain) DO UPDATE SET + ip = excluded.ip, + source = CASE WHEN COALESCE(subdomains.source,'dns') IN ('seed','vhost') + THEN subdomains.source ELSE excluded.source END, + last_seen = CURRENT_TIMESTAMP, + miss_count = 0`, + uuid.NewString(), targetID, "back.example.test", "5.6.7.8", "dns"); err != nil { + t.Fatal(err) + } + var misses int + if err := db.QueryRow(`SELECT COALESCE(miss_count,0) FROM subdomains WHERE target_id=? AND subdomain='back.example.test'`, targetID).Scan(&misses); err != nil { + t.Fatal(err) + } + if misses != 0 { + t.Fatalf("resolving pass left miss_count=%d, want 0", misses) + } +} + +func TestEscapeLike(t *testing.T) { + cases := map[string]string{ + "plain.test": "plain.test", + `back\slash`: `back\\slash`, + "pct%name": `pct\%name`, + "under_score": `under\_score`, + } + for in, want := range cases { + if got := escapeLike(in); got != want { + t.Errorf("escapeLike(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/scanner/takeover.go b/internal/scanner/takeover.go index 6fb4d5f..e2b715e 100644 --- a/internal/scanner/takeover.go +++ b/internal/scanner/takeover.go @@ -104,7 +104,7 @@ var takeoverFingerprints = []takeoverFingerprint{ func (s *TakeoverScanner) Run(ctx context.Context, targetID string, logFn LogFunc) error { logFn("info", "takeover", "Checking for subdomain takeovers...") - rows, err := s.db.QueryContext(ctx, `SELECT subdomain FROM subdomains WHERE target_id = ?`, targetID) + rows, err := s.db.QueryContext(ctx, `SELECT subdomain FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0)=0`, targetID) if err != nil { return fmt.Errorf("query subdomains: %w", err) } diff --git a/internal/scanner/wildcard.go b/internal/scanner/wildcard.go new file mode 100644 index 0000000..9195e74 --- /dev/null +++ b/internal/scanner/wildcard.go @@ -0,0 +1,568 @@ +package scanner + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "net" + "sort" + "strings" + "sync" + "time" +) + +// The wildcard gate answers one question for a DNS zone: "does a random +// hostname under this suffix resolve anyway?" If it does, every passive/ +// brute "discovery" under that suffix may be a fabricated name served by the +// wildcard instead of a real asset, and the resolution gate has to prove each +// name against the wildcard's answer pool before admitting it. +// +// Reconner previously ran a single batch of 5 fixed probes and treated any +// DNS error like NXDOMAIN. Three failure modes slipped through: +// +// 1. A flaky answer (SERVFAIL, rate limit, timeout) looked identical to +// NXDOMAIN. Fewer than 3 probes resolved => no wildcard evidence => the +// whole junk namespace was admitted (the "sometimes everything looks like +// a subdomain" bug). +// 2. Rotating CDN pools can answer more than 5 probes' worth of IPs, and +// CNAME-target rotation needs more samples for the >=3 agreement rule. +// 3. Branch wildcards (e.g. *.git.example.com under example.com) never got +// probed at all — the apex probes test only the apex. A junk fleet under +// a deeper wildcard (asdf-garbage.git.example.com) was admitted wholesale. +// +// This file fixes all three: cross-resolver retry with an explicit NXDOMAIN +// verdict (net.DNSError.IsNotFound), a second independent probe batch when +// the first is inconclusive, and branch probes generated from clustered +// candidate suffixes. + +// probeVerdict classifies one random-label lookup. The distinction matters: +// probeNX is definitive proof the label does not exist, while probeAmbiguous +// (SERVFAIL/timeouts) proves nothing and must trigger more sampling. +type probeVerdict int + +const ( + probeResolved probeVerdict = iota + probeNX + probeAmbiguous +) + +// fixedWildcardProbeStems are the original five probe labels. They are kept +// byte-for-byte so a scan reacts to the same zones previous releases +// detected, and so tests are deterministic. The second (adaptive) batch uses +// fresh random labels instead, in case an operator's zone matches the fixed +// stems for some exotic reason. +var fixedWildcardProbeStems = []string{ + "zz9z3k7q1wildcard", + "qx84jd02noexist", + "a1b2c3d4e5random", + "rcn7f4a9neverexists", + "nohost2c8e1b6d", +} + +// wildcardProbeMinAgreement is how many probes must agree before a verdict is +// trusted: 3 resolving labels prove a wildcard, 3 definitive NXDOMAINs prove +// the absence of one, 3 identical CNAME targets prove a CNAME-only wildcard. +const wildcardProbeMinAgreement = 3 + +// wildcardZone carries the answer-pool evidence for ONE suffix of DNS space. +// suffix == "" marks the apex zone (covers every candidate under the target +// domain); anything else marks a strictly deeper branch zone. +type wildcardZone struct { + suffix string // "" = apex; else e.g. "git.example.com" + ips map[string]bool + cnames map[string]bool +} + +// hasEvidence reports whether the zone is a proven wildcard (any shared IP or +// CNAME answer pool). An empty zone is NOT a wildcard, including the +// "definitively NXDOMAIN" verdict. +func (z wildcardZone) hasEvidence() bool { + return len(z.ips) > 0 || len(z.cnames) > 0 +} + +// wildcardProfile bundles the apex zone with any confirmed branch zones and +// answers the admission-gate questions per candidate name. A nil profile +// behaves like "no wildcard evidence", which keeps test paths simple. +type wildcardProfile struct { + apex wildcardZone + branches []wildcardZone +} + +// normalizeSubdomainName lowercases and trims a trailing dot so wildcard +// matching is case/format insensitive. +func normalizeSubdomainName(name string) string { + return strings.ToLower(strings.TrimSuffix(strings.TrimSpace(name), ".")) +} + +// mapKeys returns the key set of a candidate map as a slice. +func mapKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + +// zonesFor returns every zone whose suffix covers name: the apex zone always +// applies, plus each branch zone whose suffix the name sits strictly under +// (name != suffix). Branch suffixes are matched on full-suffix boundaries so +// "es.git.example.com" cannot satisfy a "git.example.com" zone via a partial +// label match. +func (p *wildcardProfile) zonesFor(name string) []wildcardZone { + name = normalizeSubdomainName(name) + zones := make([]wildcardZone, 0, 3) + if p != nil { + zones = append(zones, p.apex) + for i := range p.branches { + if strings.HasSuffix(name, "."+p.branches[i].suffix) { + zones = append(zones, p.branches[i]) + } + } + } + return zones +} + +// overlapsWildcardIPs reports whether ANY address of the record set belongs to +// a wildcard answer pool applicable to this name. The "any" rule mirrors +// rotating-pool behavior: one old-pool IP plus one unfamiliar IP is still a +// wildcard answer, so the whole RRset is treated as wildcard. +func (p *wildcardProfile) overlapsWildcardIPs(name string, ips []string) bool { + if len(ips) == 0 { + return false + } + for _, zone := range p.zonesFor(name) { + if len(zone.ips) == 0 { + continue + } + for _, ip := range ips { + if zone.ips[ip] { + return true + } + } + } + return false +} + +// firstNonWildcardIP returns the first address of the record set unless the +// set overlaps a wildcard pool applicable to the name, in which case it +// returns "" (the entire RRset is rejected, matching the pre-branch behavior). +func (p *wildcardProfile) firstNonWildcardIP(name string, ips []string) string { + if p == nil { + for _, ip := range ips { + return ip + } + return "" + } + if p.overlapsWildcardIPs(name, ips) { + return "" + } + for _, ip := range ips { + return ip + } + return "" +} + +// isWildcardCNAME reports whether a candidate's CNAME target matches a +// wildcard answer pool applicable to the name. +func (p *wildcardProfile) isWildcardCNAME(name, cname string) bool { + if cname == "" { + return false + } + cname = normalizeSubdomainName(cname) + for _, zone := range p.zonesFor(name) { + if zone.cnames[cname] { + return true + } + } + return false +} + +// apexWildcard reports whether the APEX zone alone is a wildcard. Only an apex +// wildcard makes active brute-force meaningless; a branch wildcard still +// leaves the rest of the zone worth enumerating (branch hits are filtered +// per-name by the admission gate). +func (p *wildcardProfile) apexWildcard() bool { + return p != nil && p.apex.hasEvidence() +} + +// hasAnyWildcard reports whether ANY zone carries evidence (used for the +// operator-facing warning line). +func (p *wildcardProfile) hasAnyWildcard() bool { + if p == nil { + return false + } + if p.apex.hasEvidence() { + return true + } + for _, z := range p.branches { + if z.hasEvidence() { + return true + } + } + return false +} + +// resolverServerAddress normalizes one resolver entry to a dialable host:port +// address. The configured/derived lists carry BARE hosts ("1.1.1.1") for the +// dnsx resolver file, and a bare host is NOT a valid dial target ("missing +// port in address"). Port 53 is assumed when absent; entries that already have +// a port keep it, and IPv6 literals get bracket-wrapped by JoinHostPort. +func resolverServerAddress(server string) string { + server = strings.TrimSpace(server) + if server == "" { + return "" + } + if _, _, err := net.SplitHostPort(server); err == nil { + return server + } + // A bracketed literal without a port ("[2606:4700::1111]") is not a valid + // SplitHostPort input; strip the brackets before re-joining. + host := strings.TrimPrefix(strings.TrimSuffix(server, "]"), "[") + return net.JoinHostPort(host, "53") +} + +// singleServerResolver builds a resolver pinned to ONE upstream server so a +// retry genuinely asks a DIFFERENT resolver — buildDNSResolver rotates per +// connection, which is usually enough, but an explicit pin gives the retry +// loop a guarantee. +func singleServerResolver(server string) *net.Resolver { + address := resolverServerAddress(server) + return &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, _ string) (net.Conn, error) { + d := net.Dialer{Timeout: 4 * time.Second} + return d.DialContext(ctx, network, address) + }, + } +} + +// resolveProbeWithRetry resolves one probe name, retrying ambiguous answers +// (SERVFAIL/timeout/refused) on DIFFERENT upstream resolvers. A definitive +// NXDOMAIN (dnsErr.IsNotFound) is recorded and, once seen, a later attempt +// that succeeds still wins — a resolver badly configured for the zone must not +// mask an existing name. +func resolveProbeWithRetry(r *net.Resolver, name string, servers []string, startOffset int) ([]net.IPAddr, probeVerdict) { + if r == nil { + r = net.DefaultResolver + } + attempts := len(servers) + if attempts == 0 { + attempts = 1 // system resolver only; r carries it + } + if attempts > 3 { + attempts = 3 + } + var sawNX, sawAmbiguous bool + for attempt := 0; attempt < attempts; attempt++ { + if attempt > 0 { + // Small backoff between attempts: a rate-limited resolver answered + // the first query instantly, so give its budget a beat to recover. + time.Sleep(time.Duration(attempt) * 150 * time.Millisecond) + } + var res *net.Resolver + if attempt == 0 { + // First attempt uses the shared resolver (exactly the resolver the + // candidate admission gate re-checks against), then explicitly + // pinned servers for retries. + res = r + } else { + idx := (startOffset + attempt) % len(servers) + if strings.TrimSpace(servers[idx]) == "" { + sawAmbiguous = true + continue + } + res = singleServerResolver(servers[idx]) + } + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + result, err := res.LookupIPAddr(ctx, name) + cancel() + if err == nil { + return result, probeResolved + } + if dnsErr, ok := err.(*net.DNSError); ok && dnsErr.IsNotFound { + sawNX = true + continue + } + sawAmbiguous = true + } + switch { + case sawNX && !sawAmbiguous: + return nil, probeNX + case sawNX && sawAmbiguous: + // One resolver says "no such name", another errored. The disagreement + // is exactly what extra probes are for; count it ambiguous so the + // caller runs a second batch instead of trusting either answer. + return nil, probeAmbiguous + default: + return nil, probeAmbiguous + } +} + +type wildcardProbe struct { + name string + ips []net.IPAddr + cname string + verdict probeVerdict +} + +// runWildcardProbes probes every name concurrently and returns the verdicts. +// CNAME lookups run only for names that were not definitively NXDOMAIN — +// LookupCNAME on an NXDOMAIN just burns its 3s timeout for nothing. +func runWildcardProbes(names []string, r *net.Resolver, servers []string) []wildcardProbe { + results := make([]wildcardProbe, len(names)) + var wg sync.WaitGroup + for i, name := range names { + wg.Add(1) + go func(i int, name string) { + defer wg.Done() + ips, verdict := resolveProbeWithRetry(r, name, servers, i) + probe := wildcardProbe{name: name, ips: ips, verdict: verdict} + if verdict != probeNX { + probe.cname = lookupExplicitCNAME(context.Background(), name, r) + } + results[i] = probe + }(i, name) + } + wg.Wait() + return results +} + +// summarizeWildcardProbes aggregates probe verdicts into the counts and answer +// pools the wildcard decision needs. +func summarizeWildcardProbes(probes []wildcardProbe) (resolved, nxDomain, ambiguous int, allIPs map[string]bool, cnameCounts map[string]int) { + allIPs = map[string]bool{} + cnameCounts = map[string]int{} + for _, probe := range probes { + switch probe.verdict { + case probeResolved: + resolved++ + case probeNX: + nxDomain++ + default: + ambiguous++ + } + for _, ip := range probe.ips { + if value := ip.IP.String(); value != "" { + allIPs[value] = true + } + } + if probe.cname != "" { + cnameCounts[probe.cname]++ + } + } + return resolved, nxDomain, ambiguous, allIPs, cnameCounts +} + +// decideWildcardZone turns probe aggregates into a zone. The bool reports +// whether the batch was inconclusive (neither side reached agreement) and a +// second batch should run. IP/CNAME pools stored in the zone are the union of +// all answers seen, guarded by the agreement threshold. +func decideWildcardZone(resolved, nxDomain int, allIPs map[string]bool, cnameCounts map[string]int, suffix string) (wildcardZone, bool) { + zone := wildcardZone{suffix: suffix, ips: map[string]bool{}, cnames: map[string]bool{}} + if resolved >= wildcardProbeMinAgreement { + // At least three independent random labels resolved: everything they + // answered is part of the wildcard's answer pool. Union all addresses + // so rotating CDN pools are captured even when no single IP repeats. + for ip := range allIPs { + zone.ips[ip] = true + } + for cname, n := range cnameCounts { + if n >= wildcardProbeMinAgreement { + zone.cnames[cname] = true + } + } + return zone, false + } + if nxDomain >= wildcardProbeMinAgreement { + // At least three labels are definitively nonexistent: this suffix is + // NOT a wildcard. An empty zone keeps the old "no evidence" behavior + // but now with proof, and the caller skips the second batch. + return zone, false + } + return zone, true +} + +// randomProbeLabel returns a hex label from crypto/rand. Probe labels are not +// security tokens, but they must be unpredictable per scan (a cached wildcard +// answer for a guessed label would poison the verdict), which crypto/rand +// guarantees for free. +func randomProbeLabel() string { + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + return fmt.Sprintf("rcnfallback%d", time.Now().UnixNano()&0xffff) + } + return hex.EncodeToString(buf) +} + +// probeWildcardZone runs the five fixed probes (plus a second random batch +// when the first is inconclusive) for one DNS suffix and returns the zone. +// fullSuffix is the target domain for the apex zone, or a deeper suffix (e.g. +// "git.example.com") for a branch zone. Zone.suffix stays "" for the apex so +// zonesFor matches it unconditionally. +func probeWildcardZone(fullSuffix, zoneSuffix string, r *net.Resolver, servers []string, logFn LogFunc) wildcardZone { + apex := zoneSuffix == "" + probes := runWildcardProbes(functionProbeNames(fixedWildcardProbeStems, fullSuffix), r, servers) + resolved, nxDomain, ambiguous, allIPs, cnameCounts := summarizeWildcardProbes(probes) + + zone, needsMore := decideWildcardZone(resolved, nxDomain, allIPs, cnameCounts, zoneSuffix) + if !needsMore { + if ambiguous > 0 && logFn != nil { + logFn("info", "subdomain_enum", fmt.Sprintf( + "Wildcard probes under %s settled: %d resolved, %d NXDOMAIN, %d errored — verdict reached anyway", + fullSuffix, resolved, nxDomain, ambiguous)) + } + return zone + } + + // Inconclusive: sample five fresh labels. If the resolver was rate-limiting + // or a CDN node was flaky, the new batch answers through the same retry + // logic with a fresh transaction budget. + if logFn != nil { + logFn("info", "subdomain_enum", fmt.Sprintf( + "Wildcard probes under %s inconclusive (%d resolved, %d NXDOMAIN, %d errored) — probing a second batch", + fullSuffix, resolved, nxDomain, ambiguous)) + } + second := make([]string, 0, len(fixedWildcardProbeStems)) + for range fixedWildcardProbeStems { + second = append(second, randomProbeLabel()+"."+fullSuffix) + } + probes = append(probes, runWildcardProbes(second, r, servers)...) + resolved, nxDomain, ambiguous, allIPs, cnameCounts = summarizeWildcardProbes(probes) + zone, _ = decideWildcardZone(resolved, nxDomain, allIPs, cnameCounts, zoneSuffix) + if !zone.hasEvidence() && logFn != nil && apex { + logFn("info", "subdomain_enum", fmt.Sprintf( + "Wildcard probes under %s inconclusive after both batches (%d resolved, %d NXDOMAIN, %d errored) — treating as non-wildcard", + fullSuffix, resolved, nxDomain, ambiguous)) + } + return zone +} + +// functionProbeNames joins probe stems with a DNS suffix. +func functionProbeNames(stems []string, suffix string) []string { + names := make([]string, len(stems)) + for i, stem := range stems { + names[i] = stem + "." + suffix + } + return names +} + +// Branch wildcard detection (the *.git.example.com case). +// +// A wildcard can live at ANY depth, not only at the apex. Junk fleets cluster: +// hundreds of candidates share "git." as their second label. Group the +// candidate list by intermediate suffix and probe the clusters that are large +// enough to smell like wildcard junk, but small enough to bound the DNS +// traffic. + +const ( + branchMinCandidates = 5 // junk fleets are large; 5 clustered labels justify up to ~15 DNS queries + branchMaxProbes = 8 // hard cap of branch probes per detection call + branchMaxDepth = 3 // examine candidate suffixes formed by dropping up to 3 leading labels +) + +// candidateBranchSuffixCounts counts candidates per intermediate suffix. A +// suffix qualifies structurally when it is strictly deeper than the apex +// (matched on label boundaries) — "com.au" alone is never probed. +func candidateBranchSuffixCounts(candidates []string, apex string) map[string]int { + apex = normalizeSubdomainName(apex) + apexLabels := strings.Split(apex, ".") + counts := map[string]int{} + for _, cand := range candidates { + name := normalizeSubdomainName(cand) + if name == "" || !strings.HasSuffix(name, "."+apex) { + continue + } + for drop := 1; drop <= branchMaxDepth; drop++ { + labels := strings.Split(name, ".") + if len(labels)-drop <= len(apexLabels) { + break + } + suffix := strings.Join(labels[drop:], ".") + counts[suffix]++ + } + } + return counts +} + +// detectBranchWildcards probes the largest clustered candidate suffixes and +// returns the confirmed branch wildcard zones (never nil; empty when nothing +// was probed or confirmed). maxBranches bounds the DNS traffic per call — the +// apex scan uses the full budget, recursion levels a smaller one. +func detectBranchWildcards(ctx context.Context, apex string, candidates []string, r *net.Resolver, servers []string, maxBranches int, logFn LogFunc) []wildcardZone { + if maxBranches <= 0 { + maxBranches = branchMaxProbes + } + counts := candidateBranchSuffixCounts(candidates, apex) + + type branchCandidate struct { + suffix string + count int + } + sorted := make([]branchCandidate, 0, len(counts)) + for suffix, count := range counts { + if count >= branchMinCandidates { + sorted = append(sorted, branchCandidate{suffix: suffix, count: count}) + } + } + if len(sorted) == 0 { + return nil + } + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].count != sorted[j].count { + return sorted[i].count > sorted[j].count + } + return sorted[i].suffix < sorted[j].suffix + }) + if len(sorted) > maxBranches { + sorted = sorted[:maxBranches] + } + + var confirmed []wildcardZone + probed := 0 + for _, bc := range sorted { + if ctx.Err() != nil { + break + } + probed++ + zone := probeWildcardZone(bc.suffix, bc.suffix, r, servers, logFn) + if zone.hasEvidence() { + confirmed = append(confirmed, zone) + } + } + if probed > 0 && logFn != nil { + logFn("info", "subdomain_enum", fmt.Sprintf( + "Branch wildcard probes: %d clustered suffix(es) probed, %d confirmed as wildcards (candidates clustered e.g. *.git.%s)", + probed, len(confirmed), apex)) + for _, zone := range confirmed { + logFn("warn", "subdomain_enum", fmt.Sprintf( + "Wildcard branch confirmed: *.%s (%d answer IP(s), %d CNAME target(s)) — names under it need DNS proof", + zone.suffix, len(zone.ips), len(zone.cnames))) + } + } + return confirmed +} + +// detectWildcard builds the full wildcard profile for a scan: the apex zone +// (probes under the target domain) with branch zones filled in separately by +// detectBranchWildcards once the candidate list is known. +func detectWildcard(domain string, r *net.Resolver, servers []string, logFn LogFunc) *wildcardProfile { + profile := &wildcardProfile{ + apex: probeWildcardZone(domain, "", r, servers, logFn), + } + if profile.apex.hasEvidence() { + logFn("warn", "subdomain_enum", fmt.Sprintf( + "Wildcard DNS detected (%d answer IP(s), %d CNAME target(s)) — passive results will be verified by resolution", + len(profile.apex.ips), len(profile.apex.cnames))) + } + return profile +} + +// detectBranches fills a profile with branch zones probed from the candidate +// list. It mutates the profile in place so the caller keeps one reference. +func (p *wildcardProfile) detectBranches(ctx context.Context, apex string, candidates []string, r *net.Resolver, servers []string, maxBranches int, logFn LogFunc) { + if p == nil || len(candidates) == 0 { + return + } + p.branches = detectBranchWildcards(ctx, apex, candidates, r, servers, maxBranches, logFn) +} diff --git a/internal/scanner/wildcard_test.go b/internal/scanner/wildcard_test.go new file mode 100644 index 0000000..7f9fee0 --- /dev/null +++ b/internal/scanner/wildcard_test.go @@ -0,0 +1,217 @@ +package scanner + +import ( + "net" + "sort" + "testing" +) + +func mustIP(t *testing.T, raw string) net.IPAddr { + t.Helper() + ip := net.ParseIP(raw) + if ip == nil { + t.Fatalf("bad test IP %q", raw) + } + return net.IPAddr{IP: ip} +} + +func TestCandidateBranchSuffixCounts(t *testing.T) { + apex := "realestate.com.au" + var candidates []string + for i := 0; i < 6; i++ { + candidates = append(candidates, + "junk"+string(rune('a'+i))+".git.realestate.com.au") + } + // Nested junk below a deeper label: counts both branch suffixes. + candidates = append(candidates, "word.es.git.realestate.com.au", "word2.es.git.realestate.com.au") + // Unrelated legitimate names must not create branch suffixes. + candidates = append(candidates, "mail.realestate.com.au", "api.realestate.com.au") + // The branch parent itself must not count (it equals the suffix, not a name under it). + candidates = append(candidates, "git.realestate.com.au") + candidates = append(candidates, "wikileaks.not-the-apex.org", "junk.evil-realestate.com.au") + + counts := candidateBranchSuffixCounts(candidates, apex) + if counts["git.realestate.com.au"] != 8 { + t.Fatalf("expected 8 candidates under git.realestate.com.au (nested es.git names counted too), got %d (%v)", counts["git.realestate.com.au"], counts) + } + if counts["es.git.realestate.com.au"] != 2 { + t.Fatalf("expected 2 candidates under es.git.realestate.com.au, got %d", counts["es.git.realestate.com.au"]) + } + for _, forbidden := range []string{"com.au", "realestate.com.au", "not-the-apex.org"} { + if counts[forbidden] != 0 { + t.Fatalf("suffix %q must never be counted as a branch of the apex, got %d", forbidden, counts[forbidden]) + } + } +} + +func TestWildcardProfileMatching(t *testing.T) { + profile := &wildcardProfile{ + apex: wildcardZone{ + ips: map[string]bool{"1.1.1.1": true}, + cnames: map[string]bool{ + "shared-target.realestate.com.au": true, + }, + }, + branches: []wildcardZone{ + {suffix: "git.realestate.com.au", ips: map[string]bool{"2.2.2.2": true}, cnames: map[string]bool{"git-target.realestate.com.au": true}}, + {suffix: "es.git.realestate.com.au", ips: map[string]bool{"3.3.3.3": true}}, + }, + } + + if got := profile.firstNonWildcardIP("mail.realestate.com.au", []string{"1.1.1.1"}); got != "" { + t.Fatalf("apex wildcard IP must reject, got %q", got) + } + if got := profile.firstNonWildcardIP("x.es.git.realestate.com.au", []string{"3.3.3.3"}); got != "" { + t.Fatalf("deeper branch IP must reject names under the deeper zone, got %q", got) + } + if got := profile.firstNonWildcardIP("x.git.realestate.com.au", []string{"3.3.3.3"}); got != "3.3.3.3" { + t.Fatalf("deeper-zone IP must NOT reject names outside the deeper zone, got %q", got) + } + if got := profile.firstNonWildcardIP("x.git.realestate.com.au", []string{"2.2.2.2"}); got != "" { + t.Fatalf("branch wildcard IP must reject, got %q", got) + } + if got := profile.firstNonWildcardIP("x.other.realestate.com.au", []string{"2.2.2.2"}); got != "2.2.2.2" { + t.Fatalf("branch IP must NOT reject names outside the branch, got %q", got) + } + // Label-boundary safety: "evil-git" is one label — not under *.git. + if got := profile.firstNonWildcardIP("asdf-garbage.git.realestate.com.au", []string{"9.9.9.9"}); got != "9.9.9.9" { + t.Fatalf("unrelated IP under a wildcard branch must still admit (needs HTTP proof), got %q", got) + } + if got := profile.firstNonWildcardIP("evil-git.realestate.com.au", []string{"2.2.2.2"}); got != "2.2.2.2" { + t.Fatalf("hyphenated label must not match the branch suffix, got %q (rejected would mean a false wildcard hit)", got) + } + + if profile.isWildcardCNAME("x.realestate.com.au", "shared-Target.Realestate.com.au") != true { + t.Fatalf("apex CNAME target must match case-insensitively") + } + if profile.isWildcardCNAME("x.git.realestate.com.au", "git-target.realestate.com.au") != true { + t.Fatalf("branch CNAME target must match for names under the branch") + } + if profile.isWildcardCNAME("x.other.realestate.com.au", "git-target.realestate.com.au") != false { + t.Fatalf("branch CNAME target must NOT match for names outside the branch") + } + if profile.isWildcardCNAME("x.realestate.com.au", "") != false { + t.Fatalf("empty target must never be a wildcard hit") + } + + if !profile.apexWildcard() { + t.Fatalf("apex evidence must mark the profile as apex-wildcard") + } + if !profile.hasAnyWildcard() { + t.Fatalf("branch evidence must keep the profile flagged") + } +} + +func TestWildcardProfileNilAndBoundary(t *testing.T) { + var nilProfile *wildcardProfile + if got := nilProfile.firstNonWildcardIP("a.realestate.com.au", []string{"1.1.1.1"}); got != "1.1.1.1" { + t.Fatalf("nil profile must admit, got %q", got) + } + if nilProfile.overlapsWildcardIPs("a.realestate.com.au", []string{"1.1.1.1"}) { + t.Fatalf("nil profile must never report overlaps") + } + if nilProfile.apexWildcard() || nilProfile.hasAnyWildcard() { + t.Fatalf("nil profile must not claim evidence") + } + + empty := &wildcardProfile{} + if got := empty.firstNonWildcardIP("a.realestate.com.au", []string{"1.1.1.1", "2.2.2.2"}); got != "1.1.1.1" { + t.Fatalf("empty profile must admit first IP, got %q", got) + } +} + +func TestDecideWildcardZone(t *testing.T) { + pool := map[string]bool{"10.0.0.1": true, "10.0.0.2": true, "10.0.0.3": true, "10.0.0.4": true} + cnames := map[string]int{"edge.realestate.com.au": 4} + + zone, needsMore := decideWildcardZone(3, 0, pool, cnames, "git.realestate.com.au") + if needsMore { + t.Fatalf("3 resolving probes must settle the zone") + } + if len(zone.ips) != 4 { + t.Fatalf("rotating pool: all probe IPs belong to the wildcard pool, got %v", zone.ips) + } + if len(zone.cnames) != 1 { + t.Fatalf("4 aggressive CNAME agreements must produce the CNAME pool, got %v", zone.cnames) + } + + zone, needsMore = decideWildcardZone(0, 3, pool, cnames, "git.realestate.com.au") + if needsMore || zone.hasEvidence() { + t.Fatalf("3 definitive NXDOMAINs must settle an empty zone (needsMore=%v evidence=%v)", needsMore, zone.hasEvidence()) + } + + _, needsMore = decideWildcardZone(2, 0, pool, cnames, "git.realestate.com.au") + if !needsMore { + t.Fatalf("below agreement threshold must request a second batch") + } + + _, needsMore = decideWildcardZone(2, 1, map[string]bool{}, map[string]int{}, "git.realestate.com.au") + if !needsMore { + t.Fatalf("2 resolved + 1 NX is still inconclusive") + } + + zone, needsMore = decideWildcardZone(1, 3, pool, cnames, "git.realestate.com.au") + if needsMore { + t.Fatalf("3 NX dominate a single resolve — that resolver disagrees, majority NX wins") + } + if zone.hasEvidence() { + t.Fatalf("majority-NX verdict must not create an answer pool") + } +} + +func TestSummarizeWildcardProbes(t *testing.T) { + probes := []wildcardProbe{ + {verdict: probeResolved, ips: []net.IPAddr{mustIP(t, "10.1.1.1"), mustIP(t, "10.1.1.2")}, cname: "a.realestate.com.au"}, + {verdict: probeResolved, ips: []net.IPAddr{mustIP(t, "10.1.1.1")}}, + {verdict: probeNX}, + {verdict: probeNX}, + {verdict: probeNX}, + {verdict: probeAmbiguous, cname: "b.realestate.com.au"}, + } + resolved, nx, ambiguous, allIPs, cnameCounts := summarizeWildcardProbes(probes) + if resolved != 2 || nx != 3 || ambiguous != 1 { + t.Fatalf("verdict histogram mismatch: resolved=%d nx=%d ambiguous=%d", resolved, nx, ambiguous) + } + if len(allIPs) != 2 || !allIPs["10.1.1.1"] || !allIPs["10.1.1.2"] { + t.Fatalf("IP union mismatch: %v", allIPs) + } + if cnameCounts["a.realestate.com.au"] != 1 || cnameCounts["b.realestate.com.au"] != 1 { + t.Fatalf("CNAME counts mismatch: %v", cnameCounts) + } +} + +func TestDetectBranchWildcardsOrderAndCap(t *testing.T) { + apex := "realestate.com.au" + var candidates []string + for i := 0; i < 9; i++ { + candidates = append(candidates, "junk"+string(rune('a'+i))+".git.realestate.com.au") + } + counts := candidateBranchSuffixCounts(candidates, apex) + if len(counts) != 1 || counts["git.realestate.com.au"] != 9 { + t.Fatalf("cluster must reduce to one suffix: %v", counts) + } + keys := make([]string, 0, len(counts)) + for k := range counts { + keys = append(keys, k) + } + sort.Strings(keys) + if keys[0] != "git.realestate.com.au" { + t.Fatalf("unexpected suffix: %v", keys) + } +} + +func TestResolverServerAddress(t *testing.T) { + cases := map[string]string{ + "1.1.1.1": "1.1.1.1:53", + "8.8.8.8:5353": "8.8.8.8:5353", + "2606:4700:4700::1111": "[2606:4700:4700::1111]:53", + "[2606:4700:4700::1111]": "[2606:4700:4700::1111]:53", + "[2606:4700:4700::1111]:53": "[2606:4700:4700::1111]:53", + " ": "", + } + for in, want := range cases { + if got := resolverServerAddress(in); got != want { + t.Fatalf("resolverServerAddress(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/scheduler/monitor_cancel_test.go b/internal/scheduler/monitor_cancel_test.go new file mode 100644 index 0000000..79a2278 --- /dev/null +++ b/internal/scheduler/monitor_cancel_test.go @@ -0,0 +1,88 @@ +package scheduler + +import ( + "context" + "database/sql" + "testing" +) + +// Finalizing an operator-cancelled monitor watch pass must count it as an +// attempt for the due scheduler. runDueMonitors used to re-create cancelled +// watch passes because monitor_last_run never moved — the "cancel doesn't +// work, it re-probes and scans again" loop. +func TestCancelledMonitorWatchCountsAsRunForDueScheduler(t *testing.T) { + s := newTestScheduler(t) + if _, err := s.db.Exec(`INSERT INTO targets(id,domain,monitor_enabled,monitor_interval_hours) + VALUES('mc-target','example.com',1,1)`); err != nil { + t.Fatal(err) + } + // Empty-module watch fixture keeps executeTask at the finalization path. + if _, err := s.db.Exec(`INSERT INTO tasks(id,target_id,type,status,modules,total) + VALUES('mc-cancel-watch','mc-target',?,'pending','[]',0)`, monitorWatchType); err != nil { + t.Fatal(err) + } + + // The operator cancels the pass. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cancel() + s.executeTask(ctx, "mc-cancel-watch") + + var taskStatus string + if err := s.db.QueryRow(`SELECT status FROM tasks WHERE id='mc-cancel-watch'`).Scan(&taskStatus); err != nil { + t.Fatal(err) + } + if taskStatus != "cancelled" { + t.Fatalf("watch task status = %q, want cancelled", taskStatus) + } + var last sql.NullTime + if err := s.db.QueryRow(`SELECT monitor_last_run FROM targets WHERE id='mc-target'`).Scan(&last); err != nil { + t.Fatal(err) + } + if !last.Valid { + t.Fatal("cancelled watch pass did not stamp monitor_last_run") + } + + // Expire the 30-minute creation guard, then re-run the due check so only + // the stamped monitor_last_run can hold the scheduler back. + if _, err := s.db.Exec(`UPDATE tasks SET created_at=datetime('now','-31 minutes') WHERE id='mc-cancel-watch'`); err != nil { + t.Fatal(err) + } + s.runDueMonitors() + if got := countTasks(t, s, monitorWatchType); got != 1 { + t.Fatalf("due scheduler re-created the cancelled watch pass: %d watch task(s)", got) + } +} + +// Graceful-shutdown interruptions are owned by auto-resume and must NOT be +// counted as a monitor attempt. +func TestInterruptedMonitorWatchDoesNotCountAsRun(t *testing.T) { + s := newTestScheduler(t) + if _, err := s.db.Exec(`INSERT INTO targets(id,domain,monitor_enabled,monitor_interval_hours) + VALUES('mc-interrupted','example.com',1,1)`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`INSERT INTO tasks(id,target_id,type,status,modules,total) + VALUES('mc-interrupted-watch','mc-interrupted',?,'interrupted','[]',0)`, monitorWatchType); err != nil { + t.Fatal(err) + } + + // Context cancellation alone must not override the parked 'interrupted' row. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cancel() + s.executeTask(ctx, "mc-interrupted-watch") + + var last sql.NullTime + if err := s.db.QueryRow(`SELECT monitor_last_run FROM targets WHERE id='mc-interrupted'`).Scan(&last); err != nil { + t.Fatal(err) + } + if last.Valid { + t.Fatal("interrupted watch pass was counted as a monitor attempt; auto-resume would lose its restart window") + } + + s.runDueMonitors() + if got := countTasks(t, s, monitorWatchType); got != 1 { + t.Fatalf("due scheduler must keep the interrupted watch pass guarded: %d watch task(s)", got) + } +} diff --git a/internal/scheduler/monitor_escalation_test.go b/internal/scheduler/monitor_escalation_test.go index 26603a6..840af91 100644 --- a/internal/scheduler/monitor_escalation_test.go +++ b/internal/scheduler/monitor_escalation_test.go @@ -130,3 +130,43 @@ func indexOf(hay, needle string) int { } return -1 } + +// A stylesheet redeploy churns css_change / security:stylesheet_* rows, but +// those signals are observational: they must not escalate to nuclei / +// backup discovery the way a changed JS seed or a new host does. +func TestStylesheetOnlyChangesDoNotEscalate(t *testing.T) { + s := newTestScheduler(t) + if _, err := s.db.Exec(`INSERT INTO targets (id, domain, subdomain_count) VALUES ('t','ex.com',10)`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`INSERT INTO monitoring_changes (id, target_id, url, change_type, old_value, new_value, severity, detected_at) + VALUES ('mc-css', 't', 'https://ex.com/a.css', 'css_change', 'old', 'new', 'info', CURRENT_TIMESTAMP)`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`INSERT INTO monitoring_changes (id, target_id, url, change_type, old_value, new_value, severity, detected_at) + VALUES ('mc-attr', 't', 'https://ex.com/a.css', 'security:stylesheet_added', 'old', 'new', 'info', CURRENT_TIMESTAMP)`); err != nil { + t.Fatal(err) + } + s.escalateIfChanged("t", "ex.com", 10, time.Now().Add(-time.Minute)) + if n := countTasks(t, s, monitorEscalationType); n != 0 { + t.Fatalf("stylesheet-only changes escalated to %d task(s), want 0", n) + } + + // Mixed signal: the css churn is ignored, a js_change still escalates with + // the observational JS refresh set — and only that set. + if _, err := s.db.Exec(`INSERT INTO monitoring_changes (id, target_id, url, change_type, old_value, new_value, severity, detected_at) + VALUES ('mc-js', 't', 'https://ex.com/app.js', 'js_change', 'old', 'new', 'medium', CURRENT_TIMESTAMP)`); err != nil { + t.Fatal(err) + } + s.escalateIfChanged("t", "ex.com", 10, time.Now().Add(-time.Minute)) + if n := countTasks(t, s, monitorEscalationType); n != 1 { + t.Fatalf("js_change beside css churn did not escalate (%d task(s)), want 1", n) + } + var modules string + if err := s.db.QueryRow(`SELECT modules FROM tasks WHERE type=? LIMIT 1`, monitorEscalationType).Scan(&modules); err != nil { + t.Fatal(err) + } + if !contains(modules, ModuleJSAnalysis) { + t.Fatalf("escalation modules %q missing %q", modules, ModuleJSAnalysis) + } +} diff --git a/internal/scheduler/monitor_modules_config_test.go b/internal/scheduler/monitor_modules_config_test.go new file mode 100644 index 0000000..6eaf9da --- /dev/null +++ b/internal/scheduler/monitor_modules_config_test.go @@ -0,0 +1,65 @@ +package scheduler + +import ( + "slices" + "testing" + + "github.com/recon-platform/internal/config" +) + +// The System → Scanning page replaces the historical hardcoded monitor watch +// set. Regressions here surface as silently wrong check coverage: unknown +// names must be skipped (a stale config must not poison a future upgrade), +// the snapshot module must land before the first refresh-capable module — +// but AFTER subdomain_enum, so newly-appearing names stay inside the diffed +// baseline — and an empty/invalid list must fall back to the built-in set. +func TestMonitorWatchModulesHonorsConfiguredSet(t *testing.T) { + s := newTestScheduler(t) + s.cfg = &config.Config{MonitorModules: []string{ModuleSubdomainEnum, "totally_bogus", ModuleCORS}} + got := s.monitorWatchModules() + // Unknown names are skipped (not rejected), user order is preserved, and a + // set without a snapshot module needs no hoisting. + want := []string{ModuleSubdomainEnum, ModuleCORS} + if !slices.Equal(got, want) { + t.Fatalf("configured set not honored: got %v want %v", got, want) + } +} + +func TestMonitorWatchModulesMonitorBeforeRefresh(t *testing.T) { + s := newTestScheduler(t) + s.cfg = &config.Config{MonitorModules: []string{ModuleHTTPProbe, ModuleMonitor, ModuleSubdomainEnum}} + got := s.monitorWatchModules() + want := []string{ModuleMonitor, ModuleHTTPProbe, ModuleSubdomainEnum} + if !slices.Equal(got, want) { + t.Fatalf("monitor not hoisted ahead of the refresh module: got %v want %v", got, want) + } +} + +func TestMonitorWatchModulesMonitorStaysAfterSubdomainEnum(t *testing.T) { + s := newTestScheduler(t) + s.cfg = &config.Config{MonitorModules: []string{ModuleSubdomainEnum, ModuleHTTPProbe, ModuleMonitor, ModuleCORS}} + got := s.monitorWatchModules() + // Monitor sits before http_probe (the first refresh-capable module) but is + // NOT hoisted to the absolute front ahead of subdomain enumeration. + want := []string{ModuleSubdomainEnum, ModuleMonitor, ModuleHTTPProbe, ModuleCORS} + if !slices.Equal(got, want) { + t.Fatalf("snapshot jumped ahead of subdomain_enum: got %v want %v", got, want) + } +} + +func TestMonitorWatchModulesEmptyConfigUsesBuiltInSet(t *testing.T) { + s := newTestScheduler(t) + got := s.monitorWatchModules() + if !slices.Equal(got, DefaultMonitorWatchModules) { + t.Fatalf("empty config must use the built-in watch set: got %v", got) + } +} + +func TestMonitorWatchModulesAllUnknownFallsBack(t *testing.T) { + s := newTestScheduler(t) + s.cfg = &config.Config{MonitorModules: []string{"totally_bogus", "also_bogus"}} + got := s.monitorWatchModules() + if !slices.Equal(got, DefaultMonitorWatchModules) { + t.Fatalf("all-unknown configuration must fall back to the built-in set: got %v", got) + } +} diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 93354fe..6606e03 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -116,6 +116,23 @@ const ( ModuleNetworkBruteSMB = "network_brute_smb" ) +// DefaultMonitorWatchModules is the built-in pipeline for the automatic +// periodic monitor task (runDueMonitors). Declared here because +// monitorWatchModules and its tests need the exact tokens; the order mirrors +// the historical hardcoded list in runDueMonitors. The snapshot module is +// hoisted ahead of refresh-capable modules (http_probe/js_analysis) +// server-side — "monitor" must run before those refresh, or the diff-first +// baseline would compare against already-refreshed state and real changes +// would become false negatives. +var DefaultMonitorWatchModules = []string{ + ModuleSubdomainEnum, + ModuleMonitor, + ModuleHTTPProbe, + ModuleTakeover, + ModuleCORS, + ModuleBackupDiscovery, +} + var AllModules = []string{ ModuleSubdomainEnum, ModuleHTTPProbe, @@ -1613,9 +1630,14 @@ func (s *Scheduler) executeTask(parentCtx context.Context, taskID string) { var modules []string // Honor per-scan behavior options and build the executable phase list. speed := scanner.SpeedNormal - subBrute := true // slow permutation/brute phase of subdomain enum (default on) - asnDiscovery := false // explicit opt-in only after program-scope/WHOIS verification - subRecursion := false // recursive subdomain enum (opt-in: multiplies DNS volume) + subBrute := true // slow permutation/brute phase of subdomain enum (default on) + asnDiscovery := false // explicit opt-in only after program-scope/WHOIS verification + subRecursion := false // recursive subdomain enum (opt-in: multiplies DNS volume) + if s.cfg != nil { + // The System → Scanning toggle is the deployment default; the per-scan + // "recursive_subdomain" token can still opt in when it is off. + subRecursion = s.cfg.SubdomainRecursionEnabled + } singleEndpoint := false // confine the whole scan to the seed URL(s) and paths under them wpCredAuthorized := false // operator-confirmed authorization for the active WordPress credential audit netCredAuthorized := false // operator-confirmed authorization for the active network credential audit @@ -2078,7 +2100,14 @@ func (s *Scheduler) executeTask(parentCtx context.Context, taskID string) { `, finalStatus, finalError, finalProgress, taskID) } - if taskType == monitorWatchType && finalStatus == "finished" { + // A monitor watch pass counts as an attempt for due scheduling when it + // finished OR when the operator cancelled it. A cancelled pass used to leave + // monitor_last_run stale, so the 1-minute due scheduler kept re-creating the + // very scan the operator just stopped: cancel -> <=30 min guard -> re-run -> + // cancel again, forever. Graceful-shutdown interruptions are NOT counted — + // they keep status 'interrupted' and their auto-resume restart window. + // Watchdog timeouts land in 'failed' and keep the short failure backoff. + if taskType == monitorWatchType && (finalStatus == "finished" || finalStatus == "cancelled") { _, _ = s.db.Exec(`UPDATE targets SET monitor_last_run=CURRENT_TIMESTAMP WHERE id=?`, targetID) } terminalTargetStatus := "idle" @@ -2310,7 +2339,13 @@ const ( func (s *Scheduler) escalateIfChanged(targetID, domain string, baselineSubs int, since time.Time) { var nowSubs, changes int _ = s.db.QueryRow(`SELECT COALESCE(subdomain_count,0) FROM targets WHERE id=?`, targetID).Scan(&nowSubs) - _ = s.db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND detected_at >= ?`, + // Stylesheet signals are observational only: there is no follow-up CSS + // module (unlike the JS refresh set below), so a css_change row or an + // informative security:stylesheet_* row must never fire a nuclei/ + // backup-discovery escalation just because a site redeployed its CSS. + // The Changes tab still records and diffs all of it. + _ = s.db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND detected_at >= ? + AND change_type != 'css_change' AND change_type NOT LIKE 'security:stylesheet%'`, targetID, since.UTC().Format("2006-01-02 15:04:05")).Scan(&changes) newSubs := nowSubs - baselineSubs @@ -2339,6 +2374,11 @@ func (s *Scheduler) escalateIfChanged(targetID, domain string, baselineSubs int, for rows.Next() { var typ string if rows.Scan(&typ) == nil { + // Stylesheet signals never choose escalation modules — see the + // count query above for the reasoning. + if typ == "css_change" || strings.HasPrefix(typ, "security:stylesheet") { + continue + } types[typ] = true } } @@ -2692,9 +2732,9 @@ func (s *Scheduler) unresolvedTaskPhaseCount(taskID string) int { func (s *Scheduler) updateTargetStats(targetID string) { var subdomainCount, aliveCount, findingCount int - s.db.QueryRow("SELECT COUNT(*) FROM subdomains WHERE target_id = ?", targetID).Scan(&subdomainCount) + s.db.QueryRow("SELECT COUNT(*) FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0)=0", targetID).Scan(&subdomainCount) s.db.QueryRow(`SELECT - (SELECT COUNT(*) FROM subdomains WHERE target_id=? AND is_alive=1) + + (SELECT COUNT(*) FROM subdomains WHERE target_id=? AND is_alive=1 AND COALESCE(catchall,0)=0) + (SELECT COUNT(*) FROM network_hosts WHERE target_id=? AND is_alive=1)`, targetID, targetID).Scan(&aliveCount) // The orange headline is an actionable vulnerability count, not an inventory // counter. Keep only medium+ findings; exposed-file discovery is promoted into @@ -2930,8 +2970,8 @@ func (s *Scheduler) monitorMemory() { func (s *Scheduler) notifyScanDone(taskID, targetID, domain, status string, startedAt time.Time) { duration := time.Since(startedAt) var subdomains, alive, vulns, nuclei, backups, redirects, jsFindings int - s.db.QueryRow("SELECT COUNT(*) FROM subdomains WHERE target_id=?", targetID).Scan(&subdomains) - s.db.QueryRow("SELECT COUNT(*) FROM subdomains WHERE target_id=? AND is_alive=1", targetID).Scan(&alive) + s.db.QueryRow("SELECT COUNT(*) FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0", targetID).Scan(&subdomains) + s.db.QueryRow("SELECT COUNT(*) FROM subdomains WHERE target_id=? AND is_alive=1 AND COALESCE(catchall,0)=0", targetID).Scan(&alive) s.db.QueryRow("SELECT COUNT(*) FROM vuln_findings WHERE target_id=? AND COALESCE(status,'finding')='finding' AND COALESCE(triage,'')<>'false_positive'", targetID).Scan(&vulns) s.db.QueryRow("SELECT COUNT(*) FROM nuclei_findings WHERE target_id=? AND verification='verified'", targetID).Scan(&nuclei) s.db.QueryRow("SELECT COUNT(*) FROM backup_findings WHERE target_id=?", targetID).Scan(&backups) @@ -3000,6 +3040,78 @@ func (s *Scheduler) bountyCatalogScheduler() { } } +// monitorWatchModules resolves the pipeline for the automatic periodic monitor +// task. A non-empty, validated cfg.MonitorModules (System → Scanning page) +// replaces the built-in watch set; unknown names are logged and skipped so a +// stale config can't poison future upgrades (or hide a typo silently). If +// nothing valid remains — list empty after filtering, or empty/unset config — +// the historical built-in set applies. The snapshot module ("monitor") is +// ALWAYS hoisted to position 0 server-side: baseline-before-refresh is an +// invariant of the diff-first watch pipeline, not a UI arrangement detail. +func (s *Scheduler) monitorWatchModules() []string { + cfg := s.cfg + suggested := DefaultMonitorWatchModules + if cfg != nil && len(cfg.MonitorModules) > 0 { + suggested = cfg.MonitorModules + } + valid := map[string]bool{} + for _, m := range AllModules { + valid[m] = true + } + seen := map[string]bool{} + var picked []string + for _, m := range suggested { + if valid[m] && !seen[m] { + seen[m] = true + picked = append(picked, m) + continue + } + if cfg != nil && len(cfg.MonitorModules) > 0 { + s.logger.Warn("Ignoring unknown monitor module in configured watch set", + "module", m, + "hint", "valid names: scheduler AllModules (System → Scanning page uses fixed checkboxes)") + } + } + if len(picked) == 0 { + picked = DefaultMonitorWatchModules + } + // Baseline-before-refresh invariant: the snapshot module must run BEFORE + // the first refresh-capable module, but stays AFTER subdomain_enum so + // newly-appearing names land inside the diffed baseline. + refreshModule := map[string]bool{ModuleHTTPProbe: true, ModuleJSAnalysis: true} + monitorIdx := -1 + for i, m := range picked { + if m == ModuleMonitor { + monitorIdx = i + break + } + } + if monitorIdx != -1 { + for i, m := range picked { + // A refresh-capable module sits AHEAD of the snapshot → invariant + // violated: hoist the snapshot to sit directly before it. + if refreshModule[m] && i < monitorIdx { + // Move the snapshot to sit directly before this refresh module. + without := append(picked[:monitorIdx:monitorIdx], picked[monitorIdx+1:]...) + ins := len(without) + for j, m := range without { + if refreshModule[m] { + ins = j + break + } + } + adjusted := make([]string, 0, len(without)+1) + adjusted = append(adjusted, without[:ins]...) + adjusted = append(adjusted, ModuleMonitor) + adjusted = append(adjusted, without[ins:]...) + picked = adjusted + break + } + } + } + return picked +} + func (s *Scheduler) runDueMonitors() { rows, err := s.db.Query(` SELECT id, domain FROM targets @@ -3040,20 +3152,10 @@ func (s *Scheduler) runDueMonitors() { // backup/config-file discovery, then the change detector. nuclei + heavy // dir discovery stay OFF this cheap cycle — they run only via // escalateIfChanged when this pass actually turns something up. - monitorModules := []string{ - ModuleSubdomainEnum, - // Snapshot BEFORE refresh. Otherwise http_probe/js_analysis overwrite - // the old status/title/hash and real changes become false negatives. - ModuleMonitor, - ModuleHTTPProbe, - ModuleTakeover, - // CORS is cheap (a few header probes per endpoint) and its config - // flips on redeploys, so it's worth re-checking on every watch cycle. - ModuleCORS, - // Backup/config-file discovery every cycle — a freshly-leaked .env or - // db.sql.bak is exactly the kind of thing a periodic watch should catch. - ModuleBackupDiscovery, - } + // The operator-configurable watch pipeline: snapshot ("monitor") is + // always first (server-enforced), then refresh + checks in set order. + // Empty/invalid config ⇒ the built-in historical set. + monitorModules := s.monitorWatchModules() if _, err := s.CreateTaskTyped(id, monitorModules, 3, monitorWatchType); err != nil { s.logger.Error("Failed to create watch task", "target", domain, "error", err) continue