From 8e45944a6e507b7210d7d8a1a882a49e59f974f5 Mon Sep 17 00:00:00 2001 From: void Date: Thu, 22 Oct 2026 22:26:08 +0330 Subject: [PATCH 01/14] fix(ui): keep TopBar dropdown panels above page content --- frontend/src/components/layout/TopBar.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/layout/TopBar.tsx b/frontend/src/components/layout/TopBar.tsx index d94eeb5..9af8665 100644 --- a/frontend/src/components/layout/TopBar.tsx +++ b/frontend/src/components/layout/TopBar.tsx @@ -182,7 +182,7 @@ export const TopBar = () => { return ( <> -
+
{/* Breadcrumbs */}
- setScanDraft(d => d && ({ ...d, depth: parseInt(e.target.value, 10) || 1 }))} - className="input font-mono text-xs mt-1 w-28" /> -

After the apex, each resolving subdomain is itself enumerated, down to this many levels (default 4). 1 ⇒ apex only.

+
+ setScanDraft(d => d && ({ ...d, depth: parseInt(e.target.value, 10) || 1 }))} + className="input font-mono text-xs w-28" /> + +
+

After the apex, each resolving subdomain is itself enumerated, down to this many levels (default 4). Off or depth 1 ⇒ apex only.

diff --git a/internal/api/system_handlers.go b/internal/api/system_handlers.go index 92a6e5c..d237d2e 100644 --- a/internal/api/system_handlers.go +++ b/internal/api/system_handlers.go @@ -98,10 +98,11 @@ func (h *Handler) handleUpdateSettings(w http.ResponseWriter, r *http.Request) { // reflecting over the whole Config) means only these deliberately-safe tunables // are reachable from the API — never secrets, paths, or auth settings. type scanConfigPayload struct { - DNSResolvers []string `json:"dns_resolvers"` - SubdomainRecursionDepth int `json:"subdomain_recursion_depth"` - NucleiExcludeTemplateIDs []string `json:"nuclei_exclude_template_ids"` - NucleiExtraTemplateRepos []string `json:"nuclei_extra_template_repos"` + DNSResolvers []string `json:"dns_resolvers"` + SubdomainRecursionDepth int `json:"subdomain_recursion_depth"` + SubdomainRecursionEnabled bool `json:"subdomain_recursion_enabled"` + NucleiExcludeTemplateIDs []string `json:"nuclei_exclude_template_ids"` + NucleiExtraTemplateRepos []string `json:"nuclei_extra_template_repos"` // DefaultDNSResolvers / DefaultNucleiRepos are read-only hints so the UI can // show what the built-in defaults are when the operator has set nothing. DefaultDNSResolvers []string `json:"default_dns_resolvers"` @@ -115,12 +116,13 @@ type scanConfigPayload struct { // and resolvers" surface. func (h *Handler) handleGetScanConfig(w http.ResponseWriter, _ *http.Request) { h.writeSuccess(w, scanConfigPayload{ - DNSResolvers: h.cfg.DNSResolvers, - SubdomainRecursionDepth: h.cfg.SubdomainRecursionDepth, - NucleiExcludeTemplateIDs: h.cfg.NucleiExcludeTemplateIDs, - NucleiExtraTemplateRepos: h.cfg.NucleiExtraTemplateRepos, - DefaultDNSResolvers: config.DefaultDNSResolvers, - DefaultNucleiRepos: scanner.DefaultNucleiExtraRepos(), + DNSResolvers: h.cfg.DNSResolvers, + SubdomainRecursionDepth: h.cfg.SubdomainRecursionDepth, + SubdomainRecursionEnabled: h.cfg.SubdomainRecursionEnabled, + NucleiExcludeTemplateIDs: h.cfg.NucleiExcludeTemplateIDs, + NucleiExtraTemplateRepos: h.cfg.NucleiExtraTemplateRepos, + DefaultDNSResolvers: config.DefaultDNSResolvers, + DefaultNucleiRepos: scanner.DefaultNucleiExtraRepos(), }) } @@ -130,10 +132,11 @@ func (h *Handler) handleGetScanConfig(w http.ResponseWriter, _ *http.Request) { // which third-party template repos are cloned onto the box. func (h *Handler) handleUpdateScanConfig(w http.ResponseWriter, r *http.Request) { var body struct { - DNSResolvers *[]string `json:"dns_resolvers"` - SubdomainRecursionDepth *int `json:"subdomain_recursion_depth"` - NucleiExcludeTemplateIDs *[]string `json:"nuclei_exclude_template_ids"` - NucleiExtraTemplateRepos *[]string `json:"nuclei_extra_template_repos"` + DNSResolvers *[]string `json:"dns_resolvers"` + SubdomainRecursionDepth *int `json:"subdomain_recursion_depth"` + SubdomainRecursionEnabled *bool `json:"subdomain_recursion_enabled"` + NucleiExcludeTemplateIDs *[]string `json:"nuclei_exclude_template_ids"` + NucleiExtraTemplateRepos *[]string `json:"nuclei_extra_template_repos"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { h.writeError(w, http.StatusBadRequest, "invalid body") @@ -164,6 +167,10 @@ func (h *Handler) handleUpdateScanConfig(w http.ResponseWriter, r *http.Request) } h.cfg.SubdomainRecursionDepth = d } + if body.SubdomainRecursionEnabled != nil { + // No clamping — a bool is a bool. Omitted ⇒ unchanged, like the other knobs. + h.cfg.SubdomainRecursionEnabled = *body.SubdomainRecursionEnabled + } if body.NucleiExcludeTemplateIDs != nil { h.cfg.NucleiExcludeTemplateIDs = clean(*body.NucleiExcludeTemplateIDs) } diff --git a/internal/config/config.go b/internal/config/config.go index 3f11ce6..8d4a04e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -220,6 +220,12 @@ type Config struct { // raise it to go deeper on a sprawling target. Bounded per level // (SubdomainRecursionMaxPerLevel) so it can't explode on a huge zone. SubdomainRecursionDepth int `json:"subdomain_recursion_depth"` + // SubdomainRecursionEnabled turns recursive enumeration off entirely without + // losing the remembered depth. Off ⇒ apex only (the old pre-recursion + // behavior); the depth value stays stored for when it is switched back on. + // Default true so an upgrade never silently changes scan behavior. Set via + // the System → Scanning panel, next to the depth field. + SubdomainRecursionEnabled bool `json:"subdomain_recursion_enabled"` // ScanWatchdogHours bounds one scan PHASE so a wedged module can never hold // its concurrency slot forever (the actual purpose — see scheduler.go). It // is deliberately reset after every completed phase, so the combined runtime @@ -505,15 +511,16 @@ func defaultConfig() *Config { // (see the field comments) instead of baking a fixed numeric ceiling into // every config.json ever saved — an operator who wants a REAL cap sets one // explicitly; a fresh install gets the generous built-in default. - ScanWatchdogHours: 24, // per-phase watchdog floor; reset on progress and scaled for large targets - EnableDAST: true, // native context-aware DAST (XSS/SQLi) over all insertion points - AIModel: "claude-opus-4-8", // latest Opus; used only when AIEnabled and a key is set - AIMaxIterations: 40, // hard cap on the agent loop - AuthzMode: "balanced", // two-identity BOLA: deep auth crawl + read/write differential - AuthzDestructive: false, // cross-user DELETE stays opt-in - SubdomainRecursionDepth: 4, // recurse passive+resolve into each resolving subdomain, up to 4 levels - Workers: workers, // auto-scaled to host CPU (floors = previous fixed defaults) - Limits: limits, // auto-scaled to host CPU/RAM + ScanWatchdogHours: 24, // per-phase watchdog floor; reset on progress and scaled for large targets + EnableDAST: true, // native context-aware DAST (XSS/SQLi) over all insertion points + AIModel: "claude-opus-4-8", // latest Opus; used only when AIEnabled and a key is set + AIMaxIterations: 40, // hard cap on the agent loop + AuthzMode: "balanced", // two-identity BOLA: deep auth crawl + read/write differential + AuthzDestructive: false, // cross-user DELETE stays opt-in + SubdomainRecursionEnabled: true, // recursion on by default; the System panel disables it per deployment + SubdomainRecursionDepth: 4, // recurse passive+resolve into each resolving subdomain, up to 4 levels + Workers: workers, // auto-scaled to host CPU (floors = previous fixed defaults) + Limits: limits, // auto-scaled to host CPU/RAM } } diff --git a/internal/scanner/subdomain.go b/internal/scanner/subdomain.go index 499e43b..be04991 100644 --- a/internal/scanner/subdomain.go +++ b/internal/scanner/subdomain.go @@ -369,9 +369,12 @@ func (s *SubdomainScanner) Run(ctx context.Context, targetID, domain string, log // Recursive enumeration: run passive discovery + resolution AGAINST each newly // discovered resolving subdomain as its own sub-apex, descending until the - // configured depth or until a level finds nothing new. Gated on the same deep- - // work toggle as brute/permutation so the fast passive-only scan stays fast. - if subdomainBruteEnabled(ctx) { + // configured depth or until a level finds nothing new. Gate 1 is the deep- + // work toggle that already covers brute/permutation so the fast passive-only + // scan stays fast. Gate 2 is the system-wide recursion switch in the System → + // Scanning panel: off means apex only, while the remembered depth is kept + // for when it is turned back on. + if subdomainBruteEnabled(ctx) && s.subdomainRecursionEnabled() { s.recursiveEnum(ctx, targetID, domain, logFn) } @@ -1308,6 +1311,18 @@ func (s *SubdomainScanner) subdomainRecursionDepth() int { return 4 } +// subdomainRecursionEnabled reports whether recursive enumeration runs at all. +// A nil config (test paths) behaves like the built-in defaults: recursion +// enabled. Explicit false in config.json wins; the production Config always +// comes from Load(), which starts from the defaults, so an old config.json +// without this key never silently disables recursion after an upgrade. +func (s *SubdomainScanner) subdomainRecursionEnabled() bool { + if s.cfg == nil { + return true + } + return s.cfg.SubdomainRecursionEnabled +} + // recursiveEnum drives recursive subdomain discovery: after the apex is // enumerated, every newly discovered RESOLVING subdomain is enumerated as its // own sub-apex (passive sources + the wildcard-aware resolution gate), and the diff --git a/internal/scanner/subdomain_admission_test.go b/internal/scanner/subdomain_admission_test.go index 0ca5eed..8054ab9 100644 --- a/internal/scanner/subdomain_admission_test.go +++ b/internal/scanner/subdomain_admission_test.go @@ -127,3 +127,18 @@ func TestSubdomainRecursionDepthDefault(t *testing.T) { t.Fatalf("nil-cfg recursion depth = %d, want 4", d) } } + +// TestSubdomainRecursionEnabledDefault proves the toggle keeps recursion on for +// a nil config (mirroring the depth default helper) and honors explicit values +// from config.json. +func TestSubdomainRecursionEnabledDefault(t *testing.T) { + if !(&SubdomainScanner{cfg: nil}).subdomainRecursionEnabled() { + t.Fatal("nil-cfg recursion toggle = false, want true (built-in default)") + } + if !(&SubdomainScanner{cfg: &config.Config{SubdomainRecursionEnabled: true}}).subdomainRecursionEnabled() { + t.Fatal("enabled recursion toggle = false, want true") + } + if (&SubdomainScanner{cfg: &config.Config{SubdomainRecursionEnabled: false}}).subdomainRecursionEnabled() { + t.Fatal("disabled recursion toggle = true, want false (apex only)") + } +} From a6d34162d217087797d3e6cb645979817471d150 Mon Sep 17 00:00:00 2001 From: void Date: Fri, 23 Oct 2026 08:51:48 +0330 Subject: [PATCH 05/14] feat(monitor): operator-configurable watch module set (config + UI) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runDueMonitors hardcoded the periodic watch pipeline. It now builds the task from config.monitor_modules (System -> Scanning page, live pickup via the shared in-memory config — no restart). Names are validated against AllModules: unknown tokens are logged and skipped, and an empty/invalid list falls back to the built-in set (subdomain_enum, monitor, http_probe, takeover, cors, backup_discovery) so a stale config can never murder the schedule. Order invariant preserved server-side, not by the form: the snapshot module (monitor) is hoisted ahead of the first refresh-capable module (http_probe/js_analysis) whenever the operator order puts it after, while staying after subdomain_enum so newly-appearing names stay inside the diffed baseline. Legacy default order is unchanged. PATCH /system/scan-config accepts monitor_modules (trimmed/deduped; name validation stays scheduler-side so the API never needs the catalog); GET returns it too. UI: Default monitor modules checkbox card on System -> Scanning with per-module hints, fed by the same save button. Tests: configured set honored + unknown skipped; monitor-before-refresh hoist; monitor stays after subdomain_enum; empty config -> built-in set; all-unknown -> fallback; full scheduler/api/config suites green; frontend builds. --- frontend/src/lib/api.ts | 1 + frontend/src/pages/System.tsx | 37 +++++- internal/api/system_handlers.go | 9 ++ internal/config/config.go | 14 +++ .../scheduler/monitor_modules_config_test.go | 65 +++++++++++ internal/scheduler/scheduler.go | 107 +++++++++++++++--- 6 files changed, 217 insertions(+), 16 deletions(-) create mode 100644 internal/scheduler/monitor_modules_config_test.go diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index edc68f1..a24b76f 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -335,6 +335,7 @@ export interface ScanConfig { subdomain_recursion_enabled: boolean nuclei_exclude_template_ids: string[] nuclei_extra_template_repos: string[] + monitor_modules: string[] default_dns_resolvers: string[] default_nuclei_repos: string[] } diff --git a/frontend/src/pages/System.tsx b/frontend/src/pages/System.tsx index d28ede6..eb815dd 100644 --- a/frontend/src/pages/System.tsx +++ b/frontend/src/pages/System.tsx @@ -12,6 +12,21 @@ import { useUpdateCenter } from '../components/layout/UpdateCenter' interface SystemLogLine { level: string; module: string; message: string; time: string } +// The automatic monitor watch pipeline the operator can retune on the +// System → Scanning page. Order here is display order; server-side the state +// monitor ("monitor") is always hoisted before HTTP/JS refresh when both are +// picked, so the scheduled baseline lands before anything overwrites it. +const DEFAULT_WATCH_MODULES = ['subdomain_enum', 'monitor', 'http_probe', 'takeover', 'cors', 'backup_discovery'] + +const MONITOR_MODULE_OPTIONS: { value: string; label: string; hint: string }[] = [ + { value: 'subdomain_enum', label: 'Subdomain re-enumeration', hint: 'catches newly-appeared names each cycle' }, + { value: 'monitor', label: 'State monitor / snapshot', hint: 'always runs first — snapshots the baseline' }, + { value: 'http_probe', label: 'HTTP probe', hint: 'refreshes live host titles/statuses' }, + { value: 'takeover', label: 'Takeover checks', hint: 'flags dangling DNS records' }, + { value: 'cors', label: 'CORS re-check', hint: 're-tests permissive CORS on live endpoints' }, + { value: 'backup_discovery', label: 'Backup file discovery', hint: 'looks for leaked .env/backup files' }, +] + export default function System() { const [activeTab, setActiveTab] = useState<'overview' | 'integrations' | 'corpora' | 'toolchain' | 'team'>('overview') const [tools, setTools] = useState>({}) @@ -25,7 +40,7 @@ export default function System() { const [drafts, setDrafts] = useState>({}) const [savingKeys, setSavingKeys] = useState(false) const [scanCfg, setScanCfg] = useState(null) - const [scanDraft, setScanDraft] = useState<{ resolvers: string; depth: number; enabled: boolean; excludeIDs: string; repos: string } | null>(null) + const [scanDraft, setScanDraft] = useState<{ resolvers: string; depth: number; enabled: boolean; excludeIDs: string; repos: string; monitor: string[] } | null>(null) const [savingScanCfg, setSavingScanCfg] = useState(false) const { addToast } = useUIStore() const isAdmin = useAuthStore(s => s.user?.role === 'admin') @@ -49,6 +64,8 @@ export default function System() { enabled: sc.subdomain_recursion_enabled !== false, excludeIDs: (sc.nuclei_exclude_template_ids || []).join('\n'), repos: (sc.nuclei_extra_template_repos || []).join('\n'), + // Empty on the wire ⇒ the built-in default watch set applies. + monitor: (sc.monitor_modules && sc.monitor_modules.length ? sc.monitor_modules : DEFAULT_WATCH_MODULES), }) } } @@ -100,6 +117,7 @@ export default function System() { subdomain_recursion_enabled: scanDraft.enabled, nuclei_exclude_template_ids: splitLines(scanDraft.excludeIDs), nuclei_extra_template_repos: splitLines(scanDraft.repos), + monitor_modules: scanDraft.monitor, }) setScanCfg(res) setScanDraft({ @@ -108,8 +126,9 @@ export default function System() { enabled: res.subdomain_recursion_enabled !== false, excludeIDs: (res.nuclei_exclude_template_ids || []).join('\n'), repos: (res.nuclei_extra_template_repos || []).join('\n'), + monitor: (res.monitor_modules && res.monitor_modules.length ? res.monitor_modules : DEFAULT_WATCH_MODULES), }) - addToast('success', 'Scan configuration saved — applies to the next scan') + addToast('success', 'Scan configuration saved — watch modules apply from the next monitor cycle, the rest from the next scan') } catch { addToast('error', 'Failed to save scan configuration') } finally { setSavingScanCfg(false) } @@ -340,6 +359,20 @@ export default function System() { className="input font-mono text-xs mt-1" />

Cloned on top of the official set. Empty ⇒ the two official ProjectDiscovery repos only. Community repos are a known false-positive source — add deliberately.

+
+ +
+ {MONITOR_MODULE_OPTIONS.map(m => ( + + ))} +
+

Modules the automatic monitor schedule runs; applies from its next cycle. The state monitor always snapshots first, before HTTP/JS refresh. Uncheck everything ⇒ the built-in set above applies.

+
)} diff --git a/internal/api/system_handlers.go b/internal/api/system_handlers.go index d237d2e..fada8ee 100644 --- a/internal/api/system_handlers.go +++ b/internal/api/system_handlers.go @@ -103,6 +103,7 @@ type scanConfigPayload struct { SubdomainRecursionEnabled bool `json:"subdomain_recursion_enabled"` NucleiExcludeTemplateIDs []string `json:"nuclei_exclude_template_ids"` NucleiExtraTemplateRepos []string `json:"nuclei_extra_template_repos"` + MonitorModules []string `json:"monitor_modules"` // DefaultDNSResolvers / DefaultNucleiRepos are read-only hints so the UI can // show what the built-in defaults are when the operator has set nothing. DefaultDNSResolvers []string `json:"default_dns_resolvers"` @@ -121,6 +122,7 @@ func (h *Handler) handleGetScanConfig(w http.ResponseWriter, _ *http.Request) { SubdomainRecursionEnabled: h.cfg.SubdomainRecursionEnabled, NucleiExcludeTemplateIDs: h.cfg.NucleiExcludeTemplateIDs, NucleiExtraTemplateRepos: h.cfg.NucleiExtraTemplateRepos, + MonitorModules: h.cfg.MonitorModules, DefaultDNSResolvers: config.DefaultDNSResolvers, DefaultNucleiRepos: scanner.DefaultNucleiExtraRepos(), }) @@ -137,6 +139,7 @@ func (h *Handler) handleUpdateScanConfig(w http.ResponseWriter, r *http.Request) SubdomainRecursionEnabled *bool `json:"subdomain_recursion_enabled"` NucleiExcludeTemplateIDs *[]string `json:"nuclei_exclude_template_ids"` NucleiExtraTemplateRepos *[]string `json:"nuclei_extra_template_repos"` + MonitorModules *[]string `json:"monitor_modules"` } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { h.writeError(w, http.StatusBadRequest, "invalid body") @@ -185,6 +188,12 @@ func (h *Handler) handleUpdateScanConfig(w http.ResponseWriter, r *http.Request) } h.cfg.NucleiExtraTemplateRepos = repos } + if body.MonitorModules != nil { + // Trimmed/deduped here; NAME validation happens in the scheduler + // (monitorWatchModules validates against AllModules and logs unknown + // tokens), so the API/UI never needs to know the module catalog. + h.cfg.MonitorModules = clean(*body.MonitorModules) + } if err := h.cfg.Save(); err != nil { h.writeError(w, http.StatusInternalServerError, "saved in memory but could not persist to config.json: "+err.Error()) return diff --git a/internal/config/config.go b/internal/config/config.go index 8d4a04e..2ee5568 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -212,6 +212,20 @@ type Config struct { // (written to config.json) or the RECON_DNS_RESOLVERS env var (comma-separated, // env wins). Set it to a single internal resolver to force the old behavior. DNSResolvers []string `json:"dns_resolvers"` + // MonitorModules overrides which pipeline modules the automatic periodic + // monitor watch task runs (the scheduled pass creating monitor_watch tasks + // in scheduler.runDueMonitors). Names are validated against the scheduler's + // AllModules catalog by the scheduler; unknown names are logged and skipped + // (never rejected) so neither the API layer nor the UI needs to know the + // catalog. If nothing valid remains, the built-in default watch set applies. + // The snapshot module ("monitor") is always hoisted ahead of refresh-capable + // modules (http_probe/js_analysis) server-side — snapshot-before-refresh is + // an invariant, not a form checkbox. Empty + // ⇒ the built-in watch set (subdomain_enum, monitor, http_probe, takeover, + // cors, backup_discovery). Set via the System → Scanning page (written to + // config.json; scheduler picks the list up on its next due check — no + // restart needed). + MonitorModules []string `json:"monitor_modules"` // SubdomainRecursionDepth controls recursive subdomain enumeration: after the // apex domain is enumerated, each newly-discovered RESOLVING subdomain is itself // enumerated (passive sources + resolution) as a sub-apex, and so on, until this diff --git a/internal/scheduler/monitor_modules_config_test.go b/internal/scheduler/monitor_modules_config_test.go new file mode 100644 index 0000000..6eaf9da --- /dev/null +++ b/internal/scheduler/monitor_modules_config_test.go @@ -0,0 +1,65 @@ +package scheduler + +import ( + "slices" + "testing" + + "github.com/recon-platform/internal/config" +) + +// The System → Scanning page replaces the historical hardcoded monitor watch +// set. Regressions here surface as silently wrong check coverage: unknown +// names must be skipped (a stale config must not poison a future upgrade), +// the snapshot module must land before the first refresh-capable module — +// but AFTER subdomain_enum, so newly-appearing names stay inside the diffed +// baseline — and an empty/invalid list must fall back to the built-in set. +func TestMonitorWatchModulesHonorsConfiguredSet(t *testing.T) { + s := newTestScheduler(t) + s.cfg = &config.Config{MonitorModules: []string{ModuleSubdomainEnum, "totally_bogus", ModuleCORS}} + got := s.monitorWatchModules() + // Unknown names are skipped (not rejected), user order is preserved, and a + // set without a snapshot module needs no hoisting. + want := []string{ModuleSubdomainEnum, ModuleCORS} + if !slices.Equal(got, want) { + t.Fatalf("configured set not honored: got %v want %v", got, want) + } +} + +func TestMonitorWatchModulesMonitorBeforeRefresh(t *testing.T) { + s := newTestScheduler(t) + s.cfg = &config.Config{MonitorModules: []string{ModuleHTTPProbe, ModuleMonitor, ModuleSubdomainEnum}} + got := s.monitorWatchModules() + want := []string{ModuleMonitor, ModuleHTTPProbe, ModuleSubdomainEnum} + if !slices.Equal(got, want) { + t.Fatalf("monitor not hoisted ahead of the refresh module: got %v want %v", got, want) + } +} + +func TestMonitorWatchModulesMonitorStaysAfterSubdomainEnum(t *testing.T) { + s := newTestScheduler(t) + s.cfg = &config.Config{MonitorModules: []string{ModuleSubdomainEnum, ModuleHTTPProbe, ModuleMonitor, ModuleCORS}} + got := s.monitorWatchModules() + // Monitor sits before http_probe (the first refresh-capable module) but is + // NOT hoisted to the absolute front ahead of subdomain enumeration. + want := []string{ModuleSubdomainEnum, ModuleMonitor, ModuleHTTPProbe, ModuleCORS} + if !slices.Equal(got, want) { + t.Fatalf("snapshot jumped ahead of subdomain_enum: got %v want %v", got, want) + } +} + +func TestMonitorWatchModulesEmptyConfigUsesBuiltInSet(t *testing.T) { + s := newTestScheduler(t) + got := s.monitorWatchModules() + if !slices.Equal(got, DefaultMonitorWatchModules) { + t.Fatalf("empty config must use the built-in watch set: got %v", got) + } +} + +func TestMonitorWatchModulesAllUnknownFallsBack(t *testing.T) { + s := newTestScheduler(t) + s.cfg = &config.Config{MonitorModules: []string{"totally_bogus", "also_bogus"}} + got := s.monitorWatchModules() + if !slices.Equal(got, DefaultMonitorWatchModules) { + t.Fatalf("all-unknown configuration must fall back to the built-in set: got %v", got) + } +} diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index f3bcfef..84c7992 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -116,6 +116,23 @@ const ( ModuleNetworkBruteSMB = "network_brute_smb" ) +// DefaultMonitorWatchModules is the built-in pipeline for the automatic +// periodic monitor task (runDueMonitors). Declared here because +// monitorWatchModules and its tests need the exact tokens; the order mirrors +// the historical hardcoded list in runDueMonitors. The snapshot module is +// hoisted ahead of refresh-capable modules (http_probe/js_analysis) +// server-side — "monitor" must run before those refresh, or the diff-first +// baseline would compare against already-refreshed state and real changes +// would become false negatives. +var DefaultMonitorWatchModules = []string{ + ModuleSubdomainEnum, + ModuleMonitor, + ModuleHTTPProbe, + ModuleTakeover, + ModuleCORS, + ModuleBackupDiscovery, +} + var AllModules = []string{ ModuleSubdomainEnum, ModuleHTTPProbe, @@ -3003,6 +3020,78 @@ func (s *Scheduler) bountyCatalogScheduler() { } } +// monitorWatchModules resolves the pipeline for the automatic periodic monitor +// task. A non-empty, validated cfg.MonitorModules (System → Scanning page) +// replaces the built-in watch set; unknown names are logged and skipped so a +// stale config can't poison future upgrades (or hide a typo silently). If +// nothing valid remains — list empty after filtering, or empty/unset config — +// the historical built-in set applies. The snapshot module ("monitor") is +// ALWAYS hoisted to position 0 server-side: baseline-before-refresh is an +// invariant of the diff-first watch pipeline, not a UI arrangement detail. +func (s *Scheduler) monitorWatchModules() []string { + cfg := s.cfg + suggested := DefaultMonitorWatchModules + if cfg != nil && len(cfg.MonitorModules) > 0 { + suggested = cfg.MonitorModules + } + valid := map[string]bool{} + for _, m := range AllModules { + valid[m] = true + } + seen := map[string]bool{} + var picked []string + for _, m := range suggested { + if valid[m] && !seen[m] { + seen[m] = true + picked = append(picked, m) + continue + } + if cfg != nil && len(cfg.MonitorModules) > 0 { + s.logger.Warn("Ignoring unknown monitor module in configured watch set", + "module", m, + "hint", "valid names: scheduler AllModules (System → Scanning page uses fixed checkboxes)") + } + } + if len(picked) == 0 { + picked = DefaultMonitorWatchModules + } + // Baseline-before-refresh invariant: the snapshot module must run BEFORE + // the first refresh-capable module, but stays AFTER subdomain_enum so + // newly-appearing names land inside the diffed baseline. + refreshModule := map[string]bool{ModuleHTTPProbe: true, ModuleJSAnalysis: true} + monitorIdx := -1 + for i, m := range picked { + if m == ModuleMonitor { + monitorIdx = i + break + } + } + if monitorIdx != -1 { + for i, m := range picked { + // A refresh-capable module sits AHEAD of the snapshot → invariant + // violated: hoist the snapshot to sit directly before it. + if refreshModule[m] && i < monitorIdx { + // Move the snapshot to sit directly before this refresh module. + without := append(picked[:monitorIdx:monitorIdx], picked[monitorIdx+1:]...) + ins := len(without) + for j, m := range without { + if refreshModule[m] { + ins = j + break + } + } + adjusted := make([]string, 0, len(without)+1) + adjusted = append(adjusted, without[:ins]...) + adjusted = append(adjusted, ModuleMonitor) + adjusted = append(adjusted, without[ins:]...) + picked = adjusted + break + } + } + } + return picked +} + func (s *Scheduler) runDueMonitors() { rows, err := s.db.Query(` SELECT id, domain FROM targets @@ -3034,20 +3123,10 @@ func (s *Scheduler) runDueMonitors() { // backup/config-file discovery, then the change detector. nuclei + heavy // dir discovery stay OFF this cheap cycle — they run only via // escalateIfChanged when this pass actually turns something up. - monitorModules := []string{ - ModuleSubdomainEnum, - // Snapshot BEFORE refresh. Otherwise http_probe/js_analysis overwrite - // the old status/title/hash and real changes become false negatives. - ModuleMonitor, - ModuleHTTPProbe, - ModuleTakeover, - // CORS is cheap (a few header probes per endpoint) and its config - // flips on redeploys, so it's worth re-checking on every watch cycle. - ModuleCORS, - // Backup/config-file discovery every cycle — a freshly-leaked .env or - // db.sql.bak is exactly the kind of thing a periodic watch should catch. - ModuleBackupDiscovery, - } + // The operator-configurable watch pipeline: snapshot ("monitor") is + // always first (server-enforced), then refresh + checks in set order. + // Empty/invalid config ⇒ the built-in historical set. + monitorModules := s.monitorWatchModules() if _, err := s.CreateTaskTyped(id, monitorModules, 3, monitorWatchType); err != nil { s.logger.Error("Failed to create watch task", "target", domain, "error", err) continue From 6d5da4558b857b39c37e6f1ae5b2332d9132f237 Mon Sep 17 00:00:00 2001 From: void Date: Fri, 23 Oct 2026 09:31:38 +0330 Subject: [PATCH 06/14] feat(monitor): stylesheet content diffing, noise control, changes-tab diff viewer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - css_files table (migration) + first-party and external stylesheet harvest during the monitor pass, with 256 KB body capture on detection and change - css_change rows on confirmed content change (double-fetch verified), old body rolled into old_body; js_change rows now capture bodies the same way - external churn from ?ver= cache-busting no longer creates add/remove pairs: stylesheet attr rows key on path without query, and a query-only bump with unchanged bytes stays silent; a new/removed path still alerts at info severity and no longer raises vuln findings - notification titles: 'Stylesheet content changed', 'External stylesheet added/removed' (was mislabeled as JavaScript changes) - GET /targets/{id}/css-files/body and /js-files/body (url + side params, target-scoped) expose the captured snapshots - Changes tab: filter chips (All/Security/Scripts/Stylesheets/HTTP/Tech/ Headers) with row counts, severity-driven badge colors, and a Diff drawer rendering a line-level +/− diff from the stored captures --- frontend/package-lock.json | 14 +- frontend/package.json | 1 + .../components/targets/ChangeDiffViewer.tsx | 120 +++++++ frontend/src/lib/api.ts | 6 +- frontend/src/pages/TargetDetail.tsx | 77 ++++- frontend/src/types/index.ts | 13 + internal/api/finding_handlers.go | 57 +++- internal/api/router.go | 2 + internal/database/migrations.go | 29 ++ internal/scanner/monitor.go | 220 ++++++++++++- internal/scanner/monitor_css_test.go | 303 ++++++++++++++++++ internal/scanner/monitor_engine.go | 27 +- 12 files changed, 854 insertions(+), 15 deletions(-) create mode 100644 frontend/src/components/targets/ChangeDiffViewer.tsx create mode 100644 internal/scanner/monitor_css_test.go diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c06a218..5b4e1ef 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,13 +1,14 @@ { "name": "reconner-frontend", - "version": "3.4.0", + "version": "4.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "reconner-frontend", - "version": "3.4.0", + "version": "4.7.0", "dependencies": { + "diff": "^9.0.0", "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^7.18.3", @@ -1425,6 +1426,15 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/diff": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-9.0.0.tgz", + "integrity": "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/dlv": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", diff --git a/frontend/package.json b/frontend/package.json index 1f35265..0574889 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,6 +14,7 @@ "test:e2e": "playwright test" }, "dependencies": { + "diff": "^9.0.0", "react": "^18.3.1", "react-dom": "^18.3.1", "react-router-dom": "^7.18.3", diff --git a/frontend/src/components/targets/ChangeDiffViewer.tsx b/frontend/src/components/targets/ChangeDiffViewer.tsx new file mode 100644 index 0000000..4929346 --- /dev/null +++ b/frontend/src/components/targets/ChangeDiffViewer.tsx @@ -0,0 +1,120 @@ +import { useEffect, useMemo, useState } from 'react' +import { Modal, CopyButton } from '../ui' +import { findings as findingsApi } from '../../lib/api' +import { diffLines } from 'diff' +import { cn, truncate } from '../../lib/utils' +import type { FileBodyResponse } from '../../types' + +type Props = { + targetId: string + url: string + kind: 'css' | 'js' + onClose: () => void +} + +// Line-level diff for a monitored stylesheet/script change (css_change / +// js_change rows). Both sides are read from the body captured at detection time +// (monitor.go's css_files/js_files) — never a live re-fetch: a live fetch would +// show today's bytes on both sides and pull a possibly third-party URL through +// a fresh GET on every drawer open. +export function ChangeDiffViewer({ targetId, url, kind, onClose }: Props) { + const [current, setCurrent] = useState(null) + const [previous, setPrevious] = useState(null) + const [error, setError] = useState('') + const [loading, setLoading] = useState(true) + + useEffect(() => { + let alive = true + setLoading(true) + setError('') + setCurrent(null) + setPrevious(null) + Promise.all([ + findingsApi.fileBody(targetId, kind, url, 'current'), + findingsApi.fileBody(targetId, kind, url, 'old'), + ]) + .then(([cur, prev]) => { + if (!alive) return + setCurrent(cur) + setPrevious(prev) + setLoading(false) + }) + .catch(e => { + if (!alive) return + setError(e instanceof Error ? e.message : 'Failed to load stored file bodies') + setLoading(false) + }) + return () => { alive = false } + }, [targetId, kind, url]) + + const hunks = useMemo(() => { + if (!current || !previous) return null + return diffLines(previous.content || '', current.content || '') + }, [current, previous]) + + const added = (hunks || []).reduce((n, h) => n + (h.added ? (h.count || 0) : 0), 0) + const removed = (hunks || []).reduce((n, h) => n + (h.removed ? (h.count || 0) : 0), 0) + // Identical bytes on both sides but different recorded hashes ⇒ the 256 KB + // capture cap cut the tail: the change is real, it just sits past the stored + // content. + const truncated = !!previous?.captured && !!current?.captured && added === 0 && removed === 0 && previous.hash !== current.hash + + return ( + +{added.toLocaleString()}{' / '}−{removed.toLocaleString()} lines}> +

{url}

+ {loading &&

Loading stored snapshots…

} + {!loading && error &&

{error}

} + {!loading && !error && ( +
+ {!previous?.captured || !current?.captured ? ( +

+ {!previous?.captured && !current?.captured + ? 'Neither side was captured — this row predates content tracking. Recorded hashes only.' + : !previous?.captured + ? 'Original content was not captured — this row predates content tracking. Current capture shown read-only.' + : 'Current content was not captured. Stored original shown read-only.'} +

+ ) : truncated ? ( +

+ Both snapshots hold identical bytes but the recorded hashes differ — the change sits past the 256 KB capture cap. +

+ ) : null} + {previous?.captured && current?.captured ? ( +
+ {(hunks || []).length === 0 && ( +

No stored content on either side.

+ )} + {(hunks || []).map((h, i) => ( +
+ {h.value.replace(/\n$/, '').split('\n').map((line, j) => ( +
+ {h.added ? '+' : h.removed ? '−' : ' '}{line} +
+ ))} +
+ ))} +
+ ) : ( +
+ {((current?.captured ? current.content : previous?.content) || 'Not captured.').split('\n').map((line, j) => ( +
{line}
+ ))} +
+ )} +
+
+ Old ({previous?.hash ? previous.hash.slice(0, 12) : '—'}) + {previous?.captured ? : not captured} +
+
+ Current ({current?.hash ? current.hash.slice(0, 12) : '—'}) + {current?.captured ? : not captured} +
+
+
+ )} + + ) +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index a24b76f..9cfe5c7 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -2,7 +2,7 @@ import type { Target, Subdomain, HTTPService, JSFile, JSFinding, Parameter, DirectoryFinding, AdminPanelFinding, BackupFinding, OpenRedirectFinding, NucleiFinding, VulnFinding, MonitoringChange, Task, TaskLog, TaskPhase, DashboardStats, AttackPath, - NetworkService, IngramCamera, Asset, + NetworkService, IngramCamera, Asset, FileBodyResponse, BountyProgram, BountySyncState, BountyScopeEvent, } from '../types' @@ -302,6 +302,10 @@ export const findings = { method: 'POST', body: JSON.stringify({ triage, note }), }), monitoringChanges: (id: string) => req(`/targets/${id}/monitoring-changes`), + // Stored capture of a monitored file's body at detection time — NOT a live + // fetch. Backs the Changes-tab diff drawer (css_change / js_change rows). + fileBody: (id: string, kind: 'css' | 'js', url: string, side: 'current' | 'old' = 'current') => + req(`/targets/${id}/${kind}-files/body?url=${encodeURIComponent(url)}&side=${side}`), ingram: (id: string) => req(`/targets/${id}/ingram`), // Global findings across all of the caller's targets (top-level Findings page). all: (params?: Record) => { diff --git a/frontend/src/pages/TargetDetail.tsx b/frontend/src/pages/TargetDetail.tsx index 2b3cd53..f3a8b58 100644 --- a/frontend/src/pages/TargetDetail.tsx +++ b/frontend/src/pages/TargetDetail.tsx @@ -5,6 +5,7 @@ import { useUIStore } from '../store/ui' import { Badge, Button, Spinner, Empty, CopyButton, ErrorBoundary, SkeletonRows } from '../components/ui' import { ScanModal } from '../components/targets/ScanModal' import { EvidenceViewer } from '../components/targets/EvidenceViewer' +import { ChangeDiffViewer } from '../components/targets/ChangeDiffViewer' import { LiveResults } from '../components/targets/LiveResults' import { ws } from '../lib/websocket' import { timeAgo, statusCodeColor, truncate, cn } from '../lib/utils' @@ -17,6 +18,44 @@ const isScannableProjectAsset = (asset: Asset) => (asset.kind === 'web' && ['domain', 'wildcard', 'url', 'page', 'js', 'api'].includes(asset.asset_type || 'domain')) || (asset.kind === 'network' && ['ip', 'cidr'].includes(asset.asset_type || 'ip')) +// Changes(/monitor) tab: filter buckets for the noise chips. The stylesheet +// content tracker (css_change + security:stylesheet_*) turned this tab into a +// steady stream; the chips let an operator pull one signal back out of it. +type ChangeFilter = 'all' | 'security' | 'scripts' | 'css' | 'http' | 'tech' | 'headers' +const CHANGE_FILTERS: { id: ChangeFilter; label: string }[] = [ + { id: 'all', label: 'All' }, + { id: 'security', label: 'Security' }, + { id: 'scripts', label: 'Scripts' }, + { id: 'css', label: 'Stylesheets' }, + { id: 'http', label: 'HTTP' }, + { id: 'tech', label: 'Tech' }, + { id: 'headers', label: 'Headers' }, +] + +// Bucket order matters: the stylesheet/script markers win before the generic +// "security:" prefix, so a security:stylesheet_* row lands in Stylesheets (not +// Security), a security:script_* row in Scripts, and header removals in Headers. +const monitorRowKind = (c: MonitoringChange): ChangeFilter => { + const ct = c.change_type + if (ct === 'css_change' || ct.includes('stylesheet')) return 'css' + if (ct === 'js_change' || ct.includes('script')) return 'scripts' + if (ct.includes('header')) return 'headers' + if (ct.startsWith('security')) return 'security' + if (ct === 'tech_change') return 'tech' + return 'http' +} + +// Badge color follows the stored severity (rows written since the severity +// column carry one); older rows without it keep the old type-based estimate. +const changeBadgeVariant = (c: { severity?: string; change_type: string }): 'high' | 'medium' | 'low' | 'info' => { + const sev = (c.severity || '').toLowerCase() + if (sev === 'high' || sev === 'medium' || sev === 'low' || sev === 'info') return sev + return c.change_type === 'js_change' ? 'medium' + : c.change_type.startsWith('security') || c.change_type.includes('header') ? 'high' + : c.change_type === 'tech_change' ? 'low' + : 'info' +} + // Projects created from the WP Scanner page carry this tag (see WPScanner.tsx) — // the same signal used there to decide this is a WordPress project, reused here // to decide this project's detail page should show WordPress-specific tabs. @@ -472,6 +511,9 @@ export default function TargetDetail() { const [scopeEvents, setScopeEvents] = useState([]) const [eventBusy, setEventBusy] = useState(null) const [evidence, setEvidence] = useState<{ id: string; url: string; type: string } | null>(null) + // Changes-tab noise filter + the open content-diff drawer (css/js rows). + const [changeFilter, setChangeFilter] = useState('all') + const [diffView, setDiffView] = useState<{ url: string; kind: 'css' | 'js' } | null>(null) const [isScanning, setIsScanning] = useState(false) const [monitorSaving, setMonitorSaving] = useState(false) const [paths, setPaths] = useState([]) @@ -661,6 +703,8 @@ export default function TargetDetail() { const code = Number((row as { status_code?: number })?.status_code || 0) return Math.floor(code / 100) === Number(statusClass) }) + : (tab === 'monitor' && changeFilter !== 'all') + ? (data as MonitoringChange[]).filter(row => monitorRowKind(row) === changeFilter) : data // Windowed view of the loaded rows — only the first `visibleCount` are mounted. const pageData = filteredData.slice(0, visibleCount) @@ -1105,6 +1149,26 @@ export default function TargetDetail() { ) : ( + <> + {tab === 'monitor' && ( +
+ {CHANGE_FILTERS.map(f => { + const n = f.id === 'all' + ? data.length + : (data as MonitoringChange[]).filter(row => monitorRowKind(row) === f.id).length + return ( + + ) + })} +
+ )}
@@ -1520,10 +1584,17 @@ export default function TargetDetail() {
{truncate(c.url, 50)} + {/* Content-capture diff for monitored stylesheets/scripts. + Legacy rows (recorded before capture) open too, but + show the not-captured note + hashes only. */} + {(c.change_type === 'css_change' || c.change_type === 'js_change') && ( + + )}
{/* Old/new values were truncated to 50 chars with no way to read the rest. Now each value wraps, shows the full text on hover, and has a @@ -1546,6 +1617,7 @@ export default function TargetDetail() {
- {c.change_type} + {c.change_type}
+ )} {!tabLoading && footerHasMore && (
@@ -1592,6 +1664,9 @@ export default function TargetDetail() { onClose={() => setEvidence(null)} /> )} + {diffView && ( + setDiffView(null)} /> + )}
) diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index a9cf94e..8572241 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -311,9 +311,22 @@ export interface MonitoringChange { change_type: string old_value: string new_value: string + severity: string detected_at: string } +// Body of a monitored file (css_files / js_files) as captured by the monitor +// pass when the change was detected — see GET /targets/{id}/css-files/body. +export interface FileBodyResponse { + kind: string + url: string + side: string + hash: string + size: number + content: string + captured: boolean +} + export interface IngramCamera { id: string address: string // ip:port diff --git a/internal/api/finding_handlers.go b/internal/api/finding_handlers.go index 1defb40..477480b 100644 --- a/internal/api/finding_handlers.go +++ b/internal/api/finding_handlers.go @@ -6,6 +6,7 @@ import ( "net/http" "sort" "strconv" + "strings" "github.com/gorilla/mux" "github.com/recon-platform/internal/models" @@ -560,10 +561,61 @@ func (h *Handler) handleNucleiAffected(w http.ResponseWriter, r *http.Request) { h.writeSuccess(w, out) } +// serveFileBody returns the capped content capture the MONITOR pass stored for +// a diffed asset (css_files / js_files): side=current (live capture) or +// side=old (previous capture rolled in at detection time). URL-keyed — the +// monitor records changes under the exact same URL string it tracks content +// for. Hard-scoped to the routed target: no cross-target reads. This is NOT a +// fetch proxy: it only ever returns content captured at detection time, so it +// has no request-driven SSRF surface. +func (h *Handler) serveFileBody(w http.ResponseWriter, r *http.Request, table, kind string) { + id := mux.Vars(r)["id"] + rawURL := strings.TrimSpace(r.URL.Query().Get("url")) + if rawURL == "" { + h.writeError(w, http.StatusBadRequest, "url query parameter required") + return + } + side := r.URL.Query().Get("side") + if side != "old" && side != "current" { + side = "current" + } + var storedURL, hash, content string + var size int64 + var rowErr error + if side == "old" { + rowErr = h.db.QueryRowContext(r.Context(), `SELECT url, COALESCE(old_hash,''), COALESCE(old_body,'') FROM `+table+` WHERE target_id = ? AND url = ?`, id, rawURL). + Scan(&storedURL, &hash, &content) + } else { + rowErr = h.db.QueryRowContext(r.Context(), `SELECT url, size, hash, COALESCE(body,'') FROM `+table+` WHERE target_id = ? AND url = ?`, id, rawURL). + Scan(&storedURL, &size, &hash, &content) + } + if rowErr != nil { + h.writeError(w, http.StatusNotFound, "no capture for this asset (the change predates content tracking, or the URL is not tracked)") + return + } + h.writeSuccess(w, map[string]any{ + "kind": kind, + "url": storedURL, + "side": side, + "hash": hash, + "size": size, + "content": content, + "captured": strings.TrimSpace(content) != "", + }) +} + +func (h *Handler) handleGetCSSFileBody(w http.ResponseWriter, r *http.Request) { + h.serveFileBody(w, r, "css_files", "css") +} + +func (h *Handler) handleGetJSFileBody(w http.ResponseWriter, r *http.Request) { + h.serveFileBody(w, r, "js_files", "js") +} + func (h *Handler) handleListMonitoringChanges(w http.ResponseWriter, r *http.Request) { id := mux.Vars(r)["id"] rows, err := h.db.QueryContext(r.Context(), ` - SELECT id, target_id, url, change_type, old_value, new_value, detected_at + SELECT id, target_id, url, change_type, old_value, new_value, COALESCE(severity,'info'), detected_at FROM monitoring_changes WHERE target_id = ? ORDER BY detected_at DESC LIMIT 500 `, id) @@ -580,12 +632,13 @@ func (h *Handler) handleListMonitoringChanges(w http.ResponseWriter, r *http.Req ChangeType string `json:"change_type"` OldValue string `json:"old_value"` NewValue string `json:"new_value"` + Severity string `json:"severity"` DetectedAt string `json:"detected_at"` } changes := make([]Change, 0) for rows.Next() { var c Change - if err := rows.Scan(&c.ID, &c.TargetID, &c.URL, &c.ChangeType, &c.OldValue, &c.NewValue, &c.DetectedAt); err == nil { + if err := rows.Scan(&c.ID, &c.TargetID, &c.URL, &c.ChangeType, &c.OldValue, &c.NewValue, &c.Severity, &c.DetectedAt); err == nil { changes = append(changes, c) } } diff --git a/internal/api/router.go b/internal/api/router.go index f41d8f2..c6949f0 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -204,6 +204,8 @@ func (h *Handler) Router() http.Handler { // Global findings aggregation (top-level Findings page) across all the caller's targets. api.HandleFunc("/findings", h.requireAuth(h.handleListAllFindings)).Methods("GET") api.HandleFunc("/targets/{id}/monitoring-changes", h.requireAuth(h.handleListMonitoringChanges)).Methods("GET") + api.HandleFunc("/targets/{id}/css-files/body", h.requireAuth(h.handleGetCSSFileBody)).Methods("GET") + api.HandleFunc("/targets/{id}/js-files/body", h.requireAuth(h.handleGetJSFileBody)).Methods("GET") // Tasks api.HandleFunc("/tasks", h.requireAuth(h.handleListTasks)).Methods("GET") diff --git a/internal/database/migrations.go b/internal/database/migrations.go index 6005414..d57ea0a 100644 --- a/internal/database/migrations.go +++ b/internal/database/migrations.go @@ -156,6 +156,10 @@ func RunMigrations(db *DB) error { alterTasksAddAssetsTotal, alterAdminPanelFindingsAddScreenshotID, alterVulnFindingsAddScreenshotID, + createCSSFilesTable, + alterJSFilesAddBody, + alterJSFilesAddOldHash, + alterJSFilesAddOldBody, createWPSitesTable, } @@ -1150,6 +1154,31 @@ CREATE TABLE IF NOT EXISTS js_findings ( FOREIGN KEY (js_file_id) REFERENCES js_files(id) ON DELETE CASCADE );` +const createCSSFilesTable = ` +CREATE TABLE IF NOT EXISTS css_files ( + id TEXT PRIMARY KEY, + target_id TEXT NOT NULL, + url TEXT NOT NULL, + size INTEGER DEFAULT 0, + hash TEXT DEFAULT '', + body TEXT DEFAULT '', + old_hash TEXT DEFAULT '', + old_body TEXT DEFAULT '', + last_seen DATETIME DEFAULT CURRENT_TIMESTAMP, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (target_id) REFERENCES targets(id) ON DELETE CASCADE, + UNIQUE(target_id, url) +);` + +// Body capture for diffing (monitor pass). The monitor keeps the previous +// content next to the current one so a `css_change`/`js_change` row carries a +// real line-level diff in the dashboard instead of two opaque SHA-256 hashes. +// Bodies are capped (monitor body cap) — truncated captures still diff the +// first N lines, which is where most real changes live. +const alterJSFilesAddBody = `ALTER TABLE js_files ADD COLUMN body TEXT DEFAULT '';` +const alterJSFilesAddOldHash = `ALTER TABLE js_files ADD COLUMN old_hash TEXT DEFAULT '';` +const alterJSFilesAddOldBody = `ALTER TABLE js_files ADD COLUMN old_body TEXT DEFAULT '';` + const createParametersTable = ` CREATE TABLE IF NOT EXISTS parameters ( id TEXT PRIMARY KEY, diff --git a/internal/scanner/monitor.go b/internal/scanner/monitor.go index 6b929f7..3a1b4dd 100644 --- a/internal/scanner/monitor.go +++ b/internal/scanner/monitor.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "net/url" + "regexp" "strings" "time" @@ -105,8 +106,20 @@ func (s *MonitorScanner) Run(ctx context.Context, targetID string, logFn LogFunc logFn("info", "monitor", fmt.Sprintf("Monitoring %d HTTP services for changes...", len(services))) changed := 0 + harvested := 0 knownHTTP := map[string]bool{} knownJS := map[string]bool{} + knownCSS := map[string]bool{} + cssKeyRows, err := s.db.QueryContext(ctx, `SELECT url FROM css_files WHERE target_id = ?`, targetID) + if err == nil { + for cssKeyRows.Next() { + var u string + if cssKeyRows.Scan(&u) == nil { + knownCSS[monitorURLKey(u)] = true + } + } + cssKeyRows.Close() + } for _, svc := range services { knownHTTP[monitorURLKey(svc.URL)] = true @@ -120,6 +133,7 @@ func (s *MonitorScanner) Run(ctx context.Context, targetID string, logFn LogFunc // NORMALIZED hash: volatile values (UUIDs/timestamps/CSRF/high-entropy) // are stripped first, so a dynamic page doesn't false-positive. newNormHash := normalizedHash(body) + harvested += s.harvestStylesheets(ctx, targetID, svc.URL, body, knownCSS) // ── Security-attribute change detection (supply-chain / form-hijack) ── curSnap := extractSecuritySnapshot(body, hostOf(svc.URL)) @@ -217,14 +231,19 @@ func (s *MonitorScanner) Run(ctx context.Context, targetID string, logFn LogFunc `, newStatus, newTitle, len(body), newNormHash, curSnap.toJSON(), storeServer, svc.ID) } - // Monitor JS file changes - jsRows, err := s.db.QueryContext(ctx, `SELECT id, url, hash FROM js_files WHERE target_id = ?`, targetID) + if harvested > 0 { + logFn("info", "monitor", fmt.Sprintf("Tracking %d new stylesheet URL(s) for content changes", harvested)) + } + + // Monitor JS file changes — hash AND the capped content body, so a js_change + // row can render a real line-diff in the dashboard instead of two hashes. + jsRows, err := s.db.QueryContext(ctx, `SELECT id, url, hash, COALESCE(body,'') FROM js_files WHERE target_id = ?`, targetID) if err == nil { - type jsRecord struct{ ID, URL, Hash string } + type jsRecord struct{ ID, URL, Hash, Body string } var jsFiles []jsRecord for jsRows.Next() { var r jsRecord - if err := jsRows.Scan(&r.ID, &r.URL, &r.Hash); err != nil { + if err := jsRows.Scan(&r.ID, &r.URL, &r.Hash, &r.Body); err != nil { jsRows.Close() return err } @@ -243,23 +262,95 @@ func (s *MonitorScanner) Run(ctx context.Context, targetID string, logFn LogFunc if ctx.Err() != nil { break } - newHash, newSize, err := s.fetchHashWithHeaders(ctx, jf.URL, headersFor(jf.URL)) + newHash, newSize, newBody, err := s.fetchFileWithBody(ctx, jf.URL, headersFor(jf.URL)) if err != nil || newHash == "" { continue } if jf.Hash != "" && newHash != jf.Hash { - confirmHash, _, confirmErr := s.fetchHashWithHeaders(ctx, jf.URL, headersFor(jf.URL)) + confirmHash, _, _, confirmErr := s.fetchFileWithBody(ctx, jf.URL, headersFor(jf.URL)) if confirmErr != nil || confirmHash != newHash { logFn("info", "monitor", fmt.Sprintf("Ignoring unstable one-off JavaScript response from %s", jf.URL)) continue } logFn("warn", "monitor", fmt.Sprintf("[JS CHANGE] %s hash changed", jf.URL)) s.recordChange(targetID, jf.URL, "js_change", jf.Hash, newHash, classifyChangeSeverity("js_change")) + // Roll the previous content into old_* for the diff drawer, then + // publish the new baseline in one go. + _, _ = s.db.ExecContext(ctx, `UPDATE js_files SET hash=?, size=?, body=?, old_hash=?, old_body=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, + newHash, newSize, newBody, jf.Hash, jf.Body, jf.ID) changed++ + continue } // Always establish/refresh the baseline. Previously an empty initial hash // was never written, so that JS file could change forever undetected. - _, _ = s.db.ExecContext(ctx, `UPDATE js_files SET hash=?, size=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, newHash, newSize, jf.ID) + // The content capture is written alongside it on the first pass only — + // rewriting unchanged 256KB bodies every pass is pure write churn. + if jf.Body == "" { + _, _ = s.db.ExecContext(ctx, `UPDATE js_files SET hash=?, size=?, body=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, newHash, newSize, newBody, jf.ID) + } else { + _, _ = s.db.ExecContext(ctx, `UPDATE js_files SET hash=?, size=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, newHash, newSize, jf.ID) + } + } + } + + // Monitor stylesheet content changes. css_files tracks every + // harvested from page bodies (same-origin AND + // cross-origin), so a version bump is judged by CONTENT, not by the link + // attribute: changed bytes → one css_change row (with a real line-diff in + // the drawer); a cache-buster rename with unchanged bytes → silence. + cssRows, err := s.db.QueryContext(ctx, `SELECT id, url, hash, COALESCE(body,''), COALESCE(size,0) FROM css_files WHERE target_id = ?`, targetID) + if err == nil { + type cssRecord struct { + ID, URL, Hash, Body string + Size int + } + var cssFiles []cssRecord + for cssRows.Next() { + var r cssRecord + if err := cssRows.Scan(&r.ID, &r.URL, &r.Hash, &r.Body, &r.Size); err != nil { + cssRows.Close() + return err + } + cssFiles = append(cssFiles, r) + } + if err := cssRows.Err(); err != nil { + cssRows.Close() + return err + } + cssRows.Close() + + for _, cf := range cssFiles { + knownCSS[monitorURLKey(cf.URL)] = true + if ctx.Err() != nil { + break + } + newHash, newSize, newBody, err := s.fetchFileWithBody(ctx, cf.URL, headersFor(cf.URL)) + if err != nil || newHash == "" { + continue + } + if cf.Hash != "" && newHash != cf.Hash { + confirmHash, _, _, confirmErr := s.fetchFileWithBody(ctx, cf.URL, headersFor(cf.URL)) + if confirmErr != nil || confirmHash != newHash { + logFn("info", "monitor", fmt.Sprintf("Ignoring unstable one-off stylesheet response from %s", cf.URL)) + continue + } + logFn("warn", "monitor", fmt.Sprintf("[CSS CHANGE] %s hash changed", cf.URL)) + s.recordChange(targetID, cf.URL, "css_change", cf.Hash, newHash, classifyChangeSeverity("css_change")) + _, _ = s.db.ExecContext(ctx, `UPDATE css_files SET hash=?, size=?, body=?, old_hash=?, old_body=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, + newHash, newSize, newBody, cf.Hash, cf.Body, cf.ID) + changed++ + continue + } + if cf.Hash == "" { + // Establish the baseline without ringing the bell (same as JS). + _, _ = s.db.ExecContext(ctx, `UPDATE css_files SET hash=?, size=?, body=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, newHash, newSize, newBody, cf.ID) + } else if cf.Body == "" { + // Legacy row that predates capture: backfill the content snapshot + // so the next real change has a "before" side to diff against. + _, _ = s.db.ExecContext(ctx, `UPDATE css_files SET body=?, size=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, newBody, newSize, cf.ID) + } else { + _, _ = s.db.ExecContext(ctx, `UPDATE css_files SET size=?, last_seen=CURRENT_TIMESTAMP WHERE id=?`, newSize, cf.ID) + } } } @@ -559,6 +650,12 @@ func notificationTitle(changeType string) string { switch changeType { case "js_change": return "JavaScript changed" + case "css_change": + return "Stylesheet content changed" + case "security:stylesheet_added": + return "External stylesheet added" + case "security:stylesheet_removed": + return "External stylesheet removed" case "http_change", "status_change": return "HTTP response changed" case "page_change": @@ -652,6 +749,115 @@ func (s *MonitorScanner) fetchHashWithHeaders(ctx context.Context, url string, h return h, len(body), nil } +// monitorBodyCaptureCap bounds the per-file content stored beside the hash +// (see the "Body capture for diffing" migration comment). The HASH always +// covers the full download (up to the 5MB reader cap), so a real change deep +// inside a large bundle is still detected; only the stored diff material is +// truncated to the first N bytes. +const monitorBodyCaptureCap = 256 * 1024 + +// Stylesheet `…` harvesting, attribute-order agnostic. +var ( + reLinkTag = regexp.MustCompile(`(?i)]*>`) + reLinkAttrRel = regexp.MustCompile(`(?i)\brel\s*=\s*["']?([^"'\s>]{2,20})`) + reLinkAttrHref = regexp.MustCompile(`(?i)\bhref\s*=\s*["']([^"']+)["']`) +) + +// fetchFileWithBody fetches a static asset (JS/CSS) for content tracking: +// full-content SHA-256, size, and the capped body the diff drawer renders. +// The HTML-document rejection mirrors fetchHashWithHeaders — an SPA fallback +// must never be captured as an asset's content. +func (s *MonitorScanner) fetchFileWithBody(ctx context.Context, rawURL string, headers map[string]string) (hash string, size int, body string, err error) { + req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil) + if err != nil { + return "", 0, "", err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36") + for k, v := range headers { + if strings.EqualFold(k, "Host") || strings.EqualFold(k, "Content-Length") { + continue + } + req.Header.Set(k, v) + } + + resp, err := monitorClient.Do(req) + if err != nil { + return "", 0, "", err + } + defer resp.Body.Close() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", 0, "", fmt.Errorf("unexpected asset response status %d", resp.StatusCode) + } + + raw, err := io.ReadAll(io.LimitReader(resp.Body, 5*1024*1024)) + if err != nil { + return "", 0, "", err + } + trimmed := strings.ToLower(strings.TrimSpace(string(raw))) + if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/html") && + (strings.HasPrefix(trimmed, " monitorBodyCaptureCap { + captured = captured[:monitorBodyCaptureCap] + } + h := fmt.Sprintf("%x", sha256.Sum256(raw)) + return h, len(raw), string(captured), nil +} + +// harvestStylesheets pulls URLs out of fresh page +// HTML and tracks them in css_files (same-origin AND cross-origin — that is +// what the browser actually loads), so the monitor can diff stylesheet +// CONTENT — the layer behind the Changes tab's css_change rows — rather than +// only the attribute-level "link appeared" check. Relative hrefs resolve +// against the page URL. Returns how many NEW URLs were registered. +func (s *MonitorScanner) harvestStylesheets(ctx context.Context, targetID, pageURL, body string, known map[string]bool) int { + base, err := url.Parse(strings.TrimSpace(pageURL)) + if err != nil || base.Host == "" { + return 0 + } + inserted := 0 + for _, tag := range reLinkTag.FindAllString(body, -1) { + relMatch := reLinkAttrRel.FindStringSubmatch(tag) + if relMatch == nil || !strings.Contains(strings.ToLower(relMatch[1]), "stylesheet") { + continue + } + hrefMatch := reLinkAttrHref.FindStringSubmatch(tag) + if hrefMatch == nil { + continue + } + href := strings.TrimSpace(hrefMatch[1]) + if href == "" || strings.HasPrefix(href, "#") { + continue + } + ref, err := url.Parse(href) + if err != nil { + continue + } + if ref.Scheme != "" && ref.Scheme != "http" && ref.Scheme != "https" { + continue // data:/javascript:/mailto:/tel: and friends + } + resolved := base.ResolveReference(ref) + if resolved.Scheme != "http" && resolved.Scheme != "https" { + continue + } + // Register the query-stripped URL: a `?ver=` churn maps to ONE tracked + // row (content re-fetched from the stable path each pass) instead of a + // new css_files row per cache-buster. + trackedURL := stripResourceQuery(resolved.String()) + key := monitorURLKey(trackedURL) + if key == "" || known[key] { + continue + } + known[key] = true + inserted++ + _, _ = s.db.ExecContext(ctx, `INSERT INTO css_files (id, target_id, url) VALUES (?, ?, ?) + ON CONFLICT(target_id, url) DO NOTHING`, uuid.New().String(), targetID, trackedURL) + } + return inserted +} + func (s *MonitorScanner) describeChange(oldStatus, newStatus int, oldTitle, newTitle, oldHash, newHash string) string { var parts []string if oldStatus != newStatus { diff --git a/internal/scanner/monitor_css_test.go b/internal/scanner/monitor_css_test.go new file mode 100644 index 0000000..15899b2 --- /dev/null +++ b/internal/scanner/monitor_css_test.go @@ -0,0 +1,303 @@ +package scanner + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/google/uuid" +) + +// Stylesheet change tracking: the monitor harvests +// hrefs from every fetched page into css_files, then diffs CONTENT. A version +// bump with unchanged bytes is silence; a real change yields ONE css_change +// row whose old_body rolls the previous capture — the material the Changes +// tab's diff drawer renders. +func TestStylesheetBaselineThenContentChange(t *testing.T) { + var mu sync.Mutex + state := struct { + css string + pageVer string + }{css: "body{color:blue}", pageVer: "0"} + renderPage := func(ver string) string { + return `hi` + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + pv, cs := state.pageVer, state.css + mu.Unlock() + switch { + case strings.HasSuffix(r.URL.Path, "/style.css"): + w.Header().Set("Content-Type", "text/css") + _, _ = w.Write([]byte(cs)) + default: + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(renderPage(pv))) + } + })) + defer srv.Close() + + db := newMonitorTestDB(t) + targetID := uuid.NewString() + _, _ = db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, srv.URL) + _, _ = db.Exec(`INSERT INTO http_services(id,target_id,url,status_code,title,security_snapshot,server) VALUES(?,?,?,?,?,'','')`, + uuid.NewString(), targetID, srv.URL+"/", 200, "") + + s := &MonitorScanner{db: db} + logFn := func(string, string, string) {} + + // Pass 1 establishes the baseline: URL harvested, content captured, zero rows. + mu.Lock() + state.pageVer = "1" + mu.Unlock() + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + var baselineHash, baselineBody string + if err := db.QueryRow(`SELECT hash, COALESCE(body,'') FROM css_files WHERE target_id=?`, targetID).Scan(&baselineHash, &baselineBody); err != nil { + t.Fatalf("stylesheet was not harvested into css_files: %v", err) + } + if baselineHash == "" || baselineBody == "" { + t.Fatal("first pass did not capture stylesheet content baseline") + } + var rows int + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=?`, targetID).Scan(&rows) + if rows != 0 { + t.Fatalf("baseline pass raised %d change rows, want 0", rows) + } + + // Pass 2 with a REAL content change → exactly one css_change row with the + // previous capture rolled into old_*. + mu.Lock() + state.pageVer = "2" + state.css = "body{color:red}" + mu.Unlock() + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + var oldHash, oldBody, curBody string + if err := db.QueryRow(`SELECT old_hash, old_body, body FROM css_files WHERE target_id=?`, targetID).Scan(&oldHash, &oldBody, &curBody); err != nil { + t.Fatal(err) + } + if oldHash != baselineHash || oldBody != "body{color:blue}" || !strings.Contains(curBody, "color:red") { + t.Fatalf("diff material not rolled: old_hash=%q old_body=%q body=%q", oldHash, oldBody, curBody) + } + var cssTotal int + if err := db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type='css_change'`, targetID).Scan(&cssTotal); err != nil { + t.Fatal(err) + } + if cssTotal != 1 { + t.Fatalf("content-change pass produced %d css_change rows, want 1 (a relative link never raises security: rows)", cssTotal) + } + + // Pass 3: cache-buster bump with UNCHANGED bytes → silence (no extra rows). + mu.Lock() + state.pageVer = "3" + mu.Unlock() + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type='css_change'`, targetID).Scan(&cssTotal) + if cssTotal != 1 { + t.Fatalf("no-op version bump produced extra css rows: %d css_change rows, want still 1", cssTotal) + } + var attrRows int + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type LIKE 'security:stylesheet%'`, targetID).Scan(&attrRows) + if attrRows != 0 { + t.Fatalf("relative stylesheets raised %d security: rows, want 0", attrRows) + } +} + +// A cache-busted bump on an EXTERNAL stylesheet must not spam the Changes tab +// with add/remove pairs when the bytes are identical — snapshot keys are +// path-based (stripResourceQuery), and unchanged content yields no css_change. +func TestExternalStylesheetVerBumpIsSilent(t *testing.T) { + pageVer := "1" + renderPage := func(ver string) string { + return `x` + } + + pageSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(renderPage(pageVer))) + })) + defer pageSrv.Close() + + db := newMonitorTestDB(t) + targetID := uuid.NewString() + _, _ = db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, pageSrv.URL) + _, _ = db.Exec(`INSERT INTO http_services(id,target_id,url,status_code,title,security_snapshot) VALUES(?,?,?,?,?,'')`, + uuid.NewString(), targetID, pageSrv.URL+"/", 200, "x") + + s := &MonitorScanner{db: db} + logFn := func(string, string, string) {} + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + pageVer = "2" + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + + var attrRows, cssRows int + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type LIKE 'security:stylesheet%'`, targetID).Scan(&attrRows) + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type='css_change'`, targetID).Scan(&cssRows) + if attrRows != 0 || cssRows != 0 { + t.Fatalf("ver-bump with unchanged bytes is not silent: %d security: rows, %d css_change rows", attrRows, cssRows) + } +} + +// Adding a genuinely NEW stylesheet path still alerts, at info severity. +func TestNewExternalStylesheetPathStillAlerts(t *testing.T) { + extraEnabled := false + renderPage := func(withExtra bool) string { + base := `x` + if withExtra { + return `` + + `x` + } + return base + } + + pageSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(renderPage(extraEnabled))) + })) + defer pageSrv.Close() + + db := newMonitorTestDB(t) + targetID := uuid.NewString() + _, _ = db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, pageSrv.URL) + _, _ = db.Exec(`INSERT INTO http_services(id,target_id,url,status_code,title,security_snapshot) VALUES(?,?,?,?,?,'')`, + uuid.NewString(), targetID, pageSrv.URL+"/", 200, "x") + + s := &MonitorScanner{db: db} + logFn := func(string, string, string) {} + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + extraEnabled = true + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + + var addedRows int + if err := db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type='security:stylesheet_added'`, targetID).Scan(&addedRows); err != nil { + t.Fatal(err) + } + if addedRows != 1 { + t.Fatalf("new stylesheet path produced %d security:stylesheet_added rows, want exactly 1", addedRows) + } + var sev string + _ = db.QueryRow(`SELECT severity FROM monitoring_changes WHERE target_id=? AND change_type='security:stylesheet_added'`, targetID).Scan(&sev) + if sev != "info" { + t.Fatalf("new stylesheet severity = %q, want info", sev) + } + var removedRows int + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type='security:stylesheet_removed'`, targetID).Scan(&removedRows) + if removedRows != 0 { + t.Fatalf("adding a path produced %d removed rows, want 0", removedRows) + } +} + +// js_change rows carry the same diff material: the previous capture rolls into +// old_body at detection time. +func TestJSChangeCapturesBodies(t *testing.T) { + var mu sync.Mutex + js := "console.log('v1')" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + body := js + mu.Unlock() + w.Header().Set("Content-Type", "application/javascript") + _, _ = w.Write([]byte(body)) + })) + defer srv.Close() + + db := newMonitorTestDB(t) + targetID := uuid.NewString() + _, _ = db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, srv.URL) + _, _ = db.Exec(`INSERT INTO js_files(id,target_id,url,hash) VALUES(?,?,?,'')`, uuid.NewString(), targetID, srv.URL+"/app.js") + + s := &MonitorScanner{db: db} + logFn := func(string, string, string) {} + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + + mu.Lock() + js = "console.log('v2')" + mu.Unlock() + if err := s.Run(context.Background(), targetID, logFn); err != nil { + t.Fatal(err) + } + + var oldBody, curBody string + if err := db.QueryRow(`SELECT old_body, body FROM js_files WHERE target_id=?`, targetID).Scan(&oldBody, &curBody); err != nil { + t.Fatal(err) + } + if oldBody != "console.log('v1')" || curBody != "console.log('v2')" { + t.Fatalf("js bodies not rolled: old_body=%q body=%q", oldBody, curBody) + } + var changes int + _ = db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND change_type='js_change'`, targetID).Scan(&changes) + if changes != 1 { + t.Fatalf("changed javascript produced %d js_change rows, want 1", changes) + } +} + +// ── pure-unit tests ────────────────────────────────────────────────────────── + +func TestStripResourceQuery(t *testing.T) { + cases := map[string]string{ + "https://cdn.t/app.css?ver=8&x=1": "https://cdn.t/app.css", + "https://cdn.t/app.css": "https://cdn.t/app.css", + "/app.css?ver=1": "/app.css?ver=1", // no host: leave untouched + "": "", + } + for in, want := range cases { + if got := stripResourceQuery(in); got != want { + t.Errorf("stripResourceQuery(%q) = %q, want %q", in, got, want) + } + } +} + +func TestSecuritySnapshotSuppressesVerBumpButNotNewPath(t *testing.T) { + page := func(path string) string { + return `` + } + base := extractSecuritySnapshot(page("/x.css?ver=1"), "app.test") + bumped := extractSecuritySnapshot(page("/x.css?ver=2"), "app.test") + if base.toJSON() != bumped.toJSON() { + t.Fatal("ver bump on the same stylesheet path must not alter the security snapshot") + } + newPath := extractSecuritySnapshot(page("/y.css"), "app.test") + if newPath.toJSON() == bumped.toJSON() { + t.Fatal("a genuinely new stylesheet path must alter the security snapshot") + } + withScript := extractSecuritySnapshot(page("/x.css?ver=2\")>"), "app.test") + if withScript.toJSON() == bumped.toJSON() { + t.Fatal("a new external script must alter the security snapshot") + } +} + +func TestSecurityChangeSeverityStylesheetIsInfo(t *testing.T) { + cases := []struct { + change securityChange + want string + }{ + {securityChange{action: "added", attr: securityAttr{Kind: "script", Value: "https://x/e.js"}}, "high"}, + {securityChange{action: "added", attr: securityAttr{Kind: "stylesheet", Value: "https://x/e.css"}}, "info"}, + {securityChange{action: "added", attr: securityAttr{Kind: "hidden", Value: "csrf_seen"}}, "medium"}, + {securityChange{action: "removed", attr: securityAttr{Kind: "form", Value: "/login"}}, "medium"}, + } + for _, tc := range cases { + if got := securityChangeSeverity(tc.change); got != tc.want { + t.Errorf("severity(%s %s) = %q, want %q", tc.change.action, tc.change.attr.Kind, got, tc.want) + } + } +} diff --git a/internal/scanner/monitor_engine.go b/internal/scanner/monitor_engine.go index 18bd56a..b238030 100644 --- a/internal/scanner/monitor_engine.go +++ b/internal/scanner/monitor_engine.go @@ -185,7 +185,11 @@ func extractSecuritySnapshot(body, pageHost string) securitySnapshot { } for _, m := range reLinkHref.FindAllStringSubmatch(body, -1) { if ext(m[1]) { - add("stylesheet", m[1]) + // Cache-bust churn (`?ver=`, hashed filenames) flickers the href on + // every deploy. Key the diff by path so a version bump on the SAME + // resource stops emitting add/remove pairs; the CONTENT truth lives + // in css_change rows from css_files content tracking. + add("stylesheet", stripResourceQuery(m[1])) } } for _, m := range reAHref.FindAllStringSubmatch(body, -1) { @@ -228,6 +232,20 @@ func hostOf(rawURL string) string { return strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) } +// stripResourceQuery drops the query string and fragment from an asset URL so +// cache-bust churn (`/app.css?ver=8`) keys as the SAME resource across +// security-snapshot diffs. The versioned URL is still captured and diffed by +// content via css_files, so no real change is hidden. +func stripResourceQuery(raw string) string { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Hostname() == "" { + return raw + } + u.RawQuery = "" + u.Fragment = "" + return u.String() +} + func (s securitySnapshot) toJSON() string { b, _ := json.Marshal(s) return string(b) @@ -287,8 +305,13 @@ func securityChangeSeverity(c securityChange) string { switch c.attr.Kind { case "script", "iframe", "form": return "high" - case "stylesheet", "hidden": + case "hidden": return "medium" + case "stylesheet": + // Path-keyed now (stripResourceQuery), so stylesheet rows are the + // rare genuinely-new/deleted-path case. Content-level truth lives + // in css_change rows. + return "info" default: return "low" } From d9d7213921830ce0a050dbb6f7fbed173629967a Mon Sep 17 00:00:00 2001 From: void Date: Sat, 24 Oct 2026 08:00:06 +0330 Subject: [PATCH 07/14] fix(monitor): stylesheet signals stay observational in escalation + legacy-snapshot diff keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - escalateIfChanged ignored change severity and treated any monitoring row as escalation fuel: a css_change-only redeploy (or a security: stylesheet_* attr row) fell through to the default heavy set and fired probe + backup discovery + nuclei. Stylesheet signals are observational now — they count toward Changes-tab history and the bell, never toward a watch escalation (same policy as the js_change JS-refresh set, which has no CSS equivalent). A js_change beside css churn still escalates. - diffSecuritySnapshots keyed stylesheet attrs on the raw href, so every snapshot recorded before the ?ver= stripping change emitted one spurious add+remove pair per stylesheet on the first pass after upgrade. The key strips the query for stylesheet kind on both sides; other kinds keep exact-match keys. --- internal/scanner/monitor_css_test.go | 28 +++++++++++++ internal/scanner/monitor_engine.go | 16 +++++++- internal/scheduler/monitor_escalation_test.go | 40 +++++++++++++++++++ internal/scheduler/scheduler.go | 13 +++++- 4 files changed, 94 insertions(+), 3 deletions(-) diff --git a/internal/scanner/monitor_css_test.go b/internal/scanner/monitor_css_test.go index 15899b2..963d329 100644 --- a/internal/scanner/monitor_css_test.go +++ b/internal/scanner/monitor_css_test.go @@ -301,3 +301,31 @@ func TestSecurityChangeSeverityStylesheetIsInfo(t *testing.T) { } } } + +// Legacy snapshots (recorded before extraction started stripping ?ver=) must +// not unleash a one-time add+remove pair per stylesheet on the first diff +// after an upgrade: the diff key normalizes stylesheet values on both sides. +// Non-stylesheet kinds keep exact-match keys (script bump → added + removed). +func TestSecuritySnapshotLegacyQueryKeysDiffSilently(t *testing.T) { + old := securitySnapshot{Attrs: []securityAttr{ + {Kind: "stylesheet", Value: "https://cdn.example.test/s/a.css?ver=1"}, + {Kind: "script", Value: "https://x.example.test/y.js?h=1"}, + }} + cur := securitySnapshot{Attrs: []securityAttr{ + {Kind: "stylesheet", Value: "https://cdn.example.test/s/a.css?ver=2"}, + {Kind: "script", Value: "https://x.example.test/y.js?h=2"}, + }} + got := diffSecuritySnapshots(old, cur) + sheetChanges := 0 + for _, c := range got { + if c.attr.Kind == "stylesheet" { + sheetChanges++ + } + } + if sheetChanges != 0 { + t.Fatalf("stylesheet ver-bump diffed as %d change(s), want 0", sheetChanges) + } + if len(got) != 2 { + t.Fatalf("script hash bump produced %d change(s), want exactly 2 (added+removed)", len(got)) + } +} diff --git a/internal/scanner/monitor_engine.go b/internal/scanner/monitor_engine.go index b238030..72e5705 100644 --- a/internal/scanner/monitor_engine.go +++ b/internal/scanner/monitor_engine.go @@ -264,15 +264,27 @@ type securityChange struct { attr securityAttr } +// securityKey is the diff-set key for one snapshot attribute. Stylesheets key +// on the query-stripped path so a LEGACY stored snapshot (recorded before +// extraction started stripping prompts) and a fresh extraction of the same +// resource share one key — an upgrade to this behavior must not unleash one +// add+remove pair per stylesheet on the first pass after it ships. +func securityKey(a securityAttr) string { + if a.Kind == "stylesheet" { + return a.Kind + "|" + stripResourceQuery(a.Value) + } + return a.Kind + "|" + a.Value +} + // diffSecuritySnapshots returns what security attributes were added/removed. func diffSecuritySnapshots(old, cur securitySnapshot) []securityChange { oldSet := map[string]securityAttr{} for _, a := range old.Attrs { - oldSet[a.Kind+"|"+a.Value] = a + oldSet[securityKey(a)] = a } curSet := map[string]securityAttr{} for _, a := range cur.Attrs { - curSet[a.Kind+"|"+a.Value] = a + curSet[securityKey(a)] = a } var out []securityChange for k, a := range curSet { diff --git a/internal/scheduler/monitor_escalation_test.go b/internal/scheduler/monitor_escalation_test.go index 26603a6..840af91 100644 --- a/internal/scheduler/monitor_escalation_test.go +++ b/internal/scheduler/monitor_escalation_test.go @@ -130,3 +130,43 @@ func indexOf(hay, needle string) int { } return -1 } + +// A stylesheet redeploy churns css_change / security:stylesheet_* rows, but +// those signals are observational: they must not escalate to nuclei / +// backup discovery the way a changed JS seed or a new host does. +func TestStylesheetOnlyChangesDoNotEscalate(t *testing.T) { + s := newTestScheduler(t) + if _, err := s.db.Exec(`INSERT INTO targets (id, domain, subdomain_count) VALUES ('t','ex.com',10)`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`INSERT INTO monitoring_changes (id, target_id, url, change_type, old_value, new_value, severity, detected_at) + VALUES ('mc-css', 't', 'https://ex.com/a.css', 'css_change', 'old', 'new', 'info', CURRENT_TIMESTAMP)`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec(`INSERT INTO monitoring_changes (id, target_id, url, change_type, old_value, new_value, severity, detected_at) + VALUES ('mc-attr', 't', 'https://ex.com/a.css', 'security:stylesheet_added', 'old', 'new', 'info', CURRENT_TIMESTAMP)`); err != nil { + t.Fatal(err) + } + s.escalateIfChanged("t", "ex.com", 10, time.Now().Add(-time.Minute)) + if n := countTasks(t, s, monitorEscalationType); n != 0 { + t.Fatalf("stylesheet-only changes escalated to %d task(s), want 0", n) + } + + // Mixed signal: the css churn is ignored, a js_change still escalates with + // the observational JS refresh set — and only that set. + if _, err := s.db.Exec(`INSERT INTO monitoring_changes (id, target_id, url, change_type, old_value, new_value, severity, detected_at) + VALUES ('mc-js', 't', 'https://ex.com/app.js', 'js_change', 'old', 'new', 'medium', CURRENT_TIMESTAMP)`); err != nil { + t.Fatal(err) + } + s.escalateIfChanged("t", "ex.com", 10, time.Now().Add(-time.Minute)) + if n := countTasks(t, s, monitorEscalationType); n != 1 { + t.Fatalf("js_change beside css churn did not escalate (%d task(s)), want 1", n) + } + var modules string + if err := s.db.QueryRow(`SELECT modules FROM tasks WHERE type=? LIMIT 1`, monitorEscalationType).Scan(&modules); err != nil { + t.Fatal(err) + } + if !contains(modules, ModuleJSAnalysis) { + t.Fatalf("escalation modules %q missing %q", modules, ModuleJSAnalysis) + } +} diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 84c7992..c78ec65 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -2330,7 +2330,13 @@ const ( func (s *Scheduler) escalateIfChanged(targetID, domain string, baselineSubs int, since time.Time) { var nowSubs, changes int _ = s.db.QueryRow(`SELECT COALESCE(subdomain_count,0) FROM targets WHERE id=?`, targetID).Scan(&nowSubs) - _ = s.db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND detected_at >= ?`, + // Stylesheet signals are observational only: there is no follow-up CSS + // module (unlike the JS refresh set below), so a css_change row or an + // informative security:stylesheet_* row must never fire a nuclei/ + // backup-discovery escalation just because a site redeployed its CSS. + // The Changes tab still records and diffs all of it. + _ = s.db.QueryRow(`SELECT COUNT(*) FROM monitoring_changes WHERE target_id=? AND detected_at >= ? + AND change_type != 'css_change' AND change_type NOT LIKE 'security:stylesheet%'`, targetID, since.UTC().Format("2006-01-02 15:04:05")).Scan(&changes) newSubs := nowSubs - baselineSubs @@ -2359,6 +2365,11 @@ func (s *Scheduler) escalateIfChanged(targetID, domain string, baselineSubs int, for rows.Next() { var typ string if rows.Scan(&typ) == nil { + // Stylesheet signals never choose escalation modules — see the + // count query above for the reasoning. + if typ == "css_change" || strings.HasPrefix(typ, "security:stylesheet") { + continue + } types[typ] = true } } From f9d106065f209d4f4447b740945632e4dcfbea72 Mon Sep 17 00:00:00 2001 From: void Date: Sat, 24 Oct 2026 12:31:01 +0330 Subject: [PATCH 08/14] fix(subdomain): resilient wildcard probes + branch wildcard detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old random-label gate probed 5 fixed names, treated every DNS error like NXDOMAIN, and only looked at the apex. Three holes admitted junk fleets wholesale: - A flaky answer (SERVFAIL/rate-limit) looked like NXDOMAIN, so <3 resolving probes meant no wildcard evidence and the whole junk namespace was admitted (the 'sometimes' in the bug report). - Rotating CDN pools and rotating CNAME targets need more samples than one batch of 5 can give. - Branch wildcards (*.git.example.com) were never probed at all — asdf-garbage.git.realestate.com.au style junk was admitted even with a perfectly flipped apex verdict. Fixes, all in the DNS gate so junk stops before HTTP probing: - Probe verdicts now distinguish definitive NXDOMAIN (net.DNSError.IsNotFound) from ambiguous errors, and ambiguous answers retry on DIFFERENT upstream resolvers (pinned per attempt) — one rate-limited resolver can no longer blind the gate. - When neither agreement threshold (<3 resolved AND <3 NX) is reached, a second batch of 5 fresh random labels runs before deciding. - Branch wildcards: clustered candidate suffixes (>=5 names sharing a strictly-deeper suffix, e.g. git.example.com) get probed; confirmed branches become per-name zones the admission gate, brute skip, deep resolution rechecks and vhost classification all respect. 8 branches per apex scan, 3 per recursion sub-apex, bounded DNS traffic. - wildcardIPs/wildcardCNAMEs maps replaced by wildcardProfile (apex + branch zones with label-boundary suffix matching: evil-git.name must never match *.git.name). nil profile behaves as no-evidence. - Inconclusive probe batches now log their histogram instead of silently deciding. Unit tests cover suffix clustering, zone matching (label boundaries), nil-profile safety and the decision/summary logic without network. --- internal/scanner/recon_stack.go | 12 +- internal/scanner/subdomain.go | 180 ++++------ internal/scanner/wildcard.go | 542 ++++++++++++++++++++++++++++++ internal/scanner/wildcard_test.go | 201 +++++++++++ 4 files changed, 817 insertions(+), 118 deletions(-) create mode 100644 internal/scanner/wildcard.go create mode 100644 internal/scanner/wildcard_test.go diff --git a/internal/scanner/recon_stack.go b/internal/scanner/recon_stack.go index b935d58..7ec8f01 100644 --- a/internal/scanner/recon_stack.go +++ b/internal/scanner/recon_stack.go @@ -55,8 +55,8 @@ func (s *SubdomainScanner) toolResolverList() []string { // deepDNSEnum runs the heavy DNS discovery chain (alterx permutations + // puredns/massdns brute) on top of the built-in active enum. It only adds // names that actually resolve and aren't wildcard answers. -func (s *SubdomainScanner) deepDNSEnum(ctx context.Context, targetID, domain string, found map[string]bool, mu *sync.Mutex, wildcardIPs map[string]bool, logFn LogFunc) { - if len(wildcardIPs) > 0 { +func (s *SubdomainScanner) deepDNSEnum(ctx context.Context, targetID, domain string, found map[string]bool, mu *sync.Mutex, wildcardProfile *wildcardProfile, logFn LogFunc) { + if wildcardProfile.apexWildcard() { return // wildcard domains: brute is meaningless, skip } @@ -156,7 +156,7 @@ func (s *SubdomainScanner) deepDNSEnum(ctx context.Context, targetID, domain str return } for _, n := range resolved { - ip := firstNonWildcardIP(resolveHostIPs(ctx, n, s.resolver), wildcardIPs) + ip := wildcardProfile.firstNonWildcardIP(n, resolveHostIPs(ctx, n, s.resolver)) if ip == "" { continue // wildcard-only or unresolved on recheck — not an asset } @@ -169,7 +169,7 @@ func (s *SubdomainScanner) deepDNSEnum(ctx context.Context, targetID, domain str return } - s.resolveCandidates(ctx, targetID, names, found, mu, wildcardIPs, logFn) + s.resolveCandidates(ctx, targetID, names, found, mu, wildcardProfile, logFn) } // dnsxResolve performs only DNS resolution; every returned name is subsequently @@ -357,7 +357,7 @@ func (s *SubdomainScanner) purednsResolve(ctx context.Context, targetID string, } // resolveCandidates is the built-in fallback resolver pool for a name list. -func (s *SubdomainScanner) resolveCandidates(ctx context.Context, targetID string, names []string, found map[string]bool, mu *sync.Mutex, wildcardIPs map[string]bool, logFn LogFunc) { +func (s *SubdomainScanner) resolveCandidates(ctx context.Context, targetID string, names []string, found map[string]bool, mu *sync.Mutex, wildcardProfile *wildcardProfile, logFn LogFunc) { workers := s.cfg.Workers.SubdomainEnumeration if workers <= 0 { workers = 20 @@ -374,7 +374,7 @@ func (s *SubdomainScanner) resolveCandidates(ctx context.Context, targetID strin if ctx.Err() != nil { return } - ip := firstNonWildcardIP(resolveHostIPs(ctx, name, s.resolver), wildcardIPs) + ip := wildcardProfile.firstNonWildcardIP(name, resolveHostIPs(ctx, name, s.resolver)) if ip == "" { continue } diff --git a/internal/scanner/subdomain.go b/internal/scanner/subdomain.go index be04991..25acec2 100644 --- a/internal/scanner/subdomain.go +++ b/internal/scanner/subdomain.go @@ -91,10 +91,9 @@ func (s *SubdomainScanner) Run(ctx context.Context, targetID, domain string, log } // Detect wildcard DNS up front so we can flag fake passive subdomains. - wildcardIPs, wildcardCNAMEs := detectWildcard(domain, s.resolver) - if len(wildcardIPs) > 0 || len(wildcardCNAMEs) > 0 { - logFn("warn", "subdomain_enum", fmt.Sprintf("Wildcard DNS detected (%d IPs) — passive results will be verified by resolution", len(wildcardIPs))) - } + // Probes retry across the configured public resolvers and distinguish a + // definitive NXDOMAIN from SERVFAIL/flake — see wildcard.go. + wildcardProfile := detectWildcard(domain, s.resolver, s.wildcardServers(), logFn) // Discovery and admission are deliberately separate. Passive tools are allowed // to be noisy, but their strings remain in-memory candidates until the batched @@ -326,11 +325,11 @@ func (s *SubdomainScanner) Run(ctx context.Context, targetID, domain string, log // operator disables it in the scan menu, only passive discovery + resolution + // ASN/vhost run, which is dramatically faster. if subdomainBruteEnabled(ctx) { - s.activeEnum(ctx, targetID, domain, found, &mu, wildcardIPs, logFn) + s.activeEnum(ctx, targetID, domain, found, &mu, wildcardProfile, logFn) // Deeper DNS discovery (optional, graceful): alterx permutations + // puredns/massdns brute at scale. No-op when those tools aren't installed. - s.deepDNSEnum(ctx, targetID, domain, found, &mu, wildcardIPs, logFn) + s.deepDNSEnum(ctx, targetID, domain, found, &mu, wildcardProfile, logFn) } else { logFn("info", "subdomain_enum", "Permutation brute-force disabled for this scan — passive discovery + resolution only.") } @@ -351,16 +350,29 @@ func (s *SubdomainScanner) Run(ctx context.Context, targetID, domain string, log } mu.Unlock() + // Branch wildcard detection (F3): a junk fleet can hide under a wildcard + // deeper than the apex — *.git.example.com was the live example, where + // hundreds of "asdf-garbage.git." names were admitted wholesale. Every + // discovery source has merged by now, so clustered candidate suffixes are + // known: probe the clusters before admission runs. + candidateSnapshot := make([]string, 0, len(found)) + mu.Lock() + for name := range found { + candidateSnapshot = append(candidateSnapshot, name) + } + mu.Unlock() + wildcardProfile.detectBranches(ctx, domain, candidateSnapshot, s.resolver, s.wildcardServers(), branchMaxProbes, logFn) + // Parallel admission pass. It returns candidates that had neither a genuine // A/AAAA answer nor an explicit CNAME; wildcard-only answers remain eligible // for vhost proof but are not assets yet. - rejected := s.resolveAll(ctx, targetID, found, &mu, wildcardIPs, wildcardCNAMEs, logFn) + rejected := s.resolveAll(ctx, targetID, found, &mu, wildcardProfile, logFn) // Virtual-host scan: probe candidate hostnames (that do NOT resolve in DNS) // against the IPs we already know, via the Host header, and record any that // the server actually serves — internal/forgotten vhosts DNS enumeration // can't see. Stored with source='vhost' so the UI can flag them. - s.vhostScan(ctx, targetID, domain, found, &mu, wildcardIPs, logFn) + s.vhostScan(ctx, targetID, domain, found, &mu, wildcardProfile, logFn) // Vhost verification had the final chance to promote wildcard/unresolved names. // Remove only old eager-DNS rows which still lack evidence; explicit seeds, @@ -537,10 +549,15 @@ func (s *SubdomainScanner) vhostProbeConcurrency() int { return c } -func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain string, found map[string]bool, mu *sync.Mutex, wildcardIPs map[string]bool, logFn LogFunc) { +func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain string, found map[string]bool, mu *sync.Mutex, wildcardZones *wildcardProfile, logFn LogFunc) { if ctx.Err() != nil { return } + // A nil profile (test paths, "no wildcard evidence") behaves like an empty + // one; field accesses below must not trip on it. + if wildcardZones == nil { + wildcardZones = &wildcardProfile{} + } // Distinct known IPs (cap to keep the pass bounded; raised from 40 — a huge // multi-asset scan can legitimately have far more than 40 distinct hosting // IPs, and silently dropping the rest means whole IP ranges never get @@ -566,12 +583,21 @@ func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain strin for _, ip := range ips { ipSeen[ip] = true } - for ip := range wildcardIPs { + for ip := range wildcardZones.apex.ips { if !ipSeen[ip] { ips = append(ips, ip) ipSeen[ip] = true } } + // Branch wildcard answer IPs are equally valuable vhost-probe targets. + for _, zone := range wildcardZones.branches { + for ip := range zone.ips { + if !ipSeen[ip] { + ips = append(ips, ip) + ipSeen[ip] = true + } + } + } if len(ips) == 0 { return } @@ -597,7 +623,7 @@ func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain strin for _, p := range devToolWords { add(p + "." + domain) } - if len(wildcardIPs) > 0 { + if wildcardZones.hasAnyWildcard() { // DNS brute-force is meaningless on a wildcard zone, but the same adaptive // names are valuable Host-header candidates. Feed a bounded slice into the // differential vhost verifier instead of accepting their wildcard DNS answer. @@ -634,7 +660,7 @@ func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain strin // Probe names DNS cannot distinguish: NXDOMAIN/no-address candidates and names // whose only address is the wildcard answer. Ordinary independently-resolving // hosts are already admitted and do not need expensive Host-header probing. - probeHosts := classifyVhostCandidates(ctx, candidates, wildcardIPs, s.resolver) + probeHosts := classifyVhostCandidates(ctx, candidates, wildcardZones, s.resolver) logFn("info", "vhost", fmt.Sprintf("Virtual-host scan: %d IP(s) × %d candidate host(s)...", len(ips), len(probeHosts))) @@ -760,7 +786,7 @@ func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain strin logFn("info", "vhost", fmt.Sprintf("Virtual-host scan done. %d vhost(s) discovered.", discovered)) } -func classifyVhostCandidates(ctx context.Context, candidates []string, wildcardIPs map[string]bool, r *net.Resolver) []string { +func classifyVhostCandidates(ctx context.Context, candidates []string, wildcardProfile *wildcardProfile, r *net.Resolver) []string { if len(candidates) == 0 { return nil } @@ -777,7 +803,7 @@ func classifyVhostCandidates(ctx context.Context, candidates []string, wildcardI defer wg.Done() for host := range jobs { ips := resolveHostIPs(ctx, host, r) - if len(ips) == 0 || overlapsWildcardIPs(ips, wildcardIPs) { + if len(ips) == 0 || wildcardProfile.overlapsWildcardIPs(host, ips) { hits <- host } } @@ -973,9 +999,12 @@ var bruteWords = []string{ var permPrefixes = []string{"dev", "staging", "test", "qa", "uat", "old", "new", "internal", "api", "admin"} var permSuffixes = []string{"dev", "staging", "test", "v1", "v2", "v3", "old", "new", "internal", "api"} -func (s *SubdomainScanner) activeEnum(ctx context.Context, targetID, domain string, found map[string]bool, mu *sync.Mutex, wildcardIPs map[string]bool, logFn LogFunc) { +func (s *SubdomainScanner) activeEnum(ctx context.Context, targetID, domain string, found map[string]bool, mu *sync.Mutex, wildcardProfile *wildcardProfile, logFn LogFunc) { // Skip active brute on wildcard domains (every name would "resolve"). - if len(wildcardIPs) > 0 { + // Only an APEX wildcard justifies the skip: a branch wildcard leaves the + // rest of the zone worth enumerating, and branch hits are filtered + // per-name by the admission gate. + if wildcardProfile.apexWildcard() { logFn("info", "subdomain_enum", "Wildcard domain — skipping active brute-force to avoid fake results") return } @@ -1043,7 +1072,7 @@ func (s *SubdomainScanner) activeEnum(ctx context.Context, targetID, domain stri if ctx.Err() != nil { return } - ip := firstNonWildcardIP(resolveHostIPs(ctx, name, s.resolver), wildcardIPs) + ip := wildcardProfile.firstNonWildcardIP(name, resolveHostIPs(ctx, name, s.resolver)) if ip == "" { continue } @@ -1090,7 +1119,7 @@ func (s *SubdomainScanner) admittedSubdomainNames(ctx context.Context, targetID, // resolveAll resolves every discovered subdomain concurrently and updates its // IP. Names that resolve to a wildcard IP are kept but logged so HTTP probing // can later dedupe them by content. -func (s *SubdomainScanner) resolveAll(ctx context.Context, targetID string, found map[string]bool, mu *sync.Mutex, wildcardIPs, wildcardCNAMEs map[string]bool, logFn LogFunc) map[string]bool { +func (s *SubdomainScanner) resolveAll(ctx context.Context, targetID string, found map[string]bool, mu *sync.Mutex, wildcardProfile *wildcardProfile, logFn LogFunc) map[string]bool { mu.Lock() names := make([]string, 0, len(found)) for n := range found { @@ -1153,9 +1182,9 @@ func (s *SubdomainScanner) resolveAll(ctx context.Context, targetID string, foun // the batched dnsx pass or per-name Go lookups — so both admit/reject // name-for-name identically (the gate counts below must match across paths). classify := func(name string, ips []string) { - ip := firstNonWildcardIP(ips, wildcardIPs) + ip := wildcardProfile.firstNonWildcardIP(name, ips) if ip == "" { - if len(ips) > 0 && overlapsWildcardIPs(ips, wildcardIPs) { + if len(ips) > 0 && wildcardProfile.overlapsWildcardIPs(name, ips) { cmu.Lock() rejected[name] = true wildcardHits++ @@ -1164,7 +1193,7 @@ func (s *SubdomainScanner) resolveAll(ctx context.Context, targetID string, foun } // A dangling but explicit CNAME is a real DNS asset and must remain // available to takeover checks, but it is not sent to the web pipeline. - if cname := cnameFor(name); cname != "" && !wildcardCNAMEs[cname] { + if cname := cnameFor(name); cname != "" && !wildcardProfile.isWildcardCNAME(name, cname) { cmu.Lock() cnameOnly++ cmu.Unlock() @@ -1372,9 +1401,12 @@ func (s *SubdomainScanner) recursiveEnum(ctx context.Context, targetID, apex str if len(cands) == 0 { continue } - wIPs, wCNAMEs := detectWildcard(sub, s.resolver) + wildcardProfile := detectWildcard(sub, s.resolver, s.wildcardServers(), logFn) + // Recursion probes at most 3 branch suffixes per sub-apex — a small + // budget, since up to 50 sub-apexes can run per level. + wildcardProfile.detectBranches(ctx, sub, mapKeys(cands), s.resolver, s.wildcardServers(), 3, logFn) var mu sync.Mutex - rejected := s.resolveAll(ctx, targetID, cands, &mu, wIPs, wCNAMEs, logFn) + rejected := s.resolveAll(ctx, targetID, cands, &mu, wildcardProfile, logFn) // Names that resolved (A/AAAA or CNAME) are the ones worth descending // into: everything in cands that the admission gate did NOT reject. for n := range cands { @@ -1497,68 +1529,9 @@ func (s *SubdomainScanner) dnsxCNAMECandidates(ctx context.Context, targetID str return out, targets, true } -// detectWildcard probes a few random hostnames; if they resolve, the shared -// IPs are wildcard answers we should treat with suspicion. -func detectWildcard(domain string, r *net.Resolver) (map[string]bool, map[string]bool) { - wildcardIPs := make(map[string]bool) - wildcardCNAMEs := make(map[string]bool) - probes := []string{ - "zz9z3k7q1wildcard." + domain, - "qx84jd02noexist." + domain, - "a1b2c3d4e5random." + domain, - "rcn7f4a9neverexists." + domain, - "nohost2c8e1b6d." + domain, - } - resolvedProbes := 0 - allIPs := map[string]bool{} - cnameCounts := map[string]int{} - type wildcardProbeResult struct { - ips []net.IPAddr - cname string - } - results := make(chan wildcardProbeResult, len(probes)) - var wg sync.WaitGroup - for _, p := range probes { - wg.Add(1) - go func(name string) { - defer wg.Done() - ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) - ips, _ := resolverOrDefault(r).LookupIPAddr(ctx, name) - cancel() - results <- wildcardProbeResult{ips: ips, cname: lookupExplicitCNAME(context.Background(), name, r)} - }(p) - } - wg.Wait() - close(results) - for result := range results { - if result.cname != "" { - cnameCounts[result.cname]++ - } - if len(result.ips) > 0 { - resolvedProbes++ - } - for _, ip := range result.ips { - value := ip.IP.String() - if value != "" { - allIPs[value] = true - } - } - } - // Require at least three independent random labels to resolve, then retain the - // union of their addresses. This catches rotating wildcard/CDN pools where no - // single IP necessarily appears twice, without trusting one poisoned answer. - if resolvedProbes >= 3 { - for ip := range allIPs { - wildcardIPs[ip] = true - } - } - for cname, n := range cnameCounts { - if n >= 3 { - wildcardCNAMEs[cname] = true - } - } - return wildcardIPs, wildcardCNAMEs -} +// detectWildcard in subdomain.go has been replaced by the zone-aware engine +// in wildcard.go (wildcardProfile + cross-resolver probe retry + branch +// detection). See wildcard.go. func (s *SubdomainScanner) upsertSubdomain(targetID, subdomain, ip string) error { return s.upsertSubdomainSource(targetID, subdomain, ip, "dns") @@ -1640,6 +1613,16 @@ func resolverOrDefault(r *net.Resolver) *net.Resolver { return r } +// wildcardServers returns the upstream resolver list the wildcard probes use. +// It is intentionally the SAME list the dnsx batched resolution passes supply +// via -r, so probe verdicts are drawn from the resolver pool whose answers the +// admission gate actually compares against. The first probe attempt still uses +// the shared in-process resolver, so a split-horizon zone seen from inside is +// represented too. +func (s *SubdomainScanner) wildcardServers() []string { + return s.toolResolverList() +} + func resolveHostIPs(parent context.Context, host string, r *net.Resolver) []string { ctx, cancel := context.WithTimeout(parent, 5*time.Second) defer cancel() @@ -1659,33 +1642,6 @@ func resolveHostIPs(parent context.Context, host string, r *net.Resolver) []stri return out } -func firstNonWildcardIP(ips []string, wildcard map[string]bool) string { - // If an answer overlaps the random-label wildcard profile, treat the complete - // RRset as wildcard. Rotating pools often return one familiar and one new IP; - // selecting the new address would incorrectly admit the fake hostname. - for _, ip := range ips { - if wildcard[ip] { - return "" - } - } - for _, ip := range ips { - return ip - } - return "" -} - -func overlapsWildcardIPs(ips []string, wildcard map[string]bool) bool { - if len(ips) == 0 || len(wildcard) == 0 { - return false - } - for _, ip := range ips { - if wildcard[ip] { - return true - } - } - return false -} - func lookupExplicitCNAME(parent context.Context, host string, r *net.Resolver) string { ctx, cancel := context.WithTimeout(parent, 3*time.Second) defer cancel() diff --git a/internal/scanner/wildcard.go b/internal/scanner/wildcard.go new file mode 100644 index 0000000..9776aca --- /dev/null +++ b/internal/scanner/wildcard.go @@ -0,0 +1,542 @@ +package scanner + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "net" + "sort" + "strings" + "sync" + "time" +) + +// The wildcard gate answers one question for a DNS zone: "does a random +// hostname under this suffix resolve anyway?" If it does, every passive/ +// brute "discovery" under that suffix may be a fabricated name served by the +// wildcard instead of a real asset, and the resolution gate has to prove each +// name against the wildcard's answer pool before admitting it. +// +// Reconner previously ran a single batch of 5 fixed probes and treated any +// DNS error like NXDOMAIN. Three failure modes slipped through: +// +// 1. A flaky answer (SERVFAIL, rate limit, timeout) looked identical to +// NXDOMAIN. Fewer than 3 probes resolved => no wildcard evidence => the +// whole junk namespace was admitted (the "sometimes everything looks like +// a subdomain" bug). +// 2. Rotating CDN pools can answer more than 5 probes' worth of IPs, and +// CNAME-target rotation needs more samples for the >=3 agreement rule. +// 3. Branch wildcards (e.g. *.git.example.com under example.com) never got +// probed at all — the apex probes test only the apex. A junk fleet under +// a deeper wildcard (asdf-garbage.git.example.com) was admitted wholesale. +// +// This file fixes all three: cross-resolver retry with an explicit NXDOMAIN +// verdict (net.DNSError.IsNotFound), a second independent probe batch when +// the first is inconclusive, and branch probes generated from clustered +// candidate suffixes. + +// probeVerdict classifies one random-label lookup. The distinction matters: +// probeNX is definitive proof the label does not exist, while probeAmbiguous +// (SERVFAIL/timeouts) proves nothing and must trigger more sampling. +type probeVerdict int + +const ( + probeResolved probeVerdict = iota + probeNX + probeAmbiguous +) + +// fixedWildcardProbeStems are the original five probe labels. They are kept +// byte-for-byte so a scan reacts to the same zones previous releases +// detected, and so tests are deterministic. The second (adaptive) batch uses +// fresh random labels instead, in case an operator's zone matches the fixed +// stems for some exotic reason. +var fixedWildcardProbeStems = []string{ + "zz9z3k7q1wildcard", + "qx84jd02noexist", + "a1b2c3d4e5random", + "rcn7f4a9neverexists", + "nohost2c8e1b6d", +} + +// wildcardProbeMinAgreement is how many probes must agree before a verdict is +// trusted: 3 resolving labels prove a wildcard, 3 definitive NXDOMAINs prove +// the absence of one, 3 identical CNAME targets prove a CNAME-only wildcard. +const wildcardProbeMinAgreement = 3 + +// wildcardZone carries the answer-pool evidence for ONE suffix of DNS space. +// suffix == "" marks the apex zone (covers every candidate under the target +// domain); anything else marks a strictly deeper branch zone. +type wildcardZone struct { + suffix string // "" = apex; else e.g. "git.example.com" + ips map[string]bool + cnames map[string]bool +} + +// hasEvidence reports whether the zone is a proven wildcard (any shared IP or +// CNAME answer pool). An empty zone is NOT a wildcard, including the +// "definitively NXDOMAIN" verdict. +func (z wildcardZone) hasEvidence() bool { + return len(z.ips) > 0 || len(z.cnames) > 0 +} + +// wildcardProfile bundles the apex zone with any confirmed branch zones and +// answers the admission-gate questions per candidate name. A nil profile +// behaves like "no wildcard evidence", which keeps test paths simple. +type wildcardProfile struct { + apex wildcardZone + branches []wildcardZone +} + +// normalizeSubdomainName lowercases and trims a trailing dot so wildcard +// matching is case/format insensitive. +func normalizeSubdomainName(name string) string { + return strings.ToLower(strings.TrimSuffix(strings.TrimSpace(name), ".")) +} + +// mapKeys returns the key set of a candidate map as a slice. +func mapKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + +// zonesFor returns every zone whose suffix covers name: the apex zone always +// applies, plus each branch zone whose suffix the name sits strictly under +// (name != suffix). Branch suffixes are matched on full-suffix boundaries so +// "es.git.example.com" cannot satisfy a "git.example.com" zone via a partial +// label match. +func (p *wildcardProfile) zonesFor(name string) []wildcardZone { + name = normalizeSubdomainName(name) + zones := make([]wildcardZone, 0, 3) + if p != nil { + zones = append(zones, p.apex) + for i := range p.branches { + if strings.HasSuffix(name, "."+p.branches[i].suffix) { + zones = append(zones, p.branches[i]) + } + } + } + return zones +} + +// overlapsWildcardIPs reports whether ANY address of the record set belongs to +// a wildcard answer pool applicable to this name. The "any" rule mirrors +// rotating-pool behavior: one old-pool IP plus one unfamiliar IP is still a +// wildcard answer, so the whole RRset is treated as wildcard. +func (p *wildcardProfile) overlapsWildcardIPs(name string, ips []string) bool { + if len(ips) == 0 { + return false + } + for _, zone := range p.zonesFor(name) { + if len(zone.ips) == 0 { + continue + } + for _, ip := range ips { + if zone.ips[ip] { + return true + } + } + } + return false +} + +// firstNonWildcardIP returns the first address of the record set unless the +// set overlaps a wildcard pool applicable to the name, in which case it +// returns "" (the entire RRset is rejected, matching the pre-branch behavior). +func (p *wildcardProfile) firstNonWildcardIP(name string, ips []string) string { + if p == nil { + for _, ip := range ips { + return ip + } + return "" + } + if p.overlapsWildcardIPs(name, ips) { + return "" + } + for _, ip := range ips { + return ip + } + return "" +} + +// isWildcardCNAME reports whether a candidate's CNAME target matches a +// wildcard answer pool applicable to the name. +func (p *wildcardProfile) isWildcardCNAME(name, cname string) bool { + if cname == "" { + return false + } + cname = normalizeSubdomainName(cname) + for _, zone := range p.zonesFor(name) { + if zone.cnames[cname] { + return true + } + } + return false +} + +// apexWildcard reports whether the APEX zone alone is a wildcard. Only an apex +// wildcard makes active brute-force meaningless; a branch wildcard still +// leaves the rest of the zone worth enumerating (branch hits are filtered +// per-name by the admission gate). +func (p *wildcardProfile) apexWildcard() bool { + return p != nil && p.apex.hasEvidence() +} + +// hasAnyWildcard reports whether ANY zone carries evidence (used for the +// operator-facing warning line). +func (p *wildcardProfile) hasAnyWildcard() bool { + if p == nil { + return false + } + if p.apex.hasEvidence() { + return true + } + for _, z := range p.branches { + if z.hasEvidence() { + return true + } + } + return false +} + +// singleServerResolver builds a resolver pinned to ONE upstream server so a +// retry genuinely asks a DIFFERENT resolver — buildDNSResolver rotates per +// connection, which is usually enough, but an explicit pin gives the retry +// loop a guarantee. +func singleServerResolver(server string) *net.Resolver { + res := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, _ string) (net.Conn, error) { + d := net.Dialer{Timeout: 4 * time.Second} + return d.DialContext(ctx, network, server) + }, + } + return res +} + +// resolveProbeWithRetry resolves one probe name, retrying ambiguous answers +// (SERVFAIL/timeout/refused) on DIFFERENT upstream resolvers. A definitive +// NXDOMAIN (dnsErr.IsNotFound) is recorded and, once seen, a later attempt +// that succeeds still wins — a resolver badly configured for the zone must not +// mask an existing name. +func resolveProbeWithRetry(r *net.Resolver, name string, servers []string, startOffset int) ([]net.IPAddr, probeVerdict) { + attempts := len(servers) + if attempts == 0 { + attempts = 1 // system resolver only; r carries it + } + if attempts > 3 { + attempts = 3 + } + var sawNX, sawAmbiguous bool + for attempt := 0; attempt < attempts; attempt++ { + if attempt > 0 { + // Small backoff between attempts: a rate-limited resolver answered + // the first query instantly, so give its budget a beat to recover. + time.Sleep(time.Duration(attempt) * 150 * time.Millisecond) + } + var res *net.Resolver + if attempt == 0 { + // First attempt uses the shared resolver (exactly the resolver the + // candidate admission gate re-checks against), then explicitly + // pinned servers for retries. + res = r + } else { + idx := (startOffset + attempt) % len(servers) + res = singleServerResolver(servers[idx]) + } + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + result, err := res.LookupIPAddr(ctx, name) + cancel() + if err == nil { + return result, probeResolved + } + if dnsErr, ok := err.(*net.DNSError); ok && dnsErr.IsNotFound { + sawNX = true + continue + } + sawAmbiguous = true + } + switch { + case sawNX && !sawAmbiguous: + return nil, probeNX + case sawNX && sawAmbiguous: + // One resolver says "no such name", another errored. The disagreement + // is exactly what extra probes are for; count it ambiguous so the + // caller runs a second batch instead of trusting either answer. + return nil, probeAmbiguous + default: + return nil, probeAmbiguous + } +} + +type wildcardProbe struct { + name string + ips []net.IPAddr + cname string + verdict probeVerdict +} + +// runWildcardProbes probes every name concurrently and returns the verdicts. +// CNAME lookups run only for names that were not definitively NXDOMAIN — +// LookupCNAME on an NXDOMAIN just burns its 3s timeout for nothing. +func runWildcardProbes(names []string, r *net.Resolver, servers []string) []wildcardProbe { + results := make([]wildcardProbe, len(names)) + var wg sync.WaitGroup + for i, name := range names { + wg.Add(1) + go func(i int, name string) { + defer wg.Done() + ips, verdict := resolveProbeWithRetry(r, name, servers, i) + probe := wildcardProbe{name: name, ips: ips, verdict: verdict} + if verdict != probeNX { + probe.cname = lookupExplicitCNAME(context.Background(), name, r) + } + results[i] = probe + }(i, name) + } + wg.Wait() + return results +} + +// summarizeWildcardProbes aggregates probe verdicts into the counts and answer +// pools the wildcard decision needs. +func summarizeWildcardProbes(probes []wildcardProbe) (resolved, nxDomain, ambiguous int, allIPs map[string]bool, cnameCounts map[string]int) { + allIPs = map[string]bool{} + cnameCounts = map[string]int{} + for _, probe := range probes { + switch probe.verdict { + case probeResolved: + resolved++ + case probeNX: + nxDomain++ + default: + ambiguous++ + } + for _, ip := range probe.ips { + if value := ip.IP.String(); value != "" { + allIPs[value] = true + } + } + if probe.cname != "" { + cnameCounts[probe.cname]++ + } + } + return resolved, nxDomain, ambiguous, allIPs, cnameCounts +} + +// decideWildcardZone turns probe aggregates into a zone. The bool reports +// whether the batch was inconclusive (neither side reached agreement) and a +// second batch should run. IP/CNAME pools stored in the zone are the union of +// all answers seen, guarded by the agreement threshold. +func decideWildcardZone(resolved, nxDomain int, allIPs map[string]bool, cnameCounts map[string]int, suffix string) (wildcardZone, bool) { + zone := wildcardZone{suffix: suffix, ips: map[string]bool{}, cnames: map[string]bool{}} + if resolved >= wildcardProbeMinAgreement { + // At least three independent random labels resolved: everything they + // answered is part of the wildcard's answer pool. Union all addresses + // so rotating CDN pools are captured even when no single IP repeats. + for ip := range allIPs { + zone.ips[ip] = true + } + for cname, n := range cnameCounts { + if n >= wildcardProbeMinAgreement { + zone.cnames[cname] = true + } + } + return zone, false + } + if nxDomain >= wildcardProbeMinAgreement { + // At least three labels are definitively nonexistent: this suffix is + // NOT a wildcard. An empty zone keeps the old "no evidence" behavior + // but now with proof, and the caller skips the second batch. + return zone, false + } + return zone, true +} + +// randomProbeLabel returns a hex label from crypto/rand. Probe labels are not +// security tokens, but they must be unpredictable per scan (a cached wildcard +// answer for a guessed label would poison the verdict), which crypto/rand +// guarantees for free. +func randomProbeLabel() string { + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + return fmt.Sprintf("rcnfallback%d", time.Now().UnixNano()&0xffff) + } + return hex.EncodeToString(buf) +} + +// probeWildcardZone runs the five fixed probes (plus a second random batch +// when the first is inconclusive) for one DNS suffix and returns the zone. +// fullSuffix is the target domain for the apex zone, or a deeper suffix (e.g. +// "git.example.com") for a branch zone. Zone.suffix stays "" for the apex so +// zonesFor matches it unconditionally. +func probeWildcardZone(fullSuffix, zoneSuffix string, r *net.Resolver, servers []string, logFn LogFunc) wildcardZone { + apex := zoneSuffix == "" + probes := runWildcardProbes(functionProbeNames(fixedWildcardProbeStems, fullSuffix), r, servers) + resolved, nxDomain, ambiguous, allIPs, cnameCounts := summarizeWildcardProbes(probes) + + zone, needsMore := decideWildcardZone(resolved, nxDomain, allIPs, cnameCounts, zoneSuffix) + if !needsMore { + if ambiguous > 0 && logFn != nil { + logFn("info", "subdomain_enum", fmt.Sprintf( + "Wildcard probes under %s settled: %d resolved, %d NXDOMAIN, %d errored — verdict reached anyway", + fullSuffix, resolved, nxDomain, ambiguous)) + } + return zone + } + + // Inconclusive: sample five fresh labels. If the resolver was rate-limiting + // or a CDN node was flaky, the new batch answers through the same retry + // logic with a fresh transaction budget. + if logFn != nil { + logFn("info", "subdomain_enum", fmt.Sprintf( + "Wildcard probes under %s inconclusive (%d resolved, %d NXDOMAIN, %d errored) — probing a second batch", + fullSuffix, resolved, nxDomain, ambiguous)) + } + second := make([]string, 0, len(fixedWildcardProbeStems)) + for range fixedWildcardProbeStems { + second = append(second, randomProbeLabel()+"."+fullSuffix) + } + probes = append(probes, runWildcardProbes(second, r, servers)...) + resolved, nxDomain, ambiguous, allIPs, cnameCounts = summarizeWildcardProbes(probes) + zone, _ = decideWildcardZone(resolved, nxDomain, allIPs, cnameCounts, zoneSuffix) + if !zone.hasEvidence() && logFn != nil && apex { + logFn("info", "subdomain_enum", fmt.Sprintf( + "Wildcard probes under %s inconclusive after both batches (%d resolved, %d NXDOMAIN, %d errored) — treating as non-wildcard", + fullSuffix, resolved, nxDomain, ambiguous)) + } + return zone +} + +// functionProbeNames joins probe stems with a DNS suffix. +func functionProbeNames(stems []string, suffix string) []string { + names := make([]string, len(stems)) + for i, stem := range stems { + names[i] = stem + "." + suffix + } + return names +} + +// Branch wildcard detection (the *.git.example.com case). +// +// A wildcard can live at ANY depth, not only at the apex. Junk fleets cluster: +// hundreds of candidates share "git." as their second label. Group the +// candidate list by intermediate suffix and probe the clusters that are large +// enough to smell like wildcard junk, but small enough to bound the DNS +// traffic. + +const ( + branchMinCandidates = 5 // junk fleets are large; 5 clustered labels justify up to ~15 DNS queries + branchMaxProbes = 8 // hard cap of branch probes per detection call + branchMaxDepth = 3 // examine candidate suffixes formed by dropping up to 3 leading labels +) + +// candidateBranchSuffixCounts counts candidates per intermediate suffix. A +// suffix qualifies structurally when it is strictly deeper than the apex +// (matched on label boundaries) — "com.au" alone is never probed. +func candidateBranchSuffixCounts(candidates []string, apex string) map[string]int { + apex = normalizeSubdomainName(apex) + apexLabels := strings.Split(apex, ".") + counts := map[string]int{} + for _, cand := range candidates { + name := normalizeSubdomainName(cand) + if name == "" || !strings.HasSuffix(name, "."+apex) { + continue + } + for drop := 1; drop <= branchMaxDepth; drop++ { + labels := strings.Split(name, ".") + if len(labels)-drop <= len(apexLabels) { + break + } + suffix := strings.Join(labels[drop:], ".") + counts[suffix]++ + } + } + return counts +} + +// detectBranchWildcards probes the largest clustered candidate suffixes and +// returns the confirmed branch wildcard zones (never nil; empty when nothing +// was probed or confirmed). maxBranches bounds the DNS traffic per call — the +// apex scan uses the full budget, recursion levels a smaller one. +func detectBranchWildcards(ctx context.Context, apex string, candidates []string, r *net.Resolver, servers []string, maxBranches int, logFn LogFunc) []wildcardZone { + if maxBranches <= 0 { + maxBranches = branchMaxProbes + } + counts := candidateBranchSuffixCounts(candidates, apex) + + type branchCandidate struct { + suffix string + count int + } + sorted := make([]branchCandidate, 0, len(counts)) + for suffix, count := range counts { + if count >= branchMinCandidates { + sorted = append(sorted, branchCandidate{suffix: suffix, count: count}) + } + } + if len(sorted) == 0 { + return nil + } + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].count != sorted[j].count { + return sorted[i].count > sorted[j].count + } + return sorted[i].suffix < sorted[j].suffix + }) + if len(sorted) > maxBranches { + sorted = sorted[:maxBranches] + } + + var confirmed []wildcardZone + probed := 0 + for _, bc := range sorted { + if ctx.Err() != nil { + break + } + probed++ + zone := probeWildcardZone(bc.suffix, bc.suffix, r, servers, logFn) + if zone.hasEvidence() { + confirmed = append(confirmed, zone) + } + } + if probed > 0 && logFn != nil { + logFn("info", "subdomain_enum", fmt.Sprintf( + "Branch wildcard probes: %d clustered suffix(es) probed, %d confirmed as wildcards (candidates clustered e.g. *.git.%s)", + probed, len(confirmed), apex)) + for _, zone := range confirmed { + logFn("warn", "subdomain_enum", fmt.Sprintf( + "Wildcard branch confirmed: *.%s (%d answer IP(s), %d CNAME target(s)) — names under it need DNS proof", + zone.suffix, len(zone.ips), len(zone.cnames))) + } + } + return confirmed +} + +// detectWildcard builds the full wildcard profile for a scan: the apex zone +// (probes under the target domain) with branch zones filled in separately by +// detectBranchWildcards once the candidate list is known. +func detectWildcard(domain string, r *net.Resolver, servers []string, logFn LogFunc) *wildcardProfile { + profile := &wildcardProfile{ + apex: probeWildcardZone(domain, "", r, servers, logFn), + } + if profile.apex.hasEvidence() { + logFn("warn", "subdomain_enum", fmt.Sprintf( + "Wildcard DNS detected (%d answer IP(s), %d CNAME target(s)) — passive results will be verified by resolution", + len(profile.apex.ips), len(profile.apex.cnames))) + } + return profile +} + +// detectBranches fills a profile with branch zones probed from the candidate +// list. It mutates the profile in place so the caller keeps one reference. +func (p *wildcardProfile) detectBranches(ctx context.Context, apex string, candidates []string, r *net.Resolver, servers []string, maxBranches int, logFn LogFunc) { + if p == nil || len(candidates) == 0 { + return + } + p.branches = detectBranchWildcards(ctx, apex, candidates, r, servers, maxBranches, logFn) +} diff --git a/internal/scanner/wildcard_test.go b/internal/scanner/wildcard_test.go new file mode 100644 index 0000000..1ec723d --- /dev/null +++ b/internal/scanner/wildcard_test.go @@ -0,0 +1,201 @@ +package scanner + +import ( + "net" + "sort" + "testing" +) + +func mustIP(t *testing.T, raw string) net.IPAddr { + t.Helper() + ip := net.ParseIP(raw) + if ip == nil { + t.Fatalf("bad test IP %q", raw) + } + return net.IPAddr{IP: ip} +} + +func TestCandidateBranchSuffixCounts(t *testing.T) { + apex := "realestate.com.au" + var candidates []string + for i := 0; i < 6; i++ { + candidates = append(candidates, + "junk"+string(rune('a'+i))+".git.realestate.com.au") + } + // Nested junk below a deeper label: counts both branch suffixes. + candidates = append(candidates, "word.es.git.realestate.com.au", "word2.es.git.realestate.com.au") + // Unrelated legitimate names must not create branch suffixes. + candidates = append(candidates, "mail.realestate.com.au", "api.realestate.com.au") + // The branch parent itself must not count (it equals the suffix, not a name under it). + candidates = append(candidates, "git.realestate.com.au") + candidates = append(candidates, "wikileaks.not-the-apex.org", "junk.evil-realestate.com.au") + + counts := candidateBranchSuffixCounts(candidates, apex) + if counts["git.realestate.com.au"] != 8 { + t.Fatalf("expected 8 candidates under git.realestate.com.au (nested es.git names counted too), got %d (%v)", counts["git.realestate.com.au"], counts) + } + if counts["es.git.realestate.com.au"] != 2 { + t.Fatalf("expected 2 candidates under es.git.realestate.com.au, got %d", counts["es.git.realestate.com.au"]) + } + for _, forbidden := range []string{"com.au", "realestate.com.au", "not-the-apex.org"} { + if counts[forbidden] != 0 { + t.Fatalf("suffix %q must never be counted as a branch of the apex, got %d", forbidden, counts[forbidden]) + } + } +} + +func TestWildcardProfileMatching(t *testing.T) { + profile := &wildcardProfile{ + apex: wildcardZone{ + ips: map[string]bool{"1.1.1.1": true}, + cnames: map[string]bool{ + "shared-target.realestate.com.au": true, + }, + }, + branches: []wildcardZone{ + {suffix: "git.realestate.com.au", ips: map[string]bool{"2.2.2.2": true}, cnames: map[string]bool{"git-target.realestate.com.au": true}}, + {suffix: "es.git.realestate.com.au", ips: map[string]bool{"3.3.3.3": true}}, + }, + } + + if got := profile.firstNonWildcardIP("mail.realestate.com.au", []string{"1.1.1.1"}); got != "" { + t.Fatalf("apex wildcard IP must reject, got %q", got) + } + if got := profile.firstNonWildcardIP("x.es.git.realestate.com.au", []string{"3.3.3.3"}); got != "" { + t.Fatalf("deeper branch IP must reject names under the deeper zone, got %q", got) + } + if got := profile.firstNonWildcardIP("x.git.realestate.com.au", []string{"3.3.3.3"}); got != "3.3.3.3" { + t.Fatalf("deeper-zone IP must NOT reject names outside the deeper zone, got %q", got) + } + if got := profile.firstNonWildcardIP("x.git.realestate.com.au", []string{"2.2.2.2"}); got != "" { + t.Fatalf("branch wildcard IP must reject, got %q", got) + } + if got := profile.firstNonWildcardIP("x.other.realestate.com.au", []string{"2.2.2.2"}); got != "2.2.2.2" { + t.Fatalf("branch IP must NOT reject names outside the branch, got %q", got) + } + // Label-boundary safety: "evil-git" is one label — not under *.git. + if got := profile.firstNonWildcardIP("asdf-garbage.git.realestate.com.au", []string{"9.9.9.9"}); got != "9.9.9.9" { + t.Fatalf("unrelated IP under a wildcard branch must still admit (needs HTTP proof), got %q", got) + } + if got := profile.firstNonWildcardIP("evil-git.realestate.com.au", []string{"2.2.2.2"}); got != "2.2.2.2" { + t.Fatalf("hyphenated label must not match the branch suffix, got %q (rejected would mean a false wildcard hit)", got) + } + + if profile.isWildcardCNAME("x.realestate.com.au", "shared-Target.Realestate.com.au") != true { + t.Fatalf("apex CNAME target must match case-insensitively") + } + if profile.isWildcardCNAME("x.git.realestate.com.au", "git-target.realestate.com.au") != true { + t.Fatalf("branch CNAME target must match for names under the branch") + } + if profile.isWildcardCNAME("x.other.realestate.com.au", "git-target.realestate.com.au") != false { + t.Fatalf("branch CNAME target must NOT match for names outside the branch") + } + if profile.isWildcardCNAME("x.realestate.com.au", "") != false { + t.Fatalf("empty target must never be a wildcard hit") + } + + if !profile.apexWildcard() { + t.Fatalf("apex evidence must mark the profile as apex-wildcard") + } + if !profile.hasAnyWildcard() { + t.Fatalf("branch evidence must keep the profile flagged") + } +} + +func TestWildcardProfileNilAndBoundary(t *testing.T) { + var nilProfile *wildcardProfile + if got := nilProfile.firstNonWildcardIP("a.realestate.com.au", []string{"1.1.1.1"}); got != "1.1.1.1" { + t.Fatalf("nil profile must admit, got %q", got) + } + if nilProfile.overlapsWildcardIPs("a.realestate.com.au", []string{"1.1.1.1"}) { + t.Fatalf("nil profile must never report overlaps") + } + if nilProfile.apexWildcard() || nilProfile.hasAnyWildcard() { + t.Fatalf("nil profile must not claim evidence") + } + + empty := &wildcardProfile{} + if got := empty.firstNonWildcardIP("a.realestate.com.au", []string{"1.1.1.1", "2.2.2.2"}); got != "1.1.1.1" { + t.Fatalf("empty profile must admit first IP, got %q", got) + } +} + +func TestDecideWildcardZone(t *testing.T) { + pool := map[string]bool{"10.0.0.1": true, "10.0.0.2": true, "10.0.0.3": true, "10.0.0.4": true} + cnames := map[string]int{"edge.realestate.com.au": 4} + + zone, needsMore := decideWildcardZone(3, 0, pool, cnames, "git.realestate.com.au") + if needsMore { + t.Fatalf("3 resolving probes must settle the zone") + } + if len(zone.ips) != 4 { + t.Fatalf("rotating pool: all probe IPs belong to the wildcard pool, got %v", zone.ips) + } + if len(zone.cnames) != 1 { + t.Fatalf("4 aggressive CNAME agreements must produce the CNAME pool, got %v", zone.cnames) + } + + zone, needsMore = decideWildcardZone(0, 3, pool, cnames, "git.realestate.com.au") + if needsMore || zone.hasEvidence() { + t.Fatalf("3 definitive NXDOMAINs must settle an empty zone (needsMore=%v evidence=%v)", needsMore, zone.hasEvidence()) + } + + _, needsMore = decideWildcardZone(2, 0, pool, cnames, "git.realestate.com.au") + if !needsMore { + t.Fatalf("below agreement threshold must request a second batch") + } + + _, needsMore = decideWildcardZone(2, 1, map[string]bool{}, map[string]int{}, "git.realestate.com.au") + if !needsMore { + t.Fatalf("2 resolved + 1 NX is still inconclusive") + } + + zone, needsMore = decideWildcardZone(1, 3, pool, cnames, "git.realestate.com.au") + if needsMore { + t.Fatalf("3 NX dominate a single resolve — that resolver disagrees, majority NX wins") + } + if zone.hasEvidence() { + t.Fatalf("majority-NX verdict must not create an answer pool") + } +} + +func TestSummarizeWildcardProbes(t *testing.T) { + probes := []wildcardProbe{ + {verdict: probeResolved, ips: []net.IPAddr{mustIP(t, "10.1.1.1"), mustIP(t, "10.1.1.2")}, cname: "a.realestate.com.au"}, + {verdict: probeResolved, ips: []net.IPAddr{mustIP(t, "10.1.1.1")}}, + {verdict: probeNX}, + {verdict: probeNX}, + {verdict: probeNX}, + {verdict: probeAmbiguous, cname: "b.realestate.com.au"}, + } + resolved, nx, ambiguous, allIPs, cnameCounts := summarizeWildcardProbes(probes) + if resolved != 2 || nx != 3 || ambiguous != 1 { + t.Fatalf("verdict histogram mismatch: resolved=%d nx=%d ambiguous=%d", resolved, nx, ambiguous) + } + if len(allIPs) != 2 || !allIPs["10.1.1.1"] || !allIPs["10.1.1.2"] { + t.Fatalf("IP union mismatch: %v", allIPs) + } + if cnameCounts["a.realestate.com.au"] != 1 || cnameCounts["b.realestate.com.au"] != 1 { + t.Fatalf("CNAME counts mismatch: %v", cnameCounts) + } +} + +func TestDetectBranchWildcardsOrderAndCap(t *testing.T) { + apex := "realestate.com.au" + var candidates []string + for i := 0; i < 9; i++ { + candidates = append(candidates, "junk"+string(rune('a'+i))+".git.realestate.com.au") + } + counts := candidateBranchSuffixCounts(candidates, apex) + if len(counts) != 1 || counts["git.realestate.com.au"] != 9 { + t.Fatalf("cluster must reduce to one suffix: %v", counts) + } + keys := make([]string, 0, len(counts)) + for k := range counts { + keys = append(keys, k) + } + sort.Strings(keys) + if keys[0] != "git.realestate.com.au" { + t.Fatalf("unexpected suffix: %v", keys) + } +} From 61a28bf54cdccde6cae423ce87c8de363b24dee4 Mon Sep 17 00:00:00 2001 From: void Date: Sun, 25 Oct 2026 06:11:40 +0330 Subject: [PATCH 09/14] feat(http): opt-in HTTP catch-all grouping pass (default off) Mark subdomain rows that serve identical wildcard content: a fleet of junk hosts that all redirect to the apex (>=3 members) or clone the apex homepage (>=5 members, apex itself included in the group). Reads only ALREADY-stored http_services rows - no extra network requests - and runs at the end of http_probe when enabled. Rows are marked via the new subdomains.catchall column and hidden from the Hosts tab, reports, graph, stats and wordlist learning; their http_services rows are deleted so JS/params/DAST/nuclei never see them. Seeds and vhost rows are never touched. DNS rows keep the mark across rescans; the probe loader skips them. Default OFF; enable via System -> Toolchain -> Scanning configuration. --- frontend/src/lib/api.ts | 1 + frontend/src/pages/System.tsx | 11 +- internal/api/finding_handlers.go | 4 +- internal/api/graph_handlers.go | 2 +- internal/api/htmlreport.go | 6 +- internal/api/pdfreport.go | 2 +- internal/api/system_handlers.go | 7 + internal/api/target_handlers.go | 10 +- internal/config/config.go | 10 + internal/database/migrations.go | 9 + internal/scanner/adaptive_wordlist.go | 2 +- internal/scanner/http.go | 13 +- internal/scanner/http_catchall.go | 285 +++++++++++++++++++++++++ internal/scanner/http_catchall_test.go | 167 +++++++++++++++ internal/scanner/shodan.go | 2 +- 15 files changed, 515 insertions(+), 16 deletions(-) create mode 100644 internal/scanner/http_catchall.go create mode 100644 internal/scanner/http_catchall_test.go diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 9cfe5c7..e653961 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -337,6 +337,7 @@ export interface ScanConfig { dns_resolvers: string[] subdomain_recursion_depth: number subdomain_recursion_enabled: boolean + http_catchall_detection: boolean nuclei_exclude_template_ids: string[] nuclei_extra_template_repos: string[] monitor_modules: string[] diff --git a/frontend/src/pages/System.tsx b/frontend/src/pages/System.tsx index eb815dd..63ca8ee 100644 --- a/frontend/src/pages/System.tsx +++ b/frontend/src/pages/System.tsx @@ -40,7 +40,7 @@ export default function System() { const [drafts, setDrafts] = useState>({}) const [savingKeys, setSavingKeys] = useState(false) const [scanCfg, setScanCfg] = useState(null) - const [scanDraft, setScanDraft] = useState<{ resolvers: string; depth: number; enabled: boolean; excludeIDs: string; repos: string; monitor: string[] } | null>(null) + const [scanDraft, setScanDraft] = useState<{ resolvers: string; depth: number; enabled: boolean; catchall: boolean; excludeIDs: string; repos: string; monitor: string[] } | null>(null) const [savingScanCfg, setSavingScanCfg] = useState(false) const { addToast } = useUIStore() const isAdmin = useAuthStore(s => s.user?.role === 'admin') @@ -62,6 +62,7 @@ export default function System() { resolvers: (sc.dns_resolvers || []).join('\n'), depth: sc.subdomain_recursion_depth || 4, enabled: sc.subdomain_recursion_enabled !== false, + catchall: sc.http_catchall_detection === true, excludeIDs: (sc.nuclei_exclude_template_ids || []).join('\n'), repos: (sc.nuclei_extra_template_repos || []).join('\n'), // Empty on the wire ⇒ the built-in default watch set applies. @@ -115,6 +116,7 @@ export default function System() { dns_resolvers: splitLines(scanDraft.resolvers), subdomain_recursion_depth: scanDraft.depth, subdomain_recursion_enabled: scanDraft.enabled, + http_catchall_detection: scanDraft.catchall, nuclei_exclude_template_ids: splitLines(scanDraft.excludeIDs), nuclei_extra_template_repos: splitLines(scanDraft.repos), monitor_modules: scanDraft.monitor, @@ -124,6 +126,7 @@ export default function System() { resolvers: (res.dns_resolvers || []).join('\n'), depth: res.subdomain_recursion_depth || 4, enabled: res.subdomain_recursion_enabled !== false, + catchall: res.http_catchall_detection === true, excludeIDs: (res.nuclei_exclude_template_ids || []).join('\n'), repos: (res.nuclei_extra_template_repos || []).join('\n'), monitor: (res.monitor_modules && res.monitor_modules.length ? res.monitor_modules : DEFAULT_WATCH_MODULES), @@ -342,6 +345,12 @@ export default function System() {

After the apex, each resolving subdomain is itself enumerated, down to this many levels (default 4). Off or depth 1 ⇒ apex only.

+ +

Optional, off by default. After HTTP probing, groups 2xx hosts that all serve one wildcard page — redirect-to-apex fleets or homepage clones (the "junk.git.target" catch-all case) — and hides them from the Hosts tab, reports, graph and web modules. Reads stored responses only, no extra requests. Seeds and vhosts are never touched.

diff --git a/internal/api/finding_handlers.go b/internal/api/finding_handlers.go index 477480b..83acc44 100644 --- a/internal/api/finding_handlers.go +++ b/internal/api/finding_handlers.go @@ -64,7 +64,7 @@ func (h *Handler) handleListSubdomains(w http.ResponseWriter, r *http.Request) { query := `SELECT id, target_id, subdomain, ip, is_alive, status_code, page_title, has_https, has_http, response_time_ms, technologies, server, framework, cdn, waf, COALESCE(source,'dns'), last_seen, created_at - FROM subdomains WHERE target_id = ?` + FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0) = 0` args := []any{id} if alive == "true" { @@ -79,7 +79,7 @@ func (h *Handler) handleListSubdomains(w http.ResponseWriter, r *http.Request) { } var total int - countQuery := "SELECT COUNT(*) FROM subdomains WHERE target_id = ?" + countQuery := "SELECT COUNT(*) FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0) = 0" countArgs := []any{id} if alive == "true" { countQuery += " AND is_alive = 1" diff --git a/internal/api/graph_handlers.go b/internal/api/graph_handlers.go index abdb488..dfe1389 100644 --- a/internal/api/graph_handlers.go +++ b/internal/api/graph_handlers.go @@ -67,7 +67,7 @@ func (h *Handler) buildGraph(r *http.Request, targetID string) ([]graphNode, []g // Subdomains ↔ IPs (co-location reveals pivot opportunities). ipHosts := map[string][]string{} subTech := map[string]string{} - rows, err := h.db.QueryContext(ctx, `SELECT subdomain, ip, technologies FROM subdomains WHERE target_id = ? AND is_alive = 1`, targetID) + rows, err := h.db.QueryContext(ctx, `SELECT subdomain, ip, technologies FROM subdomains WHERE target_id = ? AND is_alive = 1 AND COALESCE(catchall,0) = 0`, targetID) if err == nil { for rows.Next() { var sub, ip, techs string diff --git a/internal/api/htmlreport.go b/internal/api/htmlreport.go index ff26e7a..463c929 100644 --- a/internal/api/htmlreport.go +++ b/internal/api/htmlreport.go @@ -46,8 +46,8 @@ func (h *Handler) handleGenerateReportHTML(w http.ResponseWriter, r *http.Reques h.db.QueryRowContext(ctx, q, id).Scan(&n) counts[key] = n } - countInto("subs", "SELECT COUNT(*) FROM subdomains WHERE target_id=?") - countInto("alive", "SELECT COUNT(*) FROM subdomains WHERE target_id=? AND is_alive=1") + countInto("subs", "SELECT COUNT(*) FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0") + countInto("alive", "SELECT COUNT(*) FROM subdomains WHERE target_id=? AND is_alive=1 AND COALESCE(catchall,0)=0") countInto("hosts", "SELECT COUNT(*) FROM http_services WHERE target_id=? AND COALESCE(source,'probe')='probe'") countInto("js", "SELECT COUNT(*) FROM js_findings WHERE target_id=?") countInto("params", "SELECT COUNT(*) FROM parameters WHERE target_id=? AND is_reflected=1") @@ -126,7 +126,7 @@ func (h *Handler) handleGenerateReportHTML(w http.ResponseWriter, r *http.Reques h.section(ctx, &b, id, "subs", "All Subdomains", ` SELECT subdomain, CASE is_alive WHEN 1 THEN 'alive' ELSE '' END, COALESCE(ip,''), COALESCE(server,'') - FROM subdomains WHERE target_id=? ORDER BY is_alive DESC, subdomain LIMIT 5000`, + FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0 ORDER BY is_alive DESC, subdomain LIMIT 5000`, []string{"Subdomain", "Alive", "IP", "Server"}, -1) b.WriteString(reportFoot()) diff --git a/internal/api/pdfreport.go b/internal/api/pdfreport.go index 2954834..cfa6672 100644 --- a/internal/api/pdfreport.go +++ b/internal/api/pdfreport.go @@ -398,7 +398,7 @@ func (h *Handler) pdfReconSections(r *http.Request, p *pdfBuilder, id string) { p.rule() } - section("Subdomains", `SELECT subdomain, CASE is_alive WHEN 1 THEN 'alive' ELSE '' END, COALESCE(ip,'') FROM subdomains WHERE target_id=? ORDER BY subdomain LIMIT 1500`, 3) + section("Subdomains", `SELECT subdomain, CASE is_alive WHEN 1 THEN 'alive' ELSE '' END, COALESCE(ip,'') FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0 ORDER BY subdomain LIMIT 1500`, 3) section("Live HTTP Hosts", `SELECT url, status_code, COALESCE(title,'') FROM http_services WHERE target_id=? AND COALESCE(source,'probe')='probe' ORDER BY url LIMIT 1000`, 3) section("Reflected Parameters", `SELECT url, parameter FROM parameters WHERE target_id=? AND is_reflected=1 ORDER BY url LIMIT 800`, 2) section("Open Redirects", `SELECT url, redirect_to FROM open_redirect_findings WHERE target_id=? AND COALESCE(status,'finding')='finding' ORDER BY url LIMIT 500`, 2) diff --git a/internal/api/system_handlers.go b/internal/api/system_handlers.go index fada8ee..f8bf1b7 100644 --- a/internal/api/system_handlers.go +++ b/internal/api/system_handlers.go @@ -101,6 +101,7 @@ type scanConfigPayload struct { DNSResolvers []string `json:"dns_resolvers"` SubdomainRecursionDepth int `json:"subdomain_recursion_depth"` SubdomainRecursionEnabled bool `json:"subdomain_recursion_enabled"` + HTTPCatchallDetection bool `json:"http_catchall_detection"` NucleiExcludeTemplateIDs []string `json:"nuclei_exclude_template_ids"` NucleiExtraTemplateRepos []string `json:"nuclei_extra_template_repos"` MonitorModules []string `json:"monitor_modules"` @@ -120,6 +121,7 @@ func (h *Handler) handleGetScanConfig(w http.ResponseWriter, _ *http.Request) { DNSResolvers: h.cfg.DNSResolvers, SubdomainRecursionDepth: h.cfg.SubdomainRecursionDepth, SubdomainRecursionEnabled: h.cfg.SubdomainRecursionEnabled, + HTTPCatchallDetection: h.cfg.HTTPCatchallDetection, NucleiExcludeTemplateIDs: h.cfg.NucleiExcludeTemplateIDs, NucleiExtraTemplateRepos: h.cfg.NucleiExtraTemplateRepos, MonitorModules: h.cfg.MonitorModules, @@ -137,6 +139,7 @@ func (h *Handler) handleUpdateScanConfig(w http.ResponseWriter, r *http.Request) DNSResolvers *[]string `json:"dns_resolvers"` SubdomainRecursionDepth *int `json:"subdomain_recursion_depth"` SubdomainRecursionEnabled *bool `json:"subdomain_recursion_enabled"` + HTTPCatchallDetection *bool `json:"http_catchall_detection"` NucleiExcludeTemplateIDs *[]string `json:"nuclei_exclude_template_ids"` NucleiExtraTemplateRepos *[]string `json:"nuclei_extra_template_repos"` MonitorModules *[]string `json:"monitor_modules"` @@ -174,6 +177,10 @@ func (h *Handler) handleUpdateScanConfig(w http.ResponseWriter, r *http.Request) // No clamping — a bool is a bool. Omitted ⇒ unchanged, like the other knobs. h.cfg.SubdomainRecursionEnabled = *body.SubdomainRecursionEnabled } + if body.HTTPCatchallDetection != nil { + // Opt-in-by-default HTTP catch-all grouping; omitted ⇒ unchanged. + h.cfg.HTTPCatchallDetection = *body.HTTPCatchallDetection + } if body.NucleiExcludeTemplateIDs != nil { h.cfg.NucleiExcludeTemplateIDs = clean(*body.NucleiExcludeTemplateIDs) } diff --git a/internal/api/target_handlers.go b/internal/api/target_handlers.go index 43c1ab3..9ed2551 100644 --- a/internal/api/target_handlers.go +++ b/internal/api/target_handlers.go @@ -1561,8 +1561,8 @@ func (h *Handler) handleTargetStats(w http.ResponseWriter, r *http.Request) { query string args []any }{ - {&stats.Subdomains, "SELECT COUNT(*) FROM subdomains WHERE target_id = ?", []any{id}}, - {&stats.AliveHosts, "SELECT COUNT(*) FROM subdomains WHERE target_id = ? AND is_alive = 1", []any{id}}, + {&stats.Subdomains, "SELECT COUNT(*) FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0) = 0", []any{id}}, + {&stats.AliveHosts, "SELECT COUNT(*) FROM subdomains WHERE target_id = ? AND is_alive = 1 AND COALESCE(catchall,0) = 0", []any{id}}, {&stats.HTTPServices, "SELECT COUNT(*) FROM http_services WHERE target_id = ?", []any{id}}, {&stats.JSFiles, "SELECT COUNT(*) FROM js_files WHERE target_id = ?", []any{id}}, {&stats.JSFindings, "SELECT COUNT(*) FROM js_findings WHERE target_id = ?", []any{id}}, @@ -1974,7 +1974,7 @@ func (h *Handler) mdReconSections(r *http.Request, b *strings.Builder, id string } } - rowsList("Subdomains", `SELECT subdomain, CASE is_alive WHEN 1 THEN 'alive' ELSE '' END, COALESCE(ip,'') FROM subdomains WHERE target_id=? ORDER BY subdomain LIMIT 5000`, 3) + rowsList("Subdomains", `SELECT subdomain, CASE is_alive WHEN 1 THEN 'alive' ELSE '' END, COALESCE(ip,'') FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0 ORDER BY subdomain LIMIT 5000`, 3) rowsList("Live HTTP Hosts", `SELECT url, status_code, COALESCE(title,'') FROM http_services WHERE target_id=? AND COALESCE(source,'probe')='probe' ORDER BY url LIMIT 3000`, 3) rowsList("Reflected Parameters", `SELECT url, parameter FROM parameters WHERE target_id=? AND is_reflected=1 ORDER BY url LIMIT 2000`, 2) rowsList("Open Redirects", `SELECT url, redirect_to FROM open_redirect_findings WHERE target_id=? ORDER BY url`, 2) @@ -2014,7 +2014,7 @@ func (h *Handler) handleExportTarget(w http.ResponseWriter, r *http.Request) { writer := csv.NewWriter(w) writer.Write([]string{"type", "url", "details"}) - rows, _ := h.db.QueryContext(r.Context(), "SELECT url, status_code FROM subdomains WHERE target_id = ?", id) + rows, _ := h.db.QueryContext(r.Context(), "SELECT url, status_code FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0) = 0", id) if rows != nil { for rows.Next() { var url string @@ -2029,7 +2029,7 @@ func (h *Handler) handleExportTarget(w http.ResponseWriter, r *http.Request) { default: export := map[string]any{"target": domain} - rows, _ := h.db.QueryContext(r.Context(), "SELECT subdomain, ip FROM subdomains WHERE target_id = ?", id) + rows, _ := h.db.QueryContext(r.Context(), "SELECT subdomain, ip FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0) = 0", id) if rows != nil { var subs []map[string]string for rows.Next() { diff --git a/internal/config/config.go b/internal/config/config.go index 2ee5568..51df515 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -240,6 +240,16 @@ type Config struct { // Default true so an upgrade never silently changes scan behavior. Set via // the System → Scanning panel, next to the depth field. SubdomainRecursionEnabled bool `json:"subdomain_recursion_enabled"` + // HTTPCatchallDetection is OPT-IN (default false). When enabled, the end of + // the HTTP-probe module groups stored 2xx http_services fingerprints and + // marks subdomains that serve a wildcard catch-all page — either a redirect + // fleet aimed at the apex, or >=5 live hosts byte-identical to the apex + // homepage. Marked hosts are excluded from the Hosts tab, reports, graph and + // wordlist learning; their rows stay stored (marked, not deleted: seeds, + // vhosts and anything the operator vouched for are never touched). The pass + // reads already-stored responses and sends NO extra network requests; it + // costs under a second. Toggle via the System → Scanning panel. + HTTPCatchallDetection bool `json:"http_catchall_detection"` // ScanWatchdogHours bounds one scan PHASE so a wedged module can never hold // its concurrency slot forever (the actual purpose — see scheduler.go). It // is deliberately reset after every completed phase, so the combined runtime diff --git a/internal/database/migrations.go b/internal/database/migrations.go index d57ea0a..80c77dd 100644 --- a/internal/database/migrations.go +++ b/internal/database/migrations.go @@ -161,6 +161,7 @@ func RunMigrations(db *DB) error { alterJSFilesAddOldHash, alterJSFilesAddOldBody, createWPSitesTable, + alterSubdomainsAddCatchall, } for i, m := range migrations { @@ -1075,6 +1076,14 @@ CREATE TABLE IF NOT EXISTS subdomains ( // hosts, which are often internal/forgotten and high-value. const alterSubdomainsAddSource = `ALTER TABLE subdomains ADD COLUMN source TEXT DEFAULT 'dns';` +// catchall marks subdomains whose HTTP responses match the target's wildcard +// catch-all fleet (redirect-to-apex groups or homepage-content clones). Rows are +// marked, never deleted: operator seeds, vhost findings and previously-alive +// evidence stay untouched, and resurfaces of the same page are idempotent. +// Human-facing surfaces (Hosts tab, reports, graph, adaptive wordlist) filter +// COALESCE(catchall,0)=0. Written only when the opt-in config flag is set. +const alterSubdomainsAddCatchall = `ALTER TABLE subdomains ADD COLUMN catchall INTEGER NOT NULL DEFAULT 0;` + // ── P1 canonical lifecycle bridge ──────────────────────────────────────────── // candidates is the AUTHORITATIVE lifecycle state source. state_version is a // monotonic counter used for optimistic, concurrency-safe transitions. diff --git a/internal/scanner/adaptive_wordlist.go b/internal/scanner/adaptive_wordlist.go index 53064df..9e1f9a0 100644 --- a/internal/scanner/adaptive_wordlist.go +++ b/internal/scanner/adaptive_wordlist.go @@ -148,7 +148,7 @@ func buildAdaptiveWordlist(ctx context.Context, db *database.DB, cfg *config.Con } rows.Close() } - rows, err = db.QueryContext(ctx, `SELECT subdomain FROM subdomains WHERE target_id=? LIMIT 10000`, targetID) + rows, err = db.QueryContext(ctx, `SELECT subdomain FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0 LIMIT 10000`, targetID) if err == nil { for rows.Next() { var host string diff --git a/internal/scanner/http.go b/internal/scanner/http.go index a59f925..2d5c19f 100644 --- a/internal/scanner/http.go +++ b/internal/scanner/http.go @@ -53,9 +53,10 @@ func (s *HTTPScanner) Run(ctx context.Context, targetID string, logFn LogFunc) e logFn("info", "http_probe", "Loading subdomains for HTTP probing...") rows, err := s.db.QueryContext(ctx, ` - SELECT subdomain FROM subdomains + SELECT subdomain FROM subdomains WHERE target_id = ? AND (COALESCE(ip,'') != '' OR COALESCE(source,'dns') IN ('seed','vhost')) + AND COALESCE(catchall,0) = 0 ORDER BY subdomain `, targetID) if err != nil { @@ -203,6 +204,16 @@ func (s *HTTPScanner) Run(ctx context.Context, targetID string, logFn LogFunc) e // false positives on it. s.fingerprintHosts(ctx, targetID, logFn) + // Opt-in HTTP catch-all pass (config http_catchall_detection, default OFF): + // groups the 2xx fingerprints gathered above and marks wildcard catch-all + // fleets (redirect-to-apex / homepage clones) so they stop polluting the + // Hosts tab, reports and web modules. Reads stored data only — no requests. + if s.cfg != nil && s.cfg.HTTPCatchallDetection { + if err := s.runCatchallPass(ctx, targetID, logFn); err != nil && ctx.Err() == nil { + logFn("warn", "http_probe", fmt.Sprintf("HTTP catch-all pass failed: %v", err)) + } + } + // Authenticated-session preflight: validate configured identities NOW, at the // front of the scan, so an expired login is surfaced before every downstream // injection/authz module wastes itself against the logged-out app. diff --git a/internal/scanner/http_catchall.go b/internal/scanner/http_catchall.go new file mode 100644 index 0000000..09c763e --- /dev/null +++ b/internal/scanner/http_catchall.go @@ -0,0 +1,285 @@ +package scanner + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "sort" + "strings" +) + +// The HTTP catch-all pass closes the hole the DNS wildcard gate cannot see: +// a zone with NO DNS wildcard can still serve one shared homepage or one +// redirect for EVERY hostname under it (portslike hosting, parking, SaaS +// platforms). Those names arrive as fully "verified" subdomains — the DNS gate +// happily passes them — and flood the Hosts tab, reports and every web module +// downstream. +// +// The pass runs at the END of the HTTP-probe module (only when the operator +// enabled it in config; default is OFF) and groups the http_services rows that +// the probing ALREADY collected: +// +// - Redirect groups: 3+ hosts that share one fingerprint AND redirect to the +// apex (or www-apex). This is the classic wildcard redirect the operator +// reported ("asdf-garbage.git." style junk pointing at the homepage). +// - Homepage clones: 5+ hosts byte-identical (status), title-, server-, +// content-type/length-, tech-, WAF/CMS-fingerprint-identical to the apex +// homepage that was probed in the same run. +// +// Grouping is 2xx-only by design: 401/403 auth walls legitimately serve the +// same shell for every host and must never be marked; per-host SSO redirects +// carry state that makes their fingerprints differ. The residual risk is a +// legit byte-identical 200-fleet that is NOT apex-related — rare, and +// reversible since rows are marked, never deleted, and the whole pass is a +// config toggle. +// +// Zero network cost: no DNS, no extra HTTP requests — the pass reads the +// fingerprints stored seconds earlier. + +const ( + catchallMinRedirectGroup = 3 // redirect-to-apex fleets need less agreement: pointing at the apex is the wildcard's signature + catchallMinContentGroup = 5 // homepage cloning is only "catch-all-shaped" with more members; protects legitimate twins (staging/prod mirrors) +) + +// hostFromServiceURL extracts and normalizes the hostname of an http_services +// URL. Probing stores scheme://host[:port][/path]; the subdomain matching only +// cares about the registered hostname part. +func hostFromServiceURL(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + if u, err := url.Parse(raw); err == nil && u.Hostname() != "" { + return strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) + } + // Fallback for scheme-less or malformed rows: cut the authority. + rest := raw + rest = strings.TrimPrefix(rest, "https://") + rest = strings.TrimPrefix(rest, "http://") + if idx := strings.IndexAny(rest, "/?#"); idx >= 0 { + rest = rest[:idx] + } + if idx := strings.LastIndex(rest, "@"); idx >= 0 { // strip userinfo + rest = rest[idx+1:] + } + if idx := strings.Index(rest, ":"); idx >= 0 { // strip port + rest = rest[:idx] + } + return strings.ToLower(strings.TrimSuffix(rest, ".")) +} + +// sortedTechnologies normalizes the stored technologies JSON so two hosts with +// the same tech stack in different order share one fingerprint. +func sortedTechnologies(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" || raw == "[]" { + return "" + } + var techs []string + if err := json.Unmarshal([]byte(raw), &techs); err != nil { + return strings.ToLower(raw) // tolerate non-JSON rows; still comparable + } + sort.Strings(techs) + return strings.ToLower(strings.Join(techs, ",")) +} + +type catchallRow struct { + host string + contentFP string // status|title|server|content-type|length|techs|waf|cms + redirectHost string // lowercase target hostname, "" when no redirect +} + +// buildCatchallRow computes the grouping fields of one http_services row. +func buildCatchallRow(rawURL string, statusCode int, title, server, contentType string, contentLength int, redirectURL, technologies, waf, cms string) catchallRow { + var fp strings.Builder + fmt.Fprintf(&fp, "%d|%s|%s|%s|%d|%s|%s|%s", + statusCode, strings.ToLower(strings.TrimSpace(title)), strings.ToLower(strings.TrimSpace(server)), + strings.ToLower(strings.TrimSpace(contentType)), contentLength, + sortedTechnologies(technologies), strings.ToLower(strings.TrimSpace(waf)), strings.ToLower(strings.TrimSpace(cms))) + return catchallRow{ + host: hostFromServiceURL(rawURL), + contentFP: fp.String(), + redirectHost: normalizeSubdomainName(hostFromServiceURL(redirectURL)), + } +} + +// catchallApexTargets reports the hostnames a wildcard redirect aims at: the +// apex itself or the canonical www host. +func catchallApexTargets(apex string) map[string]bool { + targets := map[string]bool{apex: true, "www." + apex: true} + return targets +} + +// groupCatchallFingerprints groups rows by content fingerprint. +func groupCatchallFingerprints(rows []catchallRow) map[string][]catchallRow { + groups := map[string][]catchallRow{} + for _, row := range rows { + groups[row.contentFP] = append(groups[row.contentFP], row) + } + return groups +} + +// qualifiesCatchallGroup decides whether a fingerprint group is a wildcard +// fleet and returns the members to mark. rows must exclude non-2xx entries. +// The apex and www hosts are never in the returned set: they ARE the origin +// the wildcard copies, so marking them would hide the real site. +func qualifiesCatchallGroup(rows []catchallRow, apexTargets map[string]bool) (marked []catchallRow, kind string) { + if len(rows) == 0 { + return nil, "" + } + redirecting := 0 + hasApexMember := false + hasForeignRedirect := false + for _, row := range rows { + if row.redirectHost != "" { + if apexTargets[row.redirectHost] { + redirecting++ + } else { + // One member redirects somewhere else entirely — a mixed + // group of distinct apps, not a wildcard funnel. + hasForeignRedirect = true + } + } + if apexTargets[row.host] { + hasApexMember = true + } + } + if redirecting >= catchallMinRedirectGroup && !hasForeignRedirect { + // At least three identical pages funnel into the apex and NOTHING in + // the group redirects elsewhere: a wildcard redirect. The apex/www + // rows stay unmarked (they ARE the origin). + for _, row := range rows { + if !apexTargets[row.host] { + marked = append(marked, row) + } + } + return marked, "redirect" + } + if hasApexMember && len(rows) >= catchallMinContentGroup { + // The apex homepage itself sits inside a large identical-content + // group: every member serves the apex's homepage — a content + // catch-all. The apex/www rows stay unmarked (they ARE the origin); + // strictly-deeper members get marked. + for _, row := range rows { + if !apexTargets[row.host] { + marked = append(marked, row) + } + } + return marked, "homepage" + } + return nil, "" +} + +// runCatchallPass implements the opt-in step-2 pass. It resolves the target's +// apex from the targets row, fingerprints all stored 2xx services, marks the +// wildcard fleet members (DNS-source rows only) and drops their +// http_services rows so web modules never fetch them again. +func (s *HTTPScanner) runCatchallPass(ctx context.Context, targetID string, logFn LogFunc) error { + var apex string + if err := s.db.QueryRowContext(ctx, `SELECT domain FROM targets WHERE id=?`, targetID).Scan(&apex); err != nil || strings.TrimSpace(apex) == "" { + // Unknown target — the pass is best-effort; probing already logged context. + return nil + } + apex = normalizeSubdomainName(apex) + + rows, err := s.db.QueryContext(ctx, ` + SELECT url, status_code, title, COALESCE(server,''), COALESCE(content_type,''), + COALESCE(content_length,0), COALESCE(redirect_url,''), COALESCE(technologies,'[]'), + COALESCE(waf,''), COALESCE(cms,'') + FROM http_services + WHERE target_id=? AND status_code BETWEEN 200 AND 299`, targetID) + if err != nil { + return nil + } + defer rows.Close() + + var candidateRows []catchallRow + var total2xx int + for rows.Next() { + var rawURL, title, server, contentType, redirectURL, technologies, waf, cms string + var statusCode, contentLength int + if rows.Scan(&rawURL, &statusCode, &title, &server, &contentType, &contentLength, &redirectURL, &technologies, &waf, &cms) != nil { + continue + } + total2xx++ + row := buildCatchallRow(rawURL, statusCode, title, server, contentType, contentLength, redirectURL, technologies, waf, cms) + if row.host == "" { + continue + } + // Apex/www rows stay in the candidate pool (they anchor the homepage + // group) but can only ever be marked by group logic, never as strays. + candidateRows = append(candidateRows, row) + } + rows.Close() + + if len(candidateRows) == 0 { + logFn("info", "http_probe", fmt.Sprintf("HTTP catch-all pass: %d stored 2xx service(s) — nothing to group", total2xx)) + return nil + } + + apexTargets := catchallApexTargets(apex) + groups := groupCatchallFingerprints(candidateRows) + + var markedHosts []string + redirectGroups, homepageGroups := 0, 0 + for _, members := range groups { + marked, kind := qualifiesCatchallGroup(members, apexTargets) + if kind == "" { + continue + } + if kind == "redirect" { + redirectGroups++ + } else { + homepageGroups++ + } + for _, row := range marked { + markedHosts = append(markedHosts, row.host) + logFn("info", "http_probe", fmt.Sprintf("HTTP catch-all: %s serves the wildcard group (%s pattern, group of %d)", row.host, kind, len(members))) + } + } + + if len(markedHosts) == 0 { + logFn("info", "http_probe", fmt.Sprintf("HTTP catch-all pass: %d live 2xx host(s) analysed — no catch-all groups", total2xx)) + return nil + } + + sort.Strings(markedHosts) + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + stmt, err := tx.PrepareContext(ctx, `UPDATE subdomains SET catchall=1 + WHERE target_id=? AND subdomain=? AND COALESCE(source,'dns')='dns' AND COALESCE(catchall,0)=0`) + if err != nil { + _ = tx.Rollback() + return err + } + urlStmt, err := tx.PrepareContext(ctx, `DELETE FROM http_services + WHERE target_id=? AND (url LIKE 'https://'||?||'/%' OR url LIKE 'https://'||?||':%' + OR url LIKE 'http://'||?||'/%' OR url LIKE 'http://'||?||':%' OR url='https://'||? OR url='http://'||?)`) + if err != nil { + _ = tx.Rollback() + return err + } + marked := 0 + for _, host := range markedHosts { + res, _ := stmt.ExecContext(ctx, targetID, host) + if affRows, aerr := res.RowsAffected(); aerr == nil && affRows > 0 { + marked++ + // Drop the junk services so JS/params/DAST/nuclei never target + // them again. The subdomain row keeps the evidence of WHY. + _, _ = urlStmt.ExecContext(ctx, targetID, host, host, host, host, host, host) + } + } + _ = stmt.Close() + _ = urlStmt.Close() + if err := tx.Commit(); err != nil { + return err + } + + logFn("info", "http_probe", fmt.Sprintf( + "HTTP catch-all pass: %d live 2xx host(s) analysed — %d wildcard group(s) (%d redirect, %d homepage), %d host(s) marked and hidden", + total2xx, redirectGroups+homepageGroups, redirectGroups, homepageGroups, marked)) + return nil +} diff --git a/internal/scanner/http_catchall_test.go b/internal/scanner/http_catchall_test.go new file mode 100644 index 0000000..b7c50d0 --- /dev/null +++ b/internal/scanner/http_catchall_test.go @@ -0,0 +1,167 @@ +package scanner + +import ( + "context" + "fmt" + "testing" +) + +func TestHostFromServiceURL(t *testing.T) { + cases := map[string]string{ + "https://junk.realestate.com.au": "junk.realestate.com.au", + "http://junk.realestate.com.au/login": "junk.realestate.com.au", + "https://junk.realestate.com.au:8443/": "junk.realestate.com.au", + "https://user:pass@junk.realestate.com.au/": "junk.realestate.com.au", + "JUNK.REALESTATE.com.au.": "junk.realestate.com.au", + "": "", + "/only/path": "", + } + for raw, want := range cases { + if got := hostFromServiceURL(raw); got != want { + t.Fatalf("hostFromServiceURL(%q) = %q, want %q", raw, got, want) + } + } +} + +func TestSortedTechnologies(t *testing.T) { + if got := sortedTechnologies(`["nginx","WordPress"]`); got != "wordpress,nginx" { + t.Fatalf("mixed order/case must normalize (sort is byte-order, lowercase after): %q", got) + } + if got := sortedTechnologies(`[]`); got != "" { + t.Fatalf("empty list must be an empty string, got %q", got) + } + if got := sortedTechnologies(`not-json`); got != "not-json" { + t.Fatalf("non-JSON rows must stay comparable: %q", got) + } +} + +func TestQualifiesCatchallGroups(t *testing.T) { + apexTargets := catchallApexTargets("realestate.com.au") + + // Redirect fleet: 3 junk members, all funneling into the apex → marked; + // an apex/www member must never be marked. + redirect := []catchallRow{ + {host: "a.realestate.com.au", contentFP: "FP", redirectHost: "realestate.com.au"}, + {host: "b.realestate.com.au", contentFP: "FP", redirectHost: "www.realestate.com.au"}, + {host: "asdf-garbage.git.realestate.com.au", contentFP: "FP", redirectHost: "realestate.com.au"}, + } + marked, kind := qualifiesCatchallGroup(redirect, apexTargets) + if kind != "redirect" || len(marked) != 3 { + t.Fatalf("3-member redirect fleet must be marked (kind=%q n=%d)", kind, len(marked)) + } + mixed := append(append([]catchallRow{}, redirect...), catchallRow{host: "realestate.com.au", contentFP: "FP"}) + marked, kind = qualifiesCatchallGroup(mixed, apexTargets) + if kind != "redirect" || len(marked) != 3 { + t.Fatalf("fleet+apex group must mark only junk members (kind=%q n=%d)", kind, len(marked)) + } + for _, row := range marked { + if apexTargets[row.host] { + t.Fatalf("apex/www host marked as catch-all: %q", row.host) + } + } + + // Same fingerprint with members split across redirect targets → not a fleet. + half := []catchallRow{ + {host: "a.realestate.com.au", contentFP: "FP", redirectHost: "realestate.com.au"}, + {host: "b.realestate.com.au", contentFP: "FP", redirectHost: "realestate.com.au"}, + {host: "c.realestate.com.au", contentFP: "FP", redirectHost: "elsewhere.example"}, + } + if _, kind = qualifiesCatchallGroup(half, apexTargets); kind != "" { + t.Fatalf("partial redirect agreement must not mark (kind=%q)", kind) + } + + // Homepage clones: apex row anchors a group of >=5 identical fingerprints. + home := []catchallRow{{host: "realestate.com.au", contentFP: "HOMEFP"}} + for i := 0; i < 5; i++ { + home = append(home, catchallRow{host: fmt.Sprintf("clone%d.git.realestate.com.au", i), contentFP: "HOMEFP"}) + } + marked, kind = qualifiesCatchallGroup(home, apexTargets) + if kind != "homepage" || len(marked) != 5 { + t.Fatalf("apex-anchored clone group must mark the 5 junk rows (kind=%q n=%d)", kind, len(marked)) + } + + // 4 identical members without the apex row must NOT mark (a legit twin + // pair fleet is not catch-all evidence on its own). + twins := make([]catchallRow, 0, 4) + for i := 0; i < 4; i++ { + twins = append(twins, catchallRow{host: fmt.Sprintf("twin%d.realestate.com.au", i), contentFP: "TW"}) + } + if _, kind = qualifiesCatchallGroup(twins, apexTargets); kind != "" { + t.Fatalf("4 identical hosts without the apex must stay unmarked (kind=%q)", kind) + } + + // With the apex but only 3 members → below the content threshold (4 total). + short := []catchallRow{ + catchallRow{host: "realestate.com.au", contentFP: "TW"}, + catchallRow{host: "twin0.realestate.com.au", contentFP: "TW"}, + catchallRow{host: "twin1.realestate.com.au", contentFP: "TW"}, + catchallRow{host: "twin2.realestate.com.au", contentFP: "TW"}, + } + if _, kind = qualifiesCatchallGroup(short, apexTargets); kind != "" { + t.Fatalf("3 members + apex is below the content threshold (kind=%q)", kind) + } +} + +func TestRunCatchallPassEndToEnd(t *testing.T) { + db, tid := testDB(t) + ctx := context.Background() + + seed := func(host string, statusCode int, title string, redirect string, source string) { + if _, err := db.Exec(`INSERT INTO subdomains (id, target_id, subdomain, ip, source, catchall, is_alive) + VALUES (?, ?, ?, '10.0.0.1', ?, 0, 1)`, "ct-"+host, tid, host, source); err != nil { + t.Fatalf("seed subdomain %s: %v", host, err) + } + if _, err := db.Exec(`INSERT INTO http_services (id, target_id, url, status_code, title, server, redirect_url, technologies) + VALUES (?, ?, ?, ?, ?, 'nginx', ?, '[]')`, + "ct-svc-"+host, tid, "https://"+host, statusCode, title, redirect); err != nil { + t.Fatalf("seed service %s: %v", host, err) + } + } + + // The wildcard homepage fleet: apex + 5 junk clones with identical title. + seed("m.local", 200, "Real Estate Portal", "", "dns") + for i := 0; i < 5; i++ { + seed(fmt.Sprintf("junk%d.git.m.local", i), 200, "Real Estate Portal", "", "dns") + } + // An auth wall that legitimately shares a shell across hosts: 401s are + // outside the 2xx window and must never group. + seed("portal.m.local", 401, "Sign in", "", "dns") + // An operator-vouched seed that shares the homepage content: seed/vhost + // rows are never re-marked by the pass. + seed("operator-host.m.local", 200, "Real Estate Portal", "", "seed") + + scanner := &HTTPScanner{db: db} + logFn := func(_ string, _, _ string) {} + if err := scanner.runCatchallPass(ctx, tid, logFn); err != nil { + t.Fatalf("runCatchallPass: %v", err) + } + + var junk int + if err := db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE catchall=1 AND target_id=?`, tid).Scan(&junk); err != nil { + t.Fatalf("count marked: %v", err) + } + if junk != 5 { + t.Fatalf("expected the 5 junk clones marked, got %d", junk) + } + var apex int + if err := db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE target_id=? AND subdomain='m.local' AND catchall=1`, tid).Scan(&apex); err != nil { + t.Fatalf("count apex: %v", err) + } + if apex != 0 { + t.Fatalf("the apex itself must never be marked") + } + var services int + if err := db.QueryRow(`SELECT COUNT(*) FROM http_services WHERE target_id=? AND url LIKE 'https://junk%'`, tid).Scan(&services); err != nil { + t.Fatalf("count junk services: %v", err) + } + if services != 0 { + t.Fatalf("junk http_services rows must be dropped so web modules skip them, got %d", services) + } + var left int + if err := db.QueryRow(`SELECT COUNT(*) FROM http_services WHERE target_id=?`, tid).Scan(&left); err != nil { + t.Fatalf("count left services: %v", err) + } + if left != 3 { // apex + operator seed stay; the 401 auth wall is never marked, so it stays too + t.Fatalf("apex + seed + 401 services must survive, got %d", left) + } +} diff --git a/internal/scanner/shodan.go b/internal/scanner/shodan.go index cdedc2c..e3e99dd 100644 --- a/internal/scanner/shodan.go +++ b/internal/scanner/shodan.go @@ -104,7 +104,7 @@ func (s *ShodanScanner) Run(ctx context.Context, targetID string, logFn LogFunc) // gatherIPs returns a map of IP → representative hostname for resolved subdomains. func (s *ShodanScanner) gatherIPs(ctx context.Context, targetID string) map[string]string { out := map[string]string{} - rows, err := s.db.QueryContext(ctx, `SELECT subdomain, ip FROM subdomains WHERE target_id=? AND ip != ''`, targetID) + rows, err := s.db.QueryContext(ctx, `SELECT subdomain, ip FROM subdomains WHERE target_id=? AND ip != '' AND COALESCE(catchall,0)=0`, targetID) if err != nil { return out } From 61ef39615a678733d049c94c4f37603bf50ec40c Mon Sep 17 00:00:00 2001 From: void Date: Sun, 25 Oct 2026 08:33:46 +0330 Subject: [PATCH 10/14] fix: validation follow-ups for wildcard gate + catch-all pass Found during the post-implementation validation pass: - Retry probes dialed BARE resolver IPs (the dnsx resolver list strips :53), so every cross-resolver retry failed with 'missing port in address' and was counted as ambiguous. resolverServerAddress now normalizes entries to host:port (IPv4/IPv6/bracketed/non-standard port); a nil resolver falls back to the system one. - Catch-all marking now defense-in-depth rejects hosts that are not strictly under the target apex, and dedupes hosts across http/https service rows before logging/marking. - Remaining consumer surfaces now respect the catchall mark: takeover checks, asset priority/planning, recursive-enumeration frontier, admitted-name/wordlist seeds, global search, dashboard target stats and scan-done notifications. Tests: retry-address normalization + second-pass idempotency. Suites: scanner/api/scheduler/database green; frontend build green. --- internal/api/system_handlers.go | 2 +- internal/scanner/asset_priority.go | 2 +- internal/scanner/http_catchall.go | 25 ++++++++++++++++---- internal/scanner/http_catchall_test.go | 20 ++++++++++++++++ internal/scanner/subdomain.go | 6 +++-- internal/scanner/takeover.go | 2 +- internal/scanner/wildcard.go | 32 +++++++++++++++++++++++--- internal/scanner/wildcard_test.go | 16 +++++++++++++ internal/scheduler/scheduler.go | 8 +++---- 9 files changed, 96 insertions(+), 17 deletions(-) diff --git a/internal/api/system_handlers.go b/internal/api/system_handlers.go index f8bf1b7..6cec19e 100644 --- a/internal/api/system_handlers.go +++ b/internal/api/system_handlers.go @@ -759,7 +759,7 @@ func (h *Handler) handleSearch(w http.ResponseWriter, r *http.Request) { subRows, err := h.db.QueryContext(r.Context(), ` SELECT s.id, s.subdomain, s.ip, t.domain as target_domain FROM subdomains s JOIN targets t ON t.id = s.target_id - WHERE s.subdomain LIKE ? LIMIT 20 + WHERE s.subdomain LIKE ? AND COALESCE(s.catchall,0)=0 LIMIT 20 `, like) if err == nil { defer subRows.Close() diff --git a/internal/scanner/asset_priority.go b/internal/scanner/asset_priority.go index 0d0448b..da037ce 100644 --- a/internal/scanner/asset_priority.go +++ b/internal/scanner/asset_priority.go @@ -171,7 +171,7 @@ func loadAssetRows(ctx context.Context, db *database.DB, targetID string) (map[s srows, err := db.QueryContext(ctx, ` SELECT subdomain, is_alive, status_code, COALESCE(page_title,''), COALESCE(server,''), COALESCE(technologies,'[]'), COALESCE(waf,''), COALESCE(source,'dns') - FROM subdomains WHERE target_id=?`, targetID) + FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0`, targetID) if err != nil { return nil, err } diff --git a/internal/scanner/http_catchall.go b/internal/scanner/http_catchall.go index 09c763e..caf3852 100644 --- a/internal/scanner/http_catchall.go +++ b/internal/scanner/http_catchall.go @@ -11,7 +11,7 @@ import ( // The HTTP catch-all pass closes the hole the DNS wildcard gate cannot see: // a zone with NO DNS wildcard can still serve one shared homepage or one -// redirect for EVERY hostname under it (portslike hosting, parking, SaaS +// redirect for EVERY hostname under it (parking-style hosting, SaaS // platforms). Those names arrive as fully "verified" subdomains — the DNS gate // happily passes them — and flood the Hosts tab, reports and every web module // downstream. @@ -221,7 +221,7 @@ func (s *HTTPScanner) runCatchallPass(ctx context.Context, targetID string, logF apexTargets := catchallApexTargets(apex) groups := groupCatchallFingerprints(candidateRows) - var markedHosts []string + markedSet := map[string]string{} // host -> pattern kind (deduped across schemes/groups) redirectGroups, homepageGroups := 0, 0 for _, members := range groups { marked, kind := qualifiesCatchallGroup(members, apexTargets) @@ -234,17 +234,32 @@ func (s *HTTPScanner) runCatchallPass(ctx context.Context, targetID string, logF homepageGroups++ } for _, row := range marked { - markedHosts = append(markedHosts, row.host) - logFn("info", "http_probe", fmt.Sprintf("HTTP catch-all: %s serves the wildcard group (%s pattern, group of %d)", row.host, kind, len(members))) + // Defense in depth: only hosts strictly under the target apex are + // ever marked (qualifies already excludes apex/www; this also + // rejects any foreign host a stray service row might carry). + if row.host == apex || !strings.HasSuffix(row.host, "."+apex) { + continue + } + if _, seen := markedSet[row.host]; !seen { + markedSet[row.host] = kind + } } } - if len(markedHosts) == 0 { + if len(markedSet) == 0 { logFn("info", "http_probe", fmt.Sprintf("HTTP catch-all pass: %d live 2xx host(s) analysed — no catch-all groups", total2xx)) return nil } + markedHosts := make([]string, 0, len(markedSet)) + for host := range markedSet { + markedHosts = append(markedHosts, host) + } sort.Strings(markedHosts) + for _, host := range markedHosts { + logFn("info", "http_probe", fmt.Sprintf("HTTP catch-all: %s serves the wildcard group (%s pattern)", host, markedSet[host])) + } + tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err diff --git a/internal/scanner/http_catchall_test.go b/internal/scanner/http_catchall_test.go index b7c50d0..3577cee 100644 --- a/internal/scanner/http_catchall_test.go +++ b/internal/scanner/http_catchall_test.go @@ -164,4 +164,24 @@ func TestRunCatchallPassEndToEnd(t *testing.T) { if left != 3 { // apex + operator seed stay; the 401 auth wall is never marked, so it stays too t.Fatalf("apex + seed + 401 services must survive, got %d", left) } + + // Idempotency: a second pass (next scan) must not change anything — the + // junk services are already gone and the marks already exist. + if err := scanner.runCatchallPass(ctx, tid, logFn); err != nil { + t.Fatalf("second runCatchallPass: %v", err) + } + var junkAgain int + if err := db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE catchall=1 AND target_id=?`, tid).Scan(&junkAgain); err != nil { + t.Fatalf("count marked after rerun: %v", err) + } + if junkAgain != 5 { + t.Fatalf("second pass must be idempotent, marked=%d", junkAgain) + } + var leftAgain int + if err := db.QueryRow(`SELECT COUNT(*) FROM http_services WHERE target_id=?`, tid).Scan(&leftAgain); err != nil { + t.Fatalf("count left services after rerun: %v", err) + } + if leftAgain != 3 { + t.Fatalf("second pass must not delete surviving services, got %d", leftAgain) + } } diff --git a/internal/scanner/subdomain.go b/internal/scanner/subdomain.go index 25acec2..4c6166a 100644 --- a/internal/scanner/subdomain.go +++ b/internal/scanner/subdomain.go @@ -1100,7 +1100,8 @@ func (s *SubdomainScanner) activeEnum(ctx context.Context, targetID, domain stri func (s *SubdomainScanner) admittedSubdomainNames(ctx context.Context, targetID, domain string) []string { rows, err := s.db.QueryContext(ctx, `SELECT subdomain FROM subdomains - WHERE target_id=? AND (COALESCE(ip,'')!='' OR COALESCE(source,'dns') IN ('seed','vhost')) + WHERE target_id=? AND COALESCE(catchall,0)=0 + AND (COALESCE(ip,'')!='' OR COALESCE(source,'dns') IN ('seed','vhost')) ORDER BY subdomain`, targetID) if err != nil { return nil @@ -1425,7 +1426,8 @@ func (s *SubdomainScanner) recursiveEnum(ctx context.Context, targetID, apex str // for recursive enumeration. func (s *SubdomainScanner) resolvingSubdomainsUnder(ctx context.Context, targetID, apex string, visited map[string]bool) []string { rows, err := s.db.QueryContext(ctx, `SELECT subdomain FROM subdomains - WHERE target_id=? AND COALESCE(ip,'')!='' AND subdomain LIKE ?`, targetID, "%."+strings.ToLower(apex)) + WHERE target_id=? AND COALESCE(catchall,0)=0 AND COALESCE(ip,'')!='' + AND subdomain LIKE ?`, targetID, "%."+strings.ToLower(apex)) if err != nil { return nil } diff --git a/internal/scanner/takeover.go b/internal/scanner/takeover.go index 6fb4d5f..e2b715e 100644 --- a/internal/scanner/takeover.go +++ b/internal/scanner/takeover.go @@ -104,7 +104,7 @@ var takeoverFingerprints = []takeoverFingerprint{ func (s *TakeoverScanner) Run(ctx context.Context, targetID string, logFn LogFunc) error { logFn("info", "takeover", "Checking for subdomain takeovers...") - rows, err := s.db.QueryContext(ctx, `SELECT subdomain FROM subdomains WHERE target_id = ?`, targetID) + rows, err := s.db.QueryContext(ctx, `SELECT subdomain FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0)=0`, targetID) if err != nil { return fmt.Errorf("query subdomains: %w", err) } diff --git a/internal/scanner/wildcard.go b/internal/scanner/wildcard.go index 9776aca..9195e74 100644 --- a/internal/scanner/wildcard.go +++ b/internal/scanner/wildcard.go @@ -203,19 +203,38 @@ func (p *wildcardProfile) hasAnyWildcard() bool { return false } +// resolverServerAddress normalizes one resolver entry to a dialable host:port +// address. The configured/derived lists carry BARE hosts ("1.1.1.1") for the +// dnsx resolver file, and a bare host is NOT a valid dial target ("missing +// port in address"). Port 53 is assumed when absent; entries that already have +// a port keep it, and IPv6 literals get bracket-wrapped by JoinHostPort. +func resolverServerAddress(server string) string { + server = strings.TrimSpace(server) + if server == "" { + return "" + } + if _, _, err := net.SplitHostPort(server); err == nil { + return server + } + // A bracketed literal without a port ("[2606:4700::1111]") is not a valid + // SplitHostPort input; strip the brackets before re-joining. + host := strings.TrimPrefix(strings.TrimSuffix(server, "]"), "[") + return net.JoinHostPort(host, "53") +} + // singleServerResolver builds a resolver pinned to ONE upstream server so a // retry genuinely asks a DIFFERENT resolver — buildDNSResolver rotates per // connection, which is usually enough, but an explicit pin gives the retry // loop a guarantee. func singleServerResolver(server string) *net.Resolver { - res := &net.Resolver{ + address := resolverServerAddress(server) + return &net.Resolver{ PreferGo: true, Dial: func(ctx context.Context, network, _ string) (net.Conn, error) { d := net.Dialer{Timeout: 4 * time.Second} - return d.DialContext(ctx, network, server) + return d.DialContext(ctx, network, address) }, } - return res } // resolveProbeWithRetry resolves one probe name, retrying ambiguous answers @@ -224,6 +243,9 @@ func singleServerResolver(server string) *net.Resolver { // that succeeds still wins — a resolver badly configured for the zone must not // mask an existing name. func resolveProbeWithRetry(r *net.Resolver, name string, servers []string, startOffset int) ([]net.IPAddr, probeVerdict) { + if r == nil { + r = net.DefaultResolver + } attempts := len(servers) if attempts == 0 { attempts = 1 // system resolver only; r carries it @@ -246,6 +268,10 @@ func resolveProbeWithRetry(r *net.Resolver, name string, servers []string, start res = r } else { idx := (startOffset + attempt) % len(servers) + if strings.TrimSpace(servers[idx]) == "" { + sawAmbiguous = true + continue + } res = singleServerResolver(servers[idx]) } ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) diff --git a/internal/scanner/wildcard_test.go b/internal/scanner/wildcard_test.go index 1ec723d..7f9fee0 100644 --- a/internal/scanner/wildcard_test.go +++ b/internal/scanner/wildcard_test.go @@ -199,3 +199,19 @@ func TestDetectBranchWildcardsOrderAndCap(t *testing.T) { t.Fatalf("unexpected suffix: %v", keys) } } + +func TestResolverServerAddress(t *testing.T) { + cases := map[string]string{ + "1.1.1.1": "1.1.1.1:53", + "8.8.8.8:5353": "8.8.8.8:5353", + "2606:4700:4700::1111": "[2606:4700:4700::1111]:53", + "[2606:4700:4700::1111]": "[2606:4700:4700::1111]:53", + "[2606:4700:4700::1111]:53": "[2606:4700:4700::1111]:53", + " ": "", + } + for in, want := range cases { + if got := resolverServerAddress(in); got != want { + t.Fatalf("resolverServerAddress(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index c78ec65..d90afab 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -2723,9 +2723,9 @@ func (s *Scheduler) unresolvedTaskPhaseCount(taskID string) int { func (s *Scheduler) updateTargetStats(targetID string) { var subdomainCount, aliveCount, findingCount int - s.db.QueryRow("SELECT COUNT(*) FROM subdomains WHERE target_id = ?", targetID).Scan(&subdomainCount) + s.db.QueryRow("SELECT COUNT(*) FROM subdomains WHERE target_id = ? AND COALESCE(catchall,0)=0", targetID).Scan(&subdomainCount) s.db.QueryRow(`SELECT - (SELECT COUNT(*) FROM subdomains WHERE target_id=? AND is_alive=1) + + (SELECT COUNT(*) FROM subdomains WHERE target_id=? AND is_alive=1 AND COALESCE(catchall,0)=0) + (SELECT COUNT(*) FROM network_hosts WHERE target_id=? AND is_alive=1)`, targetID, targetID).Scan(&aliveCount) // The orange headline is an actionable vulnerability count, not an inventory // counter. Keep only medium+ findings; exposed-file discovery is promoted into @@ -2961,8 +2961,8 @@ func (s *Scheduler) monitorMemory() { func (s *Scheduler) notifyScanDone(taskID, targetID, domain, status string, startedAt time.Time) { duration := time.Since(startedAt) var subdomains, alive, vulns, nuclei, backups, redirects, jsFindings int - s.db.QueryRow("SELECT COUNT(*) FROM subdomains WHERE target_id=?", targetID).Scan(&subdomains) - s.db.QueryRow("SELECT COUNT(*) FROM subdomains WHERE target_id=? AND is_alive=1", targetID).Scan(&alive) + s.db.QueryRow("SELECT COUNT(*) FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0", targetID).Scan(&subdomains) + s.db.QueryRow("SELECT COUNT(*) FROM subdomains WHERE target_id=? AND is_alive=1 AND COALESCE(catchall,0)=0", targetID).Scan(&alive) s.db.QueryRow("SELECT COUNT(*) FROM vuln_findings WHERE target_id=? AND COALESCE(status,'finding')='finding' AND COALESCE(triage,'')<>'false_positive'", targetID).Scan(&vulns) s.db.QueryRow("SELECT COUNT(*) FROM nuclei_findings WHERE target_id=? AND verification='verified'", targetID).Scan(&nuclei) s.db.QueryRow("SELECT COUNT(*) FROM backup_findings WHERE target_id=?", targetID).Scan(&backups) From 0b6ebe155770aa6614fce1c0c38d224d8a313fac Mon Sep 17 00:00:00 2001 From: void Date: Sun, 25 Oct 2026 08:51:29 +0330 Subject: [PATCH 11/14] feat(scan): per-target HTTP catch-all override (global default unchanged) Adds a tri-state targets.catchall_detection column: NULL inherits the global http_catchall_detection config (code default OFF), 0 forces the catch-all pass off for the target, 1 forces it on. - scanner: catchallEnabledForTarget resolves override > global; the http_probe hook uses it instead of the raw config bool. - api: GET /targets/{id} returns catchall_detection (inherit|on|off) and catchall_effective; PUT /targets/{id} accepts the tri-state string and rejects anything else. - ui: TargetDetail gets an 'HTTP catch-all detection' select next to the monitoring settings, showing the effective state; applies from the next scan. - tests: resolver matrix (scanner) + handler round-trip set/clear/invalid (api). --- frontend/src/pages/TargetDetail.tsx | 36 +++++++++ frontend/src/types/index.ts | 2 + internal/api/target_catchall_test.go | 108 +++++++++++++++++++++++++ internal/api/target_handlers.go | 49 +++++++++-- internal/database/migrations.go | 6 ++ internal/models/models.go | 5 ++ internal/scanner/http.go | 11 +-- internal/scanner/http_catchall.go | 14 ++++ internal/scanner/http_catchall_test.go | 45 +++++++++++ 9 files changed, 264 insertions(+), 12 deletions(-) create mode 100644 internal/api/target_catchall_test.go diff --git a/frontend/src/pages/TargetDetail.tsx b/frontend/src/pages/TargetDetail.tsx index f3a8b58..81773ae 100644 --- a/frontend/src/pages/TargetDetail.tsx +++ b/frontend/src/pages/TargetDetail.tsx @@ -516,6 +516,7 @@ export default function TargetDetail() { const [diffView, setDiffView] = useState<{ url: string; kind: 'css' | 'js' } | null>(null) const [isScanning, setIsScanning] = useState(false) const [monitorSaving, setMonitorSaving] = useState(false) + const [catchallSaving, setCatchallSaving] = useState(false) const [paths, setPaths] = useState([]) const [lastFailedTask, setLastFailedTask] = useState(null) const [resuming, setResuming] = useState(false) @@ -823,6 +824,16 @@ export default function TargetDetail() { } catch { /* ignore */ } finally { setMonitorSaving(false) } } + const saveCatchall = async (value: 'inherit' | 'on' | 'off') => { + if (!id) return + setCatchallSaving(true) + try { + await targetsApi.update(id, { catchall_detection: value }) + // Refresh so catchall_effective reflects the global config on "inherit". + targetsApi.get(id).then(setTarget).catch(() => {}) + } catch { /* ignore */ } finally { setCatchallSaving(false) } + } + // Network targets' actual results (Nuclei/Vulns/Candidates) used to sit // BELOW monitoring settings, attack paths, the auth-header form and the // identities/authorization panels — none of which are commonly used for a @@ -1018,6 +1029,31 @@ export default function TargetDetail() { )}
+ {/* HTTP catch-all detection override */} +
+
+ HTTP catch-all detection + + {catchallSaving && Saving…} + + Effective: {target.catchall_effective ? 'On' : 'Off'} + +
+

+ Marks fleets that redirect to the apex or clone the apex homepage as catch-all junk (hidden from Hosts, + reports and later modules). The global default is off; the per-target override applies from the next scan. + Seeds and vhost rows are never touched. +

+
+ {/* Attack paths (correlation / intelligence layer) */} {paths.length > 0 && (
diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 8572241..de2c8a4 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -132,6 +132,8 @@ export interface Target { monitor_last_run: string | null scan_user_agent?: string scan_headers?: Record + catchall_detection?: 'inherit' | 'on' | 'off' + catchall_effective?: boolean } export interface Subdomain { diff --git a/internal/api/target_catchall_test.go b/internal/api/target_catchall_test.go new file mode 100644 index 0000000..2c5734e --- /dev/null +++ b/internal/api/target_catchall_test.go @@ -0,0 +1,108 @@ +package api + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" + "github.com/recon-platform/internal/config" + "github.com/recon-platform/internal/websocket" +) + +// TestTargetCatchallOverrideRoundTrip proves the per-target tri-state override +// end to end through the real handlers: GET exposes override + effective state, +// PUT persists 0/1/NULL, invalid values are rejected, and "inherit" tracks the +// global config without touching the column. +func TestTargetCatchallOverrideRoundTrip(t *testing.T) { + h := newTestHandler(t) + h.hub = websocket.NewHub() + h.cfg = &config.Config{HTTPCatchallDetection: true} + + if _, err := h.db.Exec(`INSERT INTO targets (id, domain) VALUES ('ct-target','catchall.example.com')`); err != nil { + t.Fatal(err) + } + + getTarget := func() map[string]any { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/api/targets/ct-target", nil) + req = mux.SetURLVars(req, map[string]string{"id": "ct-target"}) + rec := httptest.NewRecorder() + h.handleGetTarget(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("GET target status = %d, body = %s", rec.Code, rec.Body.String()) + } + var resp struct { + Data map[string]any `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode GET response: %v", err) + } + return resp.Data + } + putTarget := func(body string) int { + t.Helper() + req := httptest.NewRequest(http.MethodPut, "/api/targets/ct-target", bytes.NewBufferString(body)) + req = mux.SetURLVars(req, map[string]string{"id": "ct-target"}) + rec := httptest.NewRecorder() + h.handleUpdateTarget(rec, req) + return rec.Code + } + + // A new target inherits the global (on). + data := getTarget() + if data["catchall_detection"] != "inherit" { + t.Fatalf("default override = %v, want inherit", data["catchall_detection"]) + } + if data["catchall_effective"] != true { + t.Fatalf("inherit with global on effective = %v, want true", data["catchall_effective"]) + } + + // Force off: the override wins over the global. + if code := putTarget(`{"catchall_detection":"off"}`); code != http.StatusOK { + t.Fatalf("PUT off status = %d", code) + } + data = getTarget() + if data["catchall_detection"] != "off" || data["catchall_effective"] != false { + t.Fatalf("after off: %v/%v, want off/false", data["catchall_detection"], data["catchall_effective"]) + } + + // Force on. + if code := putTarget(`{"catchall_detection":"on"}`); code != http.StatusOK { + t.Fatalf("PUT on status = %d", code) + } + data = getTarget() + if data["catchall_detection"] != "on" || data["catchall_effective"] != true { + t.Fatalf("after on: %v/%v, want on/true", data["catchall_detection"], data["catchall_effective"]) + } + + // Clear back to inherit: the column must be NULL again. + if code := putTarget(`{"catchall_detection":"inherit"}`); code != http.StatusOK { + t.Fatalf("PUT inherit status = %d", code) + } + var stored *int + if err := h.db.QueryRow(`SELECT catchall_detection FROM targets WHERE id='ct-target'`).Scan(&stored); err != nil { + t.Fatalf("read catchall_detection column: %v", err) + } + if stored != nil { + t.Fatalf("inherit must store NULL, got %d", *stored) + } + data = getTarget() + if data["catchall_detection"] != "inherit" || data["catchall_effective"] != true { + t.Fatalf("after inherit: %v/%v, want inherit/true", data["catchall_detection"], data["catchall_effective"]) + } + + // Invalid values are rejected before any write. + if code := putTarget(`{"catchall_detection":"maybe"}`); code != http.StatusBadRequest { + t.Fatalf("invalid override status = %d, want 400", code) + } + + // Flipping the global config changes "inherit" effectiveness without a write. + h.cfg.HTTPCatchallDetection = false + data = getTarget() + if data["catchall_effective"] != false { + t.Fatalf("inherit with global off effective = %v, want false", data["catchall_effective"]) + } +} diff --git a/internal/api/target_handlers.go b/internal/api/target_handlers.go index 9ed2551..7e7d6cd 100644 --- a/internal/api/target_handlers.go +++ b/internal/api/target_handlers.go @@ -371,20 +371,20 @@ func (h *Handler) handleGetTarget(w http.ResponseWriter, r *http.Request) { var t models.Target var lastScan, monitorLastRun *string var tagsJSON, modulesJSON, scanHeadersJSON string - var monitorEnabled int + var monitorEnabled, catchallOverride int err := h.db.QueryRowContext(r.Context(), ` SELECT id, domain, COALESCE(kind,'web'), COALESCE(description,''), COALESCE(tags,'[]'), COALESCE(priority,'medium'), COALESCE(notes,''), COALESCE(status,'idle'), COALESCE(scan_status,'idle'), COALESCE(enabled_modules,'[]'), last_scan_at, created_at, updated_at, COALESCE(subdomain_count,0), COALESCE(alive_host_count,0), COALESCE(finding_count,0), COALESCE(monitor_enabled,0), COALESCE(monitor_interval_hours,2), COALESCE(name,''), COALESCE(exclude_scope,''), monitor_last_run, - COALESCE(scan_user_agent,''), COALESCE(scan_headers,'{}') + COALESCE(scan_user_agent,''), COALESCE(scan_headers,'{}'), COALESCE(catchall_detection,-1) FROM targets WHERE id = ? `, id).Scan(&t.ID, &t.Domain, &t.Kind, &t.Description, &tagsJSON, &t.Priority, &t.Notes, &t.Status, &t.ScanStatus, &modulesJSON, &lastScan, &t.CreatedAt, &t.UpdatedAt, &t.SubdomainCount, &t.AliveHostCount, &t.FindingCount, - &monitorEnabled, &t.MonitorIntervalHours, &t.Name, &t.ExcludeScope, &monitorLastRun, &t.ScanUserAgent, &scanHeadersJSON) + &monitorEnabled, &t.MonitorIntervalHours, &t.Name, &t.ExcludeScope, &monitorLastRun, &t.ScanUserAgent, &scanHeadersJSON, &catchallOverride) if err != nil { h.writeError(w, http.StatusNotFound, "target not found") return @@ -393,6 +393,17 @@ func (h *Handler) handleGetTarget(w http.ResponseWriter, r *http.Request) { t.Tags = models.JSONToStringSlice(tagsJSON) t.EnabledModules = models.JSONToStringSlice(modulesJSON) t.MonitorEnabled = monitorEnabled == 1 + switch catchallOverride { + case 1: + t.CatchallDetection = "on" + t.CatchallEffective = true + case 0: + t.CatchallDetection = "off" + t.CatchallEffective = false + default: + t.CatchallDetection = "inherit" + t.CatchallEffective = h.cfg != nil && h.cfg.HTTPCatchallDetection + } t.ScanHeaders = map[string]string{} _ = json.Unmarshal([]byte(scanHeadersJSON), &t.ScanHeaders) if lastScan != nil { @@ -421,6 +432,9 @@ func (h *Handler) handleUpdateTarget(w http.ResponseWriter, r *http.Request) { EnabledModules *[]string `json:"enabled_modules"` ScanUserAgent *string `json:"scan_user_agent"` ScanHeaders *map[string]string `json:"scan_headers"` + // Tri-state per-target override for the HTTP catch-all pass: + // "inherit" (NULL column, follow the global config), "on" (1), "off" (0). + CatchallDetection *string `json:"catchall_detection"` } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -432,10 +446,11 @@ func (h *Handler) handleUpdateTarget(w http.ResponseWriter, r *http.Request) { // omitted fields stay untouched; the old zero-value decoding silently erased // enabled_modules (and, for API clients, metadata/identity) on unrelated edits. var name, description, tagsJSON, priority, notes, excludeScope, modulesJSON, scanUserAgent, scanHeadersJSON string + var catchallOverride int err := h.db.QueryRowContext(r.Context(), `SELECT COALESCE(name,''),COALESCE(description,''),COALESCE(tags,'[]'), COALESCE(priority,'medium'),COALESCE(notes,''),COALESCE(exclude_scope,''),COALESCE(enabled_modules,'[]'), - COALESCE(scan_user_agent,''),COALESCE(scan_headers,'{}') FROM targets WHERE id=?`, id). - Scan(&name, &description, &tagsJSON, &priority, ¬es, &excludeScope, &modulesJSON, &scanUserAgent, &scanHeadersJSON) + COALESCE(scan_user_agent,''),COALESCE(scan_headers,'{}'),COALESCE(catchall_detection,-1) FROM targets WHERE id=?`, id). + Scan(&name, &description, &tagsJSON, &priority, ¬es, &excludeScope, &modulesJSON, &scanUserAgent, &scanHeadersJSON, &catchallOverride) if err == sql.ErrNoRows { h.writeError(w, http.StatusNotFound, "target not found") return @@ -480,6 +495,25 @@ func (h *Handler) handleUpdateTarget(w http.ResponseWriter, r *http.Request) { h.writeError(w, http.StatusBadRequest, err.Error()) return } + if req.CatchallDetection != nil { + // Tri-state override; "inherit" clears the column back to NULL. + switch strings.ToLower(strings.TrimSpace(*req.CatchallDetection)) { + case "inherit": + catchallOverride = -1 + case "on": + catchallOverride = 1 + case "off": + catchallOverride = 0 + default: + h.writeError(w, http.StatusBadRequest, "catchall_detection must be inherit, on, or off") + return + } + } + // nil ⇒ SQL NULL (inherit), otherwise the 0/1 override. + var catchallDB any + if catchallOverride >= 0 { + catchallDB = catchallOverride + } rawScope := "" if req.Domain != nil { @@ -512,9 +546,10 @@ func (h *Handler) handleUpdateTarget(w http.ResponseWriter, r *http.Request) { res, err := tx.ExecContext(r.Context(), ` UPDATE targets SET name = ?, description = ?, tags = ?, priority = ?, notes = ?, - exclude_scope = ?, enabled_modules = ?, scan_user_agent = ?, scan_headers = ?, updated_at = CURRENT_TIMESTAMP + exclude_scope = ?, enabled_modules = ?, scan_user_agent = ?, scan_headers = ?, + catchall_detection = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? - `, name, description, tagsJSON, priority, notes, excludeScope, modulesJSON, scanUserAgent, scanHeadersJSON, id) + `, name, description, tagsJSON, priority, notes, excludeScope, modulesJSON, scanUserAgent, scanHeadersJSON, catchallDB, id) if err != nil { h.writeError(w, http.StatusInternalServerError, "failed to update target") return diff --git a/internal/database/migrations.go b/internal/database/migrations.go index 80c77dd..bd48f36 100644 --- a/internal/database/migrations.go +++ b/internal/database/migrations.go @@ -162,6 +162,7 @@ func RunMigrations(db *DB) error { alterJSFilesAddOldBody, createWPSitesTable, alterSubdomainsAddCatchall, + alterTargetsAddCatchallDetection, } for i, m := range migrations { @@ -1084,6 +1085,11 @@ const alterSubdomainsAddSource = `ALTER TABLE subdomains ADD COLUMN source TEXT // COALESCE(catchall,0)=0. Written only when the opt-in config flag is set. const alterSubdomainsAddCatchall = `ALTER TABLE subdomains ADD COLUMN catchall INTEGER NOT NULL DEFAULT 0;` +// alterTargetsAddCatchallDetection adds the per-target tri-state override of the +// HTTP catch-all pass: NULL = inherit the global http_catchall_detection config, +// 0 = force off for this target, 1 = force on. +const alterTargetsAddCatchallDetection = `ALTER TABLE targets ADD COLUMN catchall_detection INTEGER DEFAULT NULL;` + // ── P1 canonical lifecycle bridge ──────────────────────────────────────────── // candidates is the AUTHORITATIVE lifecycle state source. state_version is a // monotonic counter used for optimistic, concurrency-safe transitions. diff --git a/internal/models/models.go b/internal/models/models.go index 916644f..c4bb26b 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -29,6 +29,11 @@ type Target struct { MonitorLastRun *time.Time `json:"monitor_last_run" db:"monitor_last_run"` ScanUserAgent string `json:"scan_user_agent" db:"scan_user_agent"` ScanHeaders map[string]string `json:"scan_headers" db:"scan_headers"` + // CatchallDetection is the per-target override of the HTTP catch-all pass: + // "inherit" (follow the global http_catchall_detection setting), "on" or + // "off". CatchallEffective is the resolved state the next scan will use. + CatchallDetection string `json:"catchall_detection"` + CatchallEffective bool `json:"catchall_effective"` } // Asset is one individually-scannable entry of a target (a domain, IP, CIDR, or diff --git a/internal/scanner/http.go b/internal/scanner/http.go index 2d5c19f..49d9700 100644 --- a/internal/scanner/http.go +++ b/internal/scanner/http.go @@ -204,11 +204,12 @@ func (s *HTTPScanner) Run(ctx context.Context, targetID string, logFn LogFunc) e // false positives on it. s.fingerprintHosts(ctx, targetID, logFn) - // Opt-in HTTP catch-all pass (config http_catchall_detection, default OFF): - // groups the 2xx fingerprints gathered above and marks wildcard catch-all - // fleets (redirect-to-apex / homepage clones) so they stop polluting the - // Hosts tab, reports and web modules. Reads stored data only — no requests. - if s.cfg != nil && s.cfg.HTTPCatchallDetection { + // Opt-in HTTP catch-all pass (global config http_catchall_detection, default + // OFF, overridable per target): groups the 2xx fingerprints gathered above and + // marks wildcard catch-all fleets (redirect-to-apex / homepage clones) so they + // stop polluting the Hosts tab, reports and web modules. Reads stored data + // only — no requests. + if s.catchallEnabledForTarget(ctx, targetID) { if err := s.runCatchallPass(ctx, targetID, logFn); err != nil && ctx.Err() == nil { logFn("warn", "http_probe", fmt.Sprintf("HTTP catch-all pass failed: %v", err)) } diff --git a/internal/scanner/http_catchall.go b/internal/scanner/http_catchall.go index caf3852..700e20d 100644 --- a/internal/scanner/http_catchall.go +++ b/internal/scanner/http_catchall.go @@ -2,6 +2,7 @@ package scanner import ( "context" + "database/sql" "encoding/json" "fmt" "net/url" @@ -175,6 +176,19 @@ func qualifiesCatchallGroup(rows []catchallRow, apexTargets map[string]bool) (ma // apex from the targets row, fingerprints all stored 2xx services, marks the // wildcard fleet members (DNS-source rows only) and drops their // http_services rows so web modules never fetch them again. +// catchallEnabledForTarget resolves the per-target tri-state override over the +// global config default: NULL/absent column inherits http_catchall_detection, +// 0 forces the pass off for this target, 1 forces it on. An unknown target or +// query error falls back to the global default (the pass is best-effort). +func (s *HTTPScanner) catchallEnabledForTarget(ctx context.Context, targetID string) bool { + global := s.cfg != nil && s.cfg.HTTPCatchallDetection + var override sql.NullInt64 + if err := s.db.QueryRowContext(ctx, `SELECT catchall_detection FROM targets WHERE id=?`, targetID).Scan(&override); err != nil || !override.Valid { + return global + } + return override.Int64 != 0 +} + func (s *HTTPScanner) runCatchallPass(ctx context.Context, targetID string, logFn LogFunc) error { var apex string if err := s.db.QueryRowContext(ctx, `SELECT domain FROM targets WHERE id=?`, targetID).Scan(&apex); err != nil || strings.TrimSpace(apex) == "" { diff --git a/internal/scanner/http_catchall_test.go b/internal/scanner/http_catchall_test.go index 3577cee..a19980b 100644 --- a/internal/scanner/http_catchall_test.go +++ b/internal/scanner/http_catchall_test.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "testing" + + "github.com/recon-platform/internal/config" ) func TestHostFromServiceURL(t *testing.T) { @@ -185,3 +187,46 @@ func TestRunCatchallPassEndToEnd(t *testing.T) { t.Fatalf("second pass must not delete surviving services, got %d", leftAgain) } } + +// TestCatchallEnabledForTargetMatrix proves the tri-state resolution order: +// a per-target override always wins; NULL (inherit) follows the global config; +// a missing target falls back to the global config too. +func TestCatchallEnabledForTargetMatrix(t *testing.T) { + db, tid := testDB(t) + ctx := context.Background() + + globalOn := &HTTPScanner{db: db, cfg: &config.Config{HTTPCatchallDetection: true}} + globalOff := &HTTPScanner{db: db, cfg: &config.Config{HTTPCatchallDetection: false}} + + setOverride := func(value any) { + t.Helper() + if _, err := db.Exec(`UPDATE targets SET catchall_detection=? WHERE id=?`, value, tid); err != nil { + t.Fatalf("set catchall_detection=%v: %v", value, err) + } + } + + setOverride(nil) // inherit + if !globalOn.catchallEnabledForTarget(ctx, tid) { + t.Fatal("inherit must follow global=on") + } + if globalOff.catchallEnabledForTarget(ctx, tid) { + t.Fatal("inherit must follow global=off") + } + + setOverride(1) // force on, even though the global is off + if !globalOff.catchallEnabledForTarget(ctx, tid) { + t.Fatal("override=on must beat global=off") + } + + setOverride(0) // force off, even though the global is on + if globalOn.catchallEnabledForTarget(ctx, tid) { + t.Fatal("override=off must beat global=on") + } + + if !globalOn.catchallEnabledForTarget(ctx, "missing-target") { + t.Fatal("missing target must fall back to the global config") + } + if globalOff.catchallEnabledForTarget(ctx, "missing-target") { + t.Fatal("missing target must fall back to the global config (off)") + } +} From 891f68d06cf69ecce22645e24269ff61868ec293 Mon Sep 17 00:00:00 2001 From: void Date: Tue, 10 Nov 2026 10:38:27 +0330 Subject: [PATCH 12/14] fix(monitor,enum): two-pass prune, LIKE escaping, changes paging, web-only assets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - subdomains: new miss_count column; pruneRejectedSubdomains deletes only after two CONSECUTIVE missed resolve passes and resolveAll's upsert resets the counter. One dnsx/resolver flake no longer wipes a previously verified DNS row (and its recursion-frontier seed) - subdomain recursion + vhost scan escape LIKE metacharacters in the apex and use ESCAPE '\', so a malformed stored domain cannot widen the pattern - monitoring-changes API takes offset (500/page, id tiebreaker for stable paging); Changes tab gains 'Load older changes' past the newest 500 rows - project-asset monitor only queries kind='web' — ip/cidr assets no longer turn into bogus https:// GETs - tests: two-miss prune, miss reset on resolve, escapeLike, network assets skipped while web assets stay monitored --- frontend/src/lib/api.ts | 5 +- frontend/src/pages/TargetDetail.tsx | 30 ++++- internal/api/finding_handlers.go | 12 +- internal/database/migrations.go | 8 ++ internal/scanner/monitor.go | 6 +- internal/scanner/monitor_assets_test.go | 93 ++++++++++++++++ internal/scanner/subdomain.go | 42 +++++-- internal/scanner/subdomain_admission_test.go | 15 ++- internal/scanner/subdomain_prune_test.go | 109 +++++++++++++++++++ 9 files changed, 305 insertions(+), 15 deletions(-) create mode 100644 internal/scanner/monitor_assets_test.go create mode 100644 internal/scanner/subdomain_prune_test.go diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index e653961..67aa77e 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -301,7 +301,10 @@ export const findings = { req<{ triage: string }>(`/targets/${id}/findings/${fid}/triage`, { method: 'POST', body: JSON.stringify({ triage, note }), }), - monitoringChanges: (id: string) => req(`/targets/${id}/monitoring-changes`), + // Changes are paged server-side, newest first (500 per page) — pass the + // number of rows already loaded to pull the next (older) slice. + monitoringChanges: (id: string, offset = 0) => + req(`/targets/${id}/monitoring-changes${offset > 0 ? `?offset=${offset}` : ''}`), // Stored capture of a monitored file's body at detection time — NOT a live // fetch. Backs the Changes-tab diff drawer (css_change / js_change rows). fileBody: (id: string, kind: 'css' | 'js', url: string, side: 'current' | 'old' = 'current') => diff --git a/frontend/src/pages/TargetDetail.tsx b/frontend/src/pages/TargetDetail.tsx index 81773ae..750ec11 100644 --- a/frontend/src/pages/TargetDetail.tsx +++ b/frontend/src/pages/TargetDetail.tsx @@ -514,6 +514,10 @@ export default function TargetDetail() { // Changes-tab noise filter + the open content-diff drawer (css/js rows). const [changeFilter, setChangeFilter] = useState('all') const [diffView, setDiffView] = useState<{ url: string; kind: 'css' | 'js' } | null>(null) + // Changes tab: the endpoint pages 500 rows at a time. Track whether older + // history remains server-side and a load flag for the footer button. + const [monitorHasMore, setMonitorHasMore] = useState(false) + const [monitorLoadingMore, setMonitorLoadingMore] = useState(false) const [isScanning, setIsScanning] = useState(false) const [monitorSaving, setMonitorSaving] = useState(false) const [catchallSaving, setCatchallSaving] = useState(false) @@ -766,13 +770,29 @@ export default function TargetDetail() { r = all.filter(f => want.has(f.type)) } else if (t === 'cameras') r = await findingsApi.ingram(id) - else if (t === 'monitor') r = await findingsApi.monitoringChanges(id) + else if (t === 'monitor') { + const batch = await findingsApi.monitoringChanges(id) + r = batch + setMonitorHasMore((batch || []).length >= 500) + } setData(r || []) setVisibleCount(PAGE) } catch { setData([]) } setTabLoading(false) } + // Changes tab: pull the next 500-row page of older history. + const loadOlderChanges = async () => { + if (!id || monitorLoadingMore) return + setMonitorLoadingMore(true) + try { + const batch = await findingsApi.monitoringChanges(id, data.length) + setData(prev => [...prev, ...batch]) + setMonitorHasMore(batch.length >= 500) + } catch (e) { console.error(e) } + finally { setMonitorLoadingMore(false) } + } + if (loading) return
if (!target) return null @@ -1669,6 +1689,14 @@ export default function TargetDetail() {
)} + {tab === 'monitor' && !tabLoading && monitorHasMore && !footerHasMore && ( +
+ Showing newest {data.length.toLocaleString()} changes + +
+ )} diff --git a/internal/api/finding_handlers.go b/internal/api/finding_handlers.go index 83acc44..37a6e33 100644 --- a/internal/api/finding_handlers.go +++ b/internal/api/finding_handlers.go @@ -614,11 +614,19 @@ func (h *Handler) handleGetJSFileBody(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleListMonitoringChanges(w http.ResponseWriter, r *http.Request) { id := mux.Vars(r)["id"] + // Newest first, 500 per page. `offset` lets the Changes tab walk further + // back than the newest page (the tab used to silently stop at 500 rows); + // id is the tiebreaker so rows sharing detected_at never shuffle between + // pages. + offset := 0 + if n, err := strconv.Atoi(r.URL.Query().Get("offset")); err == nil && n > 0 { + offset = n + } rows, err := h.db.QueryContext(r.Context(), ` SELECT id, target_id, url, change_type, old_value, new_value, COALESCE(severity,'info'), detected_at FROM monitoring_changes WHERE target_id = ? - ORDER BY detected_at DESC LIMIT 500 - `, id) + ORDER BY detected_at DESC, id DESC LIMIT 500 OFFSET ? + `, id, offset) if err != nil { h.writeError(w, http.StatusInternalServerError, "query failed") return diff --git a/internal/database/migrations.go b/internal/database/migrations.go index bd48f36..ffa84b4 100644 --- a/internal/database/migrations.go +++ b/internal/database/migrations.go @@ -100,6 +100,7 @@ func RunMigrations(db *DB) error { alterNucleiFindingsAddVerification, alterNucleiFindingsAddConfidence, alterSubdomainsAddSource, + alterSubdomainsAddMissCount, alterCandidatesAddStateVersion, alterVulnFindingsAddCandidateID, alterVulnFindingsAddLifecycle, @@ -1077,6 +1078,13 @@ CREATE TABLE IF NOT EXISTS subdomains ( // hosts, which are often internal/forgotten and high-value. const alterSubdomainsAddSource = `ALTER TABLE subdomains ADD COLUMN source TEXT DEFAULT 'dns';` +// A name is not dead after ONE missed resolve pass: a single resolver timeout +// or dnsx hiccup must not delete a previously verified row (and, with it, the +// recursion frontier seed). pruneRejectedSubdomains increments this counter on +// every rejected pass and only deletes at 2 consecutive misses; a pass that +// resolves the name resets it to 0 via the admission upsert. +const alterSubdomainsAddMissCount = `ALTER TABLE subdomains ADD COLUMN miss_count INTEGER DEFAULT 0;` + // catchall marks subdomains whose HTTP responses match the target's wildcard // catch-all fleet (redirect-to-apex groups or homepage-content clones). Rows are // marked, never deleted: operator seeds, vhost findings and previously-alive diff --git a/internal/scanner/monitor.go b/internal/scanner/monitor.go index 3a1b4dd..45dca0c 100644 --- a/internal/scanner/monitor.go +++ b/internal/scanner/monitor.go @@ -439,8 +439,12 @@ func monitorAssetCandidates(value, assetType, lastURL string) []string { } func (s *MonitorScanner) monitorProjectAssets(ctx context.Context, targetID, targetScope string, authHeaders map[string]string, knownHTTP, knownJS map[string]bool, logFn LogFunc) (int, error) { + // Only web assets are monitorable. A network asset (ip/cidr) has no page to + // fetch: building https://10.0.0.0/24 from it would just burn a request and + // record a meaningless "change" on the range string itself. rows, err := s.db.QueryContext(ctx, `SELECT id,value,COALESCE(asset_type,'domain'),COALESCE(metadata,'{}') - FROM assets WHERE target_id=? AND COALESCE(approval_status,'approved')='approved' AND COALESCE(monitor_enabled,1)=1`, targetID) + FROM assets WHERE target_id=? AND COALESCE(approval_status,'approved')='approved' AND COALESCE(monitor_enabled,1)=1 + AND COALESCE(kind,'web')='web'`, targetID) if err != nil { return 0, err } diff --git a/internal/scanner/monitor_assets_test.go b/internal/scanner/monitor_assets_test.go new file mode 100644 index 0000000..f75defe --- /dev/null +++ b/internal/scanner/monitor_assets_test.go @@ -0,0 +1,93 @@ +package scanner + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/google/uuid" +) + +// Network assets (ip/cidr) are not web pages. The page monitor must never build +// a pseudo-URL from one and fetch it — the counting server proves no request is +// made. A regression here means every monitored network range gets a bogus +// https:// GET on each watch pass. +func TestMonitorProjectAssetsSkipsNetworkAssets(t *testing.T) { + var mu sync.Mutex + hits := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + hits++ + mu.Unlock() + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(`okok`)) + })) + defer srv.Close() + + db := newMonitorTestDB(t) + targetID := uuid.NewString() + if _, err := db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, "example.test"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO assets(id,target_id,value,kind,asset_type,approval_status,monitor_enabled) + VALUES(?,?,?,?,?,?,1)`, + uuid.NewString(), targetID, strings.TrimPrefix(srv.URL, "http://"), "network", "ip", "approved"); err != nil { + t.Fatal(err) + } + + s := &MonitorScanner{db: db} + logFn := func(string, string, string) {} + changed, err := s.monitorProjectAssets(context.Background(), targetID, "", nil, + map[string]bool{}, map[string]bool{}, logFn) + if err != nil { + t.Fatal(err) + } + mu.Lock() + got := hits + mu.Unlock() + if changed != 0 || got != 0 { + t.Fatalf("network asset was monitored: changed=%d requests=%d", changed, got) + } +} + +// The counterpart: a web asset IS monitored (so the kind filter above cannot +// silently starve the whole project-asset monitor path). +func TestMonitorProjectAssetsStillMonitorsWebAssets(t *testing.T) { + var mu sync.Mutex + hits := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + hits++ + mu.Unlock() + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(`okok`)) + })) + defer srv.Close() + + db := newMonitorTestDB(t) + targetID := uuid.NewString() + if _, err := db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, "example.test"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO assets(id,target_id,value,kind,asset_type,approval_status,monitor_enabled) + VALUES(?,?,?,?,?,?,1)`, + uuid.NewString(), targetID, srv.URL, "web", "url", "approved"); err != nil { + t.Fatal(err) + } + + s := &MonitorScanner{db: db} + logFn := func(string, string, string) {} + if _, err := s.monitorProjectAssets(context.Background(), targetID, "", nil, + map[string]bool{}, map[string]bool{}, logFn); err != nil { + t.Fatal(err) + } + mu.Lock() + got := hits + mu.Unlock() + if got == 0 { + t.Fatal("web asset was never fetched — the kind filter is too broad") + } +} diff --git a/internal/scanner/subdomain.go b/internal/scanner/subdomain.go index 4c6166a..08f9aa1 100644 --- a/internal/scanner/subdomain.go +++ b/internal/scanner/subdomain.go @@ -564,8 +564,8 @@ func (s *SubdomainScanner) vhostScan(ctx context.Context, targetID, domain strin // Host-header probed at all). rows, err := s.db.QueryContext(ctx, `SELECT DISTINCT ip FROM subdomains - WHERE target_id=? AND ip!='' AND (subdomain=? OR subdomain LIKE ?) - LIMIT 100`, targetID, domain, "%."+domain) + WHERE target_id=? AND ip!='' AND (subdomain=? OR subdomain LIKE ? ESCAPE '\') + LIMIT 100`, targetID, domain, "%."+escapeLike(domain)) if err != nil { return } @@ -1290,7 +1290,8 @@ func (s *SubdomainScanner) resolveAll(ctx context.Context, targetID string, foun ip = excluded.ip, source = CASE WHEN COALESCE(subdomains.source,'dns') IN ('seed','vhost') THEN subdomains.source ELSE excluded.source END, - last_seen = CURRENT_TIMESTAMP`) + last_seen = CURRENT_TIMESTAMP, + miss_count = 0`) if perr == nil { for _, r := range rows { _, _ = stmt.ExecContext(ctx, uuid.New().String(), targetID, r.name, r.ip, r.source) @@ -1424,10 +1425,21 @@ func (s *SubdomainScanner) recursiveEnum(ctx context.Context, targetID, apex str // resolvingSubdomainsUnder returns the stored, resolving (has an IP) subdomains // strictly under apex, excluding anything already visited — the seed frontier // for recursive enumeration. +// escapeLike neutralises SQLite LIKE metacharacters ('%', '_' and the escape +// character itself) in a value folded into a LIKE pattern. Apex names are +// validated at intake, but escaping keeps a malformed stored domain from +// widening the pattern beyond its own zone. +func escapeLike(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `%`, `\%`) + s = strings.ReplaceAll(s, `_`, `\_`) + return s +} + func (s *SubdomainScanner) resolvingSubdomainsUnder(ctx context.Context, targetID, apex string, visited map[string]bool) []string { rows, err := s.db.QueryContext(ctx, `SELECT subdomain FROM subdomains WHERE target_id=? AND COALESCE(catchall,0)=0 AND COALESCE(ip,'')!='' - AND subdomain LIKE ?`, targetID, "%."+strings.ToLower(apex)) + AND subdomain LIKE ? ESCAPE '\'`, targetID, "%."+escapeLike(strings.ToLower(apex))) if err != nil { return nil } @@ -1556,6 +1568,11 @@ func (s *SubdomainScanner) upsertSubdomainSource(targetID, subdomain, ip, source // pruneRejectedSubdomains repairs rows created by older eager-admission builds. // The delete is intentionally narrow: never touch explicit seeds, CNAME records, // verified vhosts or anything that was previously observed alive. +// pruneRejectedSubdomains removes DNS-sourced rows for names that stopped +// resolving. ONE missed pass is not proof of death — a resolver timeout or a +// dnsx flake must not wipe a previously verified row (and, with it, its +// recursion-frontier seed) — so deletion requires two CONSECUTIVE misses. A +// pass that resolves the name resets the counter in resolveAll's upsert. func (s *SubdomainScanner) pruneRejectedSubdomains(ctx context.Context, targetID string, rejected map[string]bool) { if len(rejected) == 0 { return @@ -1564,20 +1581,31 @@ func (s *SubdomainScanner) pruneRejectedSubdomains(ctx context.Context, targetID if err != nil { return } - stmt, err := tx.PrepareContext(ctx, `DELETE FROM subdomains + upd, err := tx.PrepareContext(ctx, `UPDATE subdomains + SET miss_count = COALESCE(miss_count,0) + 1 WHERE target_id=? AND subdomain=? AND COALESCE(source,'dns')='dns' AND COALESCE(is_alive,0)=0`) if err != nil { _ = tx.Rollback() return } + del, err := tx.PrepareContext(ctx, `DELETE FROM subdomains + WHERE target_id=? AND subdomain=? AND COALESCE(source,'dns')='dns' + AND COALESCE(is_alive,0)=0 AND COALESCE(miss_count,0)>=2`) + if err != nil { + _ = upd.Close() + _ = tx.Rollback() + return + } for name := range rejected { if ctx.Err() != nil { break } - _, _ = stmt.ExecContext(ctx, targetID, name) + _, _ = upd.ExecContext(ctx, targetID, name) + _, _ = del.ExecContext(ctx, targetID, name) } - _ = stmt.Close() + _ = upd.Close() + _ = del.Close() _ = tx.Commit() } diff --git a/internal/scanner/subdomain_admission_test.go b/internal/scanner/subdomain_admission_test.go index 8054ab9..f1648bf 100644 --- a/internal/scanner/subdomain_admission_test.go +++ b/internal/scanner/subdomain_admission_test.go @@ -66,18 +66,27 @@ func TestRejectedAdmissionPrunesOnlyUnverifiedDNSRows(t *testing.T) { insert("vhost.m.local", "vhost", 0) s := &SubdomainScanner{db: db} - s.pruneRejectedSubdomains(context.Background(), tid, map[string]bool{ + rejected := map[string]bool{ "junk.m.local": true, "historically-live.m.local": true, "seed.m.local": true, "cname.m.local": true, "vhost.m.local": true, - }) + } + // One missed resolve pass must never delete: transient resolver failures are + // common. The never-live eager DNS row survives the first pass at miss_count=1. + s.pruneRejectedSubdomains(context.Background(), tid, rejected) var n int _ = db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE target_id=?`, tid).Scan(&n) + if n != 5 { + t.Fatalf("a single missed pass pruned rows early: rows=%d", n) + } + // The second CONSECUTIVE miss prunes only that never-live eager DNS row. + s.pruneRejectedSubdomains(context.Background(), tid, rejected) + _ = db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE target_id=?`, tid).Scan(&n) if n != 4 { t.Fatalf("expected only the never-live eager DNS row to be pruned, rows=%d", n) } _ = db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE target_id=? AND subdomain='junk.m.local'`, tid).Scan(&n) if n != 0 { - t.Fatal("unverified junk DNS row survived admission cleanup") + t.Fatal("unverified junk DNS row survived two consecutive missed passes") } } diff --git a/internal/scanner/subdomain_prune_test.go b/internal/scanner/subdomain_prune_test.go new file mode 100644 index 0000000..160ddce --- /dev/null +++ b/internal/scanner/subdomain_prune_test.go @@ -0,0 +1,109 @@ +package scanner + +import ( + "context" + "testing" + + "github.com/google/uuid" +) + +// A single missed resolve pass must not delete a previously known row. The +// second CONSECUTIVE miss does. This guards against one dnsx/resolver hiccup +// wiping a target's DNS history and its recursion-frontier seeds. +func TestPruneRejectedSubdomainsRequiresTwoConsecutiveMisses(t *testing.T) { + db := newMonitorTestDB(t) + targetID := uuid.NewString() + if _, err := db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, "example.test"); err != nil { + t.Fatal(err) + } + ins := func(name, ip, source string, alive int) { + if _, err := db.Exec(`INSERT INTO subdomains(id,target_id,subdomain,ip,source,is_alive) VALUES(?,?,?,?,?,?)`, + uuid.NewString(), targetID, name, ip, source, alive); err != nil { + t.Fatal(err) + } + } + ins("dns.example.test", "1.2.3.4", "dns", 0) + ins("alive.example.test", "1.2.3.5", "dns", 1) + ins("seed.example.test", "1.2.3.6", "seed", 0) + + s := &SubdomainScanner{db: db} + rejected := map[string]bool{ + "dns.example.test": true, + "alive.example.test": true, + "seed.example.test": true, + } + count := func(name string) int { + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM subdomains WHERE target_id=? AND subdomain=?`, targetID, name).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + misses := func(name string) int { + var n int + if err := db.QueryRow(`SELECT COALESCE(miss_count,0) FROM subdomains WHERE target_id=? AND subdomain=?`, targetID, name).Scan(&n); err != nil { + t.Fatal(err) + } + return n + } + + s.pruneRejectedSubdomains(context.Background(), targetID, rejected) + if count("dns.example.test") != 1 || misses("dns.example.test") != 1 { + t.Fatalf("first miss deleted the row (count=%d misses=%d)", count("dns.example.test"), misses("dns.example.test")) + } + s.pruneRejectedSubdomains(context.Background(), targetID, rejected) + if count("dns.example.test") != 0 { + t.Fatal("second consecutive miss did not prune the stale DNS row") + } + if count("alive.example.test") != 1 || count("seed.example.test") != 1 { + t.Fatal("prune must never touch alive or operator-seeded rows") + } +} + +// A pass that resolves the name resets the miss counter, so two flakes years +// apart can never accumulate into a deletion without two CONSECUTIVE misses. +func TestResolvingUpsertResetsMissCount(t *testing.T) { + db := newMonitorTestDB(t) + targetID := uuid.NewString() + if _, err := db.Exec(`INSERT INTO targets(id,domain) VALUES(?,?)`, targetID, "example.test"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`INSERT INTO subdomains(id,target_id,subdomain,ip,source,is_alive,miss_count) VALUES(?,?,?,?,?,0,1)`, + uuid.NewString(), targetID, "back.example.test", "1.2.3.4", "dns"); err != nil { + t.Fatal(err) + } + // The exact upsert resolveAll runs when a name is admitted this pass. + if _, err := db.Exec(` + INSERT INTO subdomains (id, target_id, subdomain, ip, source, last_seen) + VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(target_id, subdomain) DO UPDATE SET + ip = excluded.ip, + source = CASE WHEN COALESCE(subdomains.source,'dns') IN ('seed','vhost') + THEN subdomains.source ELSE excluded.source END, + last_seen = CURRENT_TIMESTAMP, + miss_count = 0`, + uuid.NewString(), targetID, "back.example.test", "5.6.7.8", "dns"); err != nil { + t.Fatal(err) + } + var misses int + if err := db.QueryRow(`SELECT COALESCE(miss_count,0) FROM subdomains WHERE target_id=? AND subdomain='back.example.test'`, targetID).Scan(&misses); err != nil { + t.Fatal(err) + } + if misses != 0 { + t.Fatalf("resolving pass left miss_count=%d, want 0", misses) + } +} + +func TestEscapeLike(t *testing.T) { + cases := map[string]string{ + "plain.test": "plain.test", + `back\slash`: `back\\slash`, + "pct%name": `pct\%name`, + "under_score": `under\_score`, + } + for in, want := range cases { + if got := escapeLike(in); got != want { + t.Errorf("escapeLike(%q) = %q, want %q", in, got, want) + } + } +} From 035539fc5c6f7569daaf38764d37b23169240b83 Mon Sep 17 00:00:00 2001 From: void Date: Tue, 10 Nov 2026 10:44:47 +0330 Subject: [PATCH 13/14] chore: address minor findings from catch-all override audit - List/create target responses omit catchall_detection/catchall_effective instead of emitting empty values; the detail endpoint now always sends an explicit effective bool via a pointer field. - Move the runCatchallPass doc comment back above its function. - Surface a toast when saving the per-target catch-all override fails. - Record the audit findings and status in TODO-minor-findings.md. --- TODO-minor-findings.md | 60 +++++++++++++++++++++++++++++ frontend/src/pages/TargetDetail.tsx | 5 ++- internal/api/target_handlers.go | 9 +++-- internal/models/models.go | 7 +++- internal/scanner/http_catchall.go | 8 ++-- 5 files changed, 79 insertions(+), 10 deletions(-) create mode 100644 TODO-minor-findings.md diff --git a/TODO-minor-findings.md b/TODO-minor-findings.md new file mode 100644 index 0000000..46d7918 --- /dev/null +++ b/TODO-minor-findings.md @@ -0,0 +1,60 @@ +# TODO — minor findings + +Recorded: 2026-10-25 +Source: validation audit of commit `0b6ebe1` (per-target HTTP catch-all override) +Verdict: no functional bugs. Items below are polish/consistency, none block use. + +Status: items 1–3 fixed in the follow-up commit (see git log). Items 4–5 remain +open by design / pending a decision. + +## 1. List endpoints emit empty override strings (API consistency) + +`GET /targets` (handleListTargets) and the `POST /targets` create response +build `models.Target` without the new catchall fields, so JSON contains: + + "catchall_detection": "", + "catchall_effective": false + +The detail endpoint (`GET /targets/{id}`) always returns `inherit|on|off`. +The UI reads these fields only from the detail endpoint, so there is no UI +effect today. External API clients could misread the empty string. + +Fix options: +- Fill the fields in the list/create handlers (compute like handleGetTarget), or +- add `omitempty` to both JSON tags in `internal/models/models.go`. + +## 2. Doc comment misattached (cosmetic) + +In `internal/scanner/http_catchall.go`, the `runCatchallPass` doc comment now +sits above `catchallEnabledForTarget` because the helper was inserted between +the comment and its function. `runCatchallPass` has no doc comment of its own. + +Fix: move the comment down, or give each function its own short comment. + +## 3. Silent UI failure on catch-all save (UX) + +`saveCatchall` in `frontend/src/pages/TargetDetail.tsx` swallows errors +(`catch { /* ignore */ }`), same pattern as `saveMonitor`. On a failed PUT the +select snaps back to the previous value with no toast. + +Fix: add an error toast (the page already has `addToast`). + +## 4. DB error during override lookup falls back to global (edge) + +`catchallEnabledForTarget` returns the global config when the targets query +errors. An explicitly-Off target could therefore run one catch-all pass during +a transient DB error. The pass is best-effort by design; noted so the behavior +is not a surprise. + +Optional fix: fail closed for explicit overrides by caching the resolved value +per scan instead of re-querying. + +## 5. Turning the override Off does not unmark existing rows (by design) + +Marked rows keep `catchall = 1` and stay hidden from the Hosts tab, reports, +graph, stats and wordlist learning. This is the mark-never-delete design: the +override only controls FUTURE passes. + +Optional follow-up (needs a decision): a small "unmark" action or a cleanup +query (`UPDATE subdomains SET catchall = 0 WHERE target_id = ?`) exposed in +the UI. diff --git a/frontend/src/pages/TargetDetail.tsx b/frontend/src/pages/TargetDetail.tsx index 750ec11..5f86294 100644 --- a/frontend/src/pages/TargetDetail.tsx +++ b/frontend/src/pages/TargetDetail.tsx @@ -851,7 +851,10 @@ export default function TargetDetail() { await targetsApi.update(id, { catchall_detection: value }) // Refresh so catchall_effective reflects the global config on "inherit". targetsApi.get(id).then(setTarget).catch(() => {}) - } catch { /* ignore */ } finally { setCatchallSaving(false) } + addToast('success', 'Catch-all detection updated') + } catch (e) { + addToast('error', e instanceof Error ? e.message : 'Failed to update catch-all detection') + } finally { setCatchallSaving(false) } } // Network targets' actual results (Nuclei/Vulns/Candidates) used to sit diff --git a/internal/api/target_handlers.go b/internal/api/target_handlers.go index 7e7d6cd..3424a33 100644 --- a/internal/api/target_handlers.go +++ b/internal/api/target_handlers.go @@ -396,13 +396,16 @@ func (h *Handler) handleGetTarget(w http.ResponseWriter, r *http.Request) { switch catchallOverride { case 1: t.CatchallDetection = "on" - t.CatchallEffective = true + on := true + t.CatchallEffective = &on case 0: t.CatchallDetection = "off" - t.CatchallEffective = false + off := false + t.CatchallEffective = &off default: t.CatchallDetection = "inherit" - t.CatchallEffective = h.cfg != nil && h.cfg.HTTPCatchallDetection + effective := h.cfg != nil && h.cfg.HTTPCatchallDetection + t.CatchallEffective = &effective } t.ScanHeaders = map[string]string{} _ = json.Unmarshal([]byte(scanHeadersJSON), &t.ScanHeaders) diff --git a/internal/models/models.go b/internal/models/models.go index c4bb26b..ee91f8a 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -32,8 +32,11 @@ type Target struct { // CatchallDetection is the per-target override of the HTTP catch-all pass: // "inherit" (follow the global http_catchall_detection setting), "on" or // "off". CatchallEffective is the resolved state the next scan will use. - CatchallDetection string `json:"catchall_detection"` - CatchallEffective bool `json:"catchall_effective"` + CatchallDetection string `json:"catchall_detection,omitempty"` + // CatchallEffective is a pointer so list/create responses (which do not + // resolve the override) omit the field entirely, while the detail endpoint + // always sends an explicit true/false. + CatchallEffective *bool `json:"catchall_effective,omitempty"` } // Asset is one individually-scannable entry of a target (a domain, IP, CIDR, or diff --git a/internal/scanner/http_catchall.go b/internal/scanner/http_catchall.go index 700e20d..8da48c7 100644 --- a/internal/scanner/http_catchall.go +++ b/internal/scanner/http_catchall.go @@ -172,10 +172,6 @@ func qualifiesCatchallGroup(rows []catchallRow, apexTargets map[string]bool) (ma return nil, "" } -// runCatchallPass implements the opt-in step-2 pass. It resolves the target's -// apex from the targets row, fingerprints all stored 2xx services, marks the -// wildcard fleet members (DNS-source rows only) and drops their -// http_services rows so web modules never fetch them again. // catchallEnabledForTarget resolves the per-target tri-state override over the // global config default: NULL/absent column inherits http_catchall_detection, // 0 forces the pass off for this target, 1 forces it on. An unknown target or @@ -189,6 +185,10 @@ func (s *HTTPScanner) catchallEnabledForTarget(ctx context.Context, targetID str return override.Int64 != 0 } +// runCatchallPass implements the opt-in step-2 pass. It resolves the target's +// apex from the targets row, fingerprints all stored 2xx services, marks the +// wildcard fleet members (DNS-source rows only) and drops their +// http_services rows so web modules never fetch them again. func (s *HTTPScanner) runCatchallPass(ctx context.Context, targetID string, logFn LogFunc) error { var apex string if err := s.db.QueryRowContext(ctx, `SELECT domain FROM targets WHERE id=?`, targetID).Scan(&apex); err != nil || strings.TrimSpace(apex) == "" { From 6ec3a63ac56c76b4b2d4f5f7597428a635b99ea3 Mon Sep 17 00:00:00 2001 From: void Date: Tue, 10 Nov 2026 10:51:12 +0330 Subject: [PATCH 14/14] fix(ui): prevent Changes tab crash on tab switch TargetDetail shares one data state across tabs and replaces it only after the next tab's fetch resolves. Clicking Changes with rows loaded in the previous tab therefore rendered those rows under the monitor branch for one frame; the filter chips and severity badge dereferenced change_type on rows that do not have it: TypeError: Cannot read properties of undefined (reading 'includes') The inline throw is outside the page's own ErrorBoundary, so the route unmounted instead of showing a tab-level error. - tab click now clears data and shows the skeleton before switching, so a tab never renders another tab's row shape (silent refresh paths keep the existing rows) - monitorRowKind/changeBadgeVariant tolerate a missing change_type - inner ErrorBoundary keys on the active tab so a caught error cannot stick after a switch - regression test for foreign-row classification --- .../src/pages/TargetDetail.monitor.test.ts | 31 ++++++++++++++++ frontend/src/pages/TargetDetail.tsx | 35 ++++++++++++++----- 2 files changed, 58 insertions(+), 8 deletions(-) create mode 100644 frontend/src/pages/TargetDetail.monitor.test.ts diff --git a/frontend/src/pages/TargetDetail.monitor.test.ts b/frontend/src/pages/TargetDetail.monitor.test.ts new file mode 100644 index 0000000..ca95a13 --- /dev/null +++ b/frontend/src/pages/TargetDetail.monitor.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from 'vitest' +import { changeBadgeVariant, monitorRowKind } from './TargetDetail' +import type { MonitoringChange } from '../types' + +// Regression: clicking the Changes tab used to render the PREVIOUS tab's rows +// for one frame (data is replaced only after the fetch resolves). A subdomain +// row has no change_type, and the old helpers dereferenced it directly: +// TypeError: Cannot read properties of undefined (reading 'includes') +// The chips call monitorRowKind() for every loaded row, so the throw happened +// before React could commit the tab. Both helpers must classify any row shape. +describe('Changes tab row helpers tolerate foreign rows', () => { + it('classifies a stale non-monitor row instead of throwing', () => { + const stale = { id: 's1', target_id: 't1', subdomain: 'a.example.test' } as unknown as MonitoringChange + expect(monitorRowKind(stale)).toBe('http') + expect(changeBadgeVariant(stale)).toBe('info') + }) + + it('still classifies real monitor rows', () => { + const css = { change_type: 'css_change', severity: 'info' } as MonitoringChange + const js = { change_type: 'js_change', severity: 'medium' } as MonitoringChange + const sheet = { change_type: 'security:stylesheet_added', severity: 'info' } as MonitoringChange + const header = { change_type: 'security:header_removed', severity: 'high' } as MonitoringChange + expect(monitorRowKind(css)).toBe('css') + expect(monitorRowKind(js)).toBe('scripts') + expect(monitorRowKind(sheet)).toBe('css') + expect(monitorRowKind(header)).toBe('headers') + expect(changeBadgeVariant(css)).toBe('info') + expect(changeBadgeVariant(js)).toBe('medium') + expect(changeBadgeVariant(header)).toBe('high') + }) +}) diff --git a/frontend/src/pages/TargetDetail.tsx b/frontend/src/pages/TargetDetail.tsx index 5f86294..ad55337 100644 --- a/frontend/src/pages/TargetDetail.tsx +++ b/frontend/src/pages/TargetDetail.tsx @@ -35,8 +35,11 @@ const CHANGE_FILTERS: { id: ChangeFilter; label: string }[] = [ // Bucket order matters: the stylesheet/script markers win before the generic // "security:" prefix, so a security:stylesheet_* row lands in Stylesheets (not // Security), a security:script_* row in Scripts, and header removals in Headers. -const monitorRowKind = (c: MonitoringChange): ChangeFilter => { - const ct = c.change_type +export const monitorRowKind = (c: MonitoringChange): ChangeFilter => { + // One render can hold rows from the PREVIOUS tab while the next tab's fetch + // is in flight (data is replaced only after the request resolves). Never + // assume the row really is a monitoring_changes row. + const ct = typeof c?.change_type === 'string' ? c.change_type : '' if (ct === 'css_change' || ct.includes('stylesheet')) return 'css' if (ct === 'js_change' || ct.includes('script')) return 'scripts' if (ct.includes('header')) return 'headers' @@ -47,12 +50,15 @@ const monitorRowKind = (c: MonitoringChange): ChangeFilter => { // Badge color follows the stored severity (rows written since the severity // column carry one); older rows without it keep the old type-based estimate. -const changeBadgeVariant = (c: { severity?: string; change_type: string }): 'high' | 'medium' | 'low' | 'info' => { +export const changeBadgeVariant = (c: { severity?: string; change_type?: string }): 'high' | 'medium' | 'low' | 'info' => { const sev = (c.severity || '').toLowerCase() if (sev === 'high' || sev === 'medium' || sev === 'low' || sev === 'info') return sev - return c.change_type === 'js_change' ? 'medium' - : c.change_type.startsWith('security') || c.change_type.includes('header') ? 'high' - : c.change_type === 'tech_change' ? 'low' + // Same stale-row hazard as monitorRowKind: the badge must render even when a + // previous tab's row is briefly shown under the Changes tab. + const ct = typeof c.change_type === 'string' ? c.change_type : '' + return ct === 'js_change' ? 'medium' + : ct.startsWith('security') || ct.includes('header') ? 'high' + : ct === 'tech_change' ? 'low' : 'info' } @@ -1112,7 +1118,18 @@ export default function TargetDetail() { {gi > 0 && } {g.group} {g.tabs.map(t => ( -