feat(data): point-in-time fundamental fields (growth / valuation / quality)

src/quant_forge/agent_workspace/tools.py

@@ -40,13 +40,14 @@ def __init__(
             horizon_days_matrix=horizon_days_matrix,
         )
         self.factor_root = factor_root
+        self.data_root = data_root
         self.factor_values_root = factor_values_root
         self.factor_values_manifest_root = factor_values_manifest_root
         self.artifact_root = artifact_root
 
     def read_catalog(self) -> dict[str, object]:
         return {
-            "fields": read_models.list_available_fields(),
+            "fields": read_models.list_runtime_available_fields(self.data_root),
             "operators": read_models.list_available_operators(),
             "factors": read_models.list_factors(
                 self.factor_root,

src/quant_forge/apps/cli/main.py

@@ -16,7 +16,7 @@
     load_config,
     validate_llm_runtime,
 )
-from quant_forge.data.local import create_demo_workspace, validate_data_root
+from quant_forge.data.local import LocalPanelDataProvider, create_demo_workspace, validate_data_root
 from quant_forge.factor_library.catalog import (
     FactorCatalog,
     discover_factor_value_roots,
@@ -109,6 +109,19 @@ def build_parser() -> argparse.ArgumentParser:
     validate.add_argument("--data-root", type=Path)
     validate.set_defaults(handler=_cmd_data_validate)
 
+    build_fund = data_subcommands.add_parser(
+        "build-fundamentals",
+        help="materialize a point-in-time fundamentals overlay from a local source layer",
+    )
+    _add_config_options(build_fund)
+    build_fund.add_argument("--data-root", type=Path)
+    build_fund.add_argument(
+        "--output",
+        type=Path,
+        help="overlay parquet path (default: fundamentals.parquet next to the panel)",
+    )
+    build_fund.set_defaults(handler=_cmd_data_build_fundamentals)
+
     factor = subcommands.add_parser("factor", help="factor library commands")
     factor_subcommands = factor.add_subparsers(dest="factor_command", required=True)
     list_cmd = factor_subcommands.add_parser("list", help="list factors")
@@ -485,6 +498,54 @@ def _cmd_data_validate(args: argparse.Namespace) -> int:
     return 0 if result.ok else 2
 
 
+def _cmd_data_build_fundamentals(args: argparse.Namespace) -> int:
+    """Materialize the PIT fundamentals overlay from a locally-materialized
+    source layer (its path comes from the local config only) and write the
+    daily [trade_date, instrument, <fields>] overlay next to the panel."""
+
+    import sys
+
+    import pandas as pd
+
+    from quant_forge.data.fundamentals import build_fundamentals_overlay
+    from quant_forge.data.local import FUNDAMENTALS_OVERLAY_FILE, resolve_panel_path
+
+    paths = _runtime_paths(args)
+    source_root = paths.fundamentals_source_root
+    if source_root is None:
+        print(
+            "fundamentals_source_root is not configured; set paths.fundamentals_source_root "
+            "in a local config that points at a locally-materialized source layer",
+            file=sys.stderr,
+        )
+        return 2
+    panel_path = resolve_panel_path(paths.data_root)
+    if not panel_path.exists():
+        print(f"panel not found under data_root (looked for {panel_path.name})", file=sys.stderr)
+        return 2
+
+    panel_keys = pd.read_parquet(panel_path, columns=["trade_date", "instrument"])
+    overlay = build_fundamentals_overlay(source_root, panel_keys)
+
+    # Default output is the fundamentals.parquet sibling of the panel (what the
+    # loader auto-discovers); --output overrides for a custom location.
+    output = args.output if args.output is not None else panel_path.parent / FUNDAMENTALS_OVERLAY_FILE
+    output.parent.mkdir(parents=True, exist_ok=True)
+    overlay.to_parquet(output, index=False)
+
+    fields = [c for c in overlay.columns if c not in ("trade_date", "instrument")]
+    _print_json(
+        {
+            "output": output.name,
+            "rows": int(len(overlay)),
+            "instruments": int(overlay["instrument"].nunique()),
+            "fields": fields,
+            "non_null_counts": {c: int(overlay[c].notna().sum()) for c in fields},
+        }
+    )
+    return 0
+
+
 def _cmd_doctor(args: argparse.Namespace) -> int:
     payload = _doctor_payload(args)
     _print_json(payload)
@@ -582,7 +643,13 @@ def _cmd_llm_smoke(args: argparse.Namespace) -> int:
     config = _config(args)
     selected = config.llm.select_provider(args.provider)
     validate_llm_runtime(config.llm, args.provider)
-    parsed = parse_factor_idea(args.text, selected, mode="llm")
+    available_fields = LocalPanelDataProvider(config.paths.data_root).available_field_names()
+    parsed = parse_factor_idea(
+        args.text,
+        selected,
+        mode="llm",
+        available_fields=available_fields,
+    )
     _print_json(
         {
             "ok": True,
@@ -1571,6 +1638,7 @@ def _runtime_paths_from_config(args: argparse.Namespace, config: QuantForgeConfi
         or paths.factor_values_overlay_root,
         factor_values_manifest_root=getattr(args, "factor_values_manifest_root", None)
         or paths.factor_values_manifest_root,
+        fundamentals_source_root=paths.fundamentals_source_root,
         artifact_root=getattr(args, "artifact_root", None) or paths.artifact_root,
         output_root=paths.output_root,
     )

src/quant_forge/apps/web/api.py

@@ -54,6 +54,7 @@
     apply_test_period,
     prepare_factor_scores_result,
 )
+from quant_forge.factor_engine.formula_parser import formula_input_fields
 from quant_forge.factor_engine.value_store import FactorValueStore
 from quant_forge.factor_library.catalog import FactorCatalog, is_precomputed_formula
 from quant_forge.factor_library.repository import FactorRepository
@@ -234,7 +235,13 @@ def _parse_idea(
     if parser_mode == "llm":
         validate_llm_runtime(config.llm, llm_provider)
     _raise_if_cancelled(cancel_event)
-    parsed = _server.parse_factor_idea(text, llm_settings, mode=parser_mode)
+    available_fields = LocalPanelDataProvider(config.paths.data_root).available_field_names()
+    parsed = _server.parse_factor_idea(
+        text,
+        llm_settings,
+        mode=parser_mode,
+        available_fields=available_fields,
+    )
     _raise_if_cancelled(cancel_event)
     return parsed
 
@@ -799,7 +806,16 @@ def _prepare_multi_factor_backtest(
         members, directions=directions, universe_filters=universe_filters
     )
 
-    panel = LocalPanelDataProvider(config.paths.data_root).load_panel()
+    required_fields = tuple(
+        dict.fromkeys(
+            field
+            for member in members
+            for field in formula_input_fields(member.formula)
+        )
+    )
+    panel = LocalPanelDataProvider(config.paths.data_root).load_panel(
+        required_fields=required_fields
+    )
     working_panel = apply_test_period(panel, settings.profile)
     dates = tuple(sorted(working_panel["trade_date"].drop_duplicates()))
     period_count = require_backtest_window(

src/quant_forge/apps/web/pipeline.py

@@ -117,6 +117,7 @@
 
 from __future__ import annotations
 
+from collections.abc import Iterable
 from dataclasses import dataclass
 from dataclasses import replace as dataclass_replace
 from datetime import UTC, datetime, timedelta
@@ -147,6 +148,7 @@
 )
 from quant_forge.config import QuantForgeConfig
 from quant_forge.core.contracts import FactorDefinition
+from quant_forge.data.local import LocalPanelDataProvider
 from quant_forge.factor_engine.value_store import remove_stored_values_for_factor_id
 from quant_forge.factor_library.repository import FactorRepository
 from quant_forge.operator_registry import canonical_formula_fingerprint
@@ -2207,6 +2209,7 @@ def create_pipeline_from_edited_formula(
             name=str(parent.factor.get("name", "")),
             horizon_days=effective_horizon,
             universe_filters=new_filters,
+            available_fields=LocalPanelDataProvider(config.paths.data_root).available_field_names(),
         )
         if pre.get("status") != "ready":
             raise PipelineConflictError(
@@ -2291,10 +2294,15 @@ def pre_validate_formula(
     name: Any = "",
     horizon_days: Any = 5,
     universe_filters: tuple[str, ...] = (),
+    available_fields: Iterable[str] | None = None,
 ) -> dict[str, Any]:
     """Canonicalize + ValidationGate a formula WITHOUT persisting, evaluating,
     or backtesting (spec §5.3).
 
+    When ``available_fields`` is provided by a runtime surface, capability and
+    current-data availability are checked separately before the formula can be
+    confirmed.
+
     Returns a CANONICAL fingerprint of the (canonical-formula, sorted-filters,
     validated-horizon) spec -- via the operator_registry's own
     ``canonical_formula_fingerprint`` helper (F3), so alias / whitespace / case
@@ -2348,7 +2356,7 @@ def pre_validate_formula(
             horizon_days=horizon,
             universe_filters=filters,
         )
-        result = validate_factor_spec(spec)
+        result = validate_factor_spec(spec, available_fields=available_fields)
     except Exception:
         # A malformed formula (parse error) is an honest "blocked", never a
         # 500 and never an execution. The message is generic and leak-free

src/quant_forge/apps/web/routing.py

@@ -81,7 +81,8 @@
 from quant_forge.specs.narration import NarrationNode
 from quant_forge.apps.web.memory_review import memory_review_payload, review_promoted, review_rule
 from quant_forge.config import QuantForgeConfig
-from quant_forge.mcp.read_models import list_available_fields, list_available_operators
+from quant_forge.data.local import LocalPanelDataProvider
+from quant_forge.mcp.read_models import list_available_operators, list_runtime_available_fields
 from quant_forge.research_loop.config import ResearchLoopConfig, load_research_loop_config
 from quant_forge.research_loop.memory import ResearchMemoryStore
 from quant_forge.research_loop.scheduler import (
@@ -388,7 +389,12 @@ def do_GET(self) -> None:
                     self._json({"ok": True})
                 elif path == "/catalog":
                     self._require_control_token()
-                    self._json({"fields": list_available_fields(), "operators": list_available_operators()})
+                    self._json(
+                        {
+                            "fields": list_runtime_available_fields(config.paths.data_root),
+                            "operators": list_available_operators(),
+                        }
+                    )
                 elif path == "/api/research/history":
                     self._require_control_token()
                     # ValueError (limit validation) is reflected for the new
@@ -786,6 +792,9 @@ def do_POST(self) -> None:
                             name=payload.get("name", ""),
                             horizon_days=payload.get("horizon_days", 5),
                             universe_filters=_optional_universe_filters(payload.get("universe_filters")),
+                            available_fields=LocalPanelDataProvider(
+                                config.paths.data_root
+                            ).available_field_names(),
                         )
                     )
                     return

src/quant_forge/apps/web/tools.py

@@ -697,7 +697,14 @@ def _validate_draft_formula(self, args: dict[str, Any]) -> tuple[dict[str, Any],
         except ValueError as exc:
             return {"status": "blocked", "blocking_reasons": [str(exc)], "unresolved_operators": [],
                     "unresolved_fields": []}, []
-        result = validate_factor_spec(spec)
+        from quant_forge.data.local import LocalPanelDataProvider
+
+        result = validate_factor_spec(
+            spec,
+            available_fields=LocalPanelDataProvider(
+                self._config.paths.data_root
+            ).available_field_names(),
+        )
         payload = {
             "status": result.status,
             "unresolved_operators": list(result.unresolved_operators),

src/quant_forge/backtesting/service.py

@@ -27,6 +27,7 @@
     require_minimum_display_trading_days,
     simulation_profile_suffix,
 )
+from quant_forge.factor_engine.formula_parser import formula_input_fields
 from quant_forge.factor_library.catalog import FactorCatalog
 from quant_forge.utils import write_json
 
@@ -124,7 +125,9 @@ def run_factor_backtest(
     holding = holding_days or factor.horizon_days
     if holding < 1:
         raise ValueError("holding_days must be positive")
-    panel = LocalPanelDataProvider(data_root).load_panel()
+    panel = LocalPanelDataProvider(data_root).load_panel(
+        required_fields=formula_input_fields(factor.formula)
+    )
     working_panel = apply_test_period(panel, profile)
     require_minimum_display_trading_days(
         working_panel,

src/quant_forge/config.py

@@ -22,6 +22,12 @@ class PathSettings:
     factor_values_root: Path | None = None
     factor_values_overlay_root: Path | None = None
     factor_values_manifest_root: Path | None = None
+    # Fundamental data plane (optional; None -> price-volume-only panel).
+    # ``fundamentals_source_root`` points at a locally-materialized source layer
+    # (its real path lives only in an ignored local config); the CLI
+    # ``qf data build-fundamentals`` reads it and writes the PIT overlay as a
+    # ``fundamentals.parquet`` sibling of the panel, which the loader merges.
+    fundamentals_source_root: Path | None = None
     artifact_root: Path = Path("artifacts")
     output_root: Path = Path("outputs")
 
@@ -139,6 +145,7 @@ def resolve(self, workspace: Path | None = None) -> "QuantForgeConfig":
                 factor_values_root=_optional_under(root, self.paths.factor_values_root),
                 factor_values_overlay_root=_optional_under(root, self.paths.factor_values_overlay_root),
                 factor_values_manifest_root=_optional_under(root, self.paths.factor_values_manifest_root),
+                fundamentals_source_root=_optional_under(root, self.paths.fundamentals_source_root),
                 artifact_root=_under(root, self.paths.artifact_root),
                 output_root=_under(root, self.paths.output_root),
             ),
@@ -175,6 +182,7 @@ def load_config(config_path: Path | None = None, workspace: Path | None = None)
             factor_values_root=_optional_path_setting(raw, "factor_values_root"),
             factor_values_overlay_root=_optional_path_setting(raw, "factor_values_overlay_root"),
             factor_values_manifest_root=_optional_path_setting(raw, "factor_values_manifest_root"),
+            fundamentals_source_root=_optional_path_setting(raw, "fundamentals_source_root"),
             artifact_root=_path_setting(raw, "artifact_root", default="artifacts"),
             output_root=_path_setting(raw, "output_root", default="outputs"),
         ),
@@ -554,6 +562,7 @@ def _require_gitless_runtime_env_file_pattern(relative: Path, env_file: Path) ->
     "factor_values_root",
     "factor_values_overlay_root",
     "factor_values_manifest_root",
+    "fundamentals_source_root",
     "artifact_root",
     "output_root",
 )