From a8d791d14ddde5e41608b469472c9b428e65e56c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 5 Oct 2026 14:11:02 +0200 Subject: [PATCH 1/3] Migrate warehouse queries to the effect-orm query builder Imports move from @maple-dev/effect-clickhouse to @maple-dev/effect-orm (/clickhouse replaces the root and /types entries). Warehouse tables are external `CH.table` definitions, since their DDL stays in datasources.ts. OrgId columns are branded, so every tenant filter compares against the shared `orgIdParam` and warehouse query signatures take `OrgId`. Queries whose variant is chosen at runtime compile per branch, so each pairs with the params it actually needs. The dependency is a local `link:` until effect-orm is published. --- CLAUDE.md | 2 +- apps/api/package.json | 2 +- apps/api/scripts/bench-queries.ts | 5 +- apps/api/scripts/bench-suites/services.ts | 2 +- .../catalog.clickhouse.e2e.test.ts | 6 +- apps/api/src/routes/queries.ts | 99 ++++++------ apps/cli/src/server/service-map-rollup.ts | 8 +- apps/clickhouse-builder-docs/package.json | 2 +- .../scripts/sync-content.ts | 2 +- apps/local-ui/src/lib/constants.ts | 5 +- apps/local-ui/src/lib/query.ts | 2 +- bun.lock | 16 +- docs/query-benchmarking.md | 2 +- packages/backend/package.json | 2 +- .../services/ai-sessions/ai-session-reads.ts | 40 +++-- .../backend/src/services/auth/tinybird-jwt.ts | 2 +- ...rehouseQueryService.clickhouse.e2e.test.ts | 2 +- .../warehouse/WarehouseQueryService.test.ts | 2 +- .../warehouse/ai-tools.clickhouse.e2e.test.ts | 2 +- ...dex-materialization.clickhouse.e2e.test.ts | 2 +- .../series-cap.clickhouse.e2e.test.ts | 10 +- packages/domain/package.json | 2 +- packages/domain/src/raw-sql.ts | 2 +- .../domain/src/tinybird/ai-crawler-columns.ts | 6 +- .../domain/src/tinybird/gen-ai-columns.ts | 8 +- .../src/tinybird/semconv-renames.test.ts | 4 +- .../domain/src/tinybird/semconv-renames.ts | 6 +- .../domain/src/tinybird/span-display-name.ts | 6 +- .../query-engine-integrations/package.json | 2 +- .../src/ai/ai-sessions.test.ts | 15 +- .../src/ai/ai-sessions.ts | 72 +++++---- .../src/ai/ai-span-columns.test.ts | 6 +- .../src/ai/ai-span-columns.ts | 8 +- .../src/ai/ai-tools.test.ts | 7 +- .../src/ai/ai-tools.ts | 66 +++++--- .../src/benchmark/catalog.test.ts | 2 +- .../src/benchmark/index.ts | 3 +- .../cloudflare-infra-breakdowns.test.ts | 5 +- .../cloudflare/cloudflare-infra-breakdowns.ts | 14 +- .../cloudflare-infra-extended.test.ts | 5 +- .../cloudflare/cloudflare-infra-extended.ts | 18 +-- .../cloudflare-infra-filters.test.ts | 5 +- .../cloudflare/cloudflare-infra-filters.ts | 4 +- .../src/cloudflare/cloudflare-infra.test.ts | 10 +- .../src/cloudflare/cloudflare-infra.ts | 22 +-- .../src/cloudflare/cloudflare-map.test.ts | 10 +- .../src/cloudflare/cloudflare-map.ts | 10 +- .../src/cloudflare/cloudflare-usage.test.ts | 17 +- .../src/cloudflare/cloudflare-usage.ts | 10 +- .../src/planetscale/planetscale-infra.test.ts | 11 +- .../src/planetscale/planetscale-infra.ts | 10 +- .../src/planetscale/planetscale-map.test.ts | 10 +- .../src/planetscale/planetscale-map.ts | 18 +-- .../planetscale/planetscale-storage.test.ts | 5 +- .../src/product/billing-usage.test.ts | 5 +- .../src/product/billing-usage.ts | 12 +- .../src/product/internal.ts | 10 +- .../src/product/setup-audit.test.ts | 9 +- .../src/product/setup-audit.ts | 36 +++-- .../src/railway/railway-infra.ts | 10 +- packages/query-engine/package.json | 2 +- .../query-engine/src/benchmark/builders.ts | 12 +- .../src/benchmark/catalog.test.ts | 1 + .../query-engine/src/benchmark/catalog.ts | 4 +- packages/query-engine/src/benchmark/index.ts | 2 +- packages/query-engine/src/ch/ch.test.ts | 103 +++++++----- .../src/ch/endpoint-types.test-d.ts | 2 +- packages/query-engine/src/ch/index.ts | 5 +- packages/query-engine/src/ch/pipe-dispatch.ts | 48 ++++-- .../src/ch/queries/activity.test.ts | 2 +- .../query-engine/src/ch/queries/activity.ts | 2 +- .../src/ch/queries/alert-checks.test.ts | 7 +- .../src/ch/queries/alert-checks.ts | 42 ++--- .../src/ch/queries/anomaly.test.ts | 5 +- .../query-engine/src/ch/queries/anomaly.ts | 26 +-- .../src/ch/queries/attribute-keys.test.ts | 5 +- .../src/ch/queries/attribute-keys.ts | 24 +-- .../query-engine/src/ch/queries/audit-log.ts | 32 ++-- .../src/ch/queries/containers.test.ts | 7 +- .../query-engine/src/ch/queries/containers.ts | 20 +-- .../src/ch/queries/count-semantics.test.ts | 13 +- .../src/ch/queries/errors.test.ts | 5 +- .../query-engine/src/ch/queries/errors.ts | 125 +++++++------- .../query-engine/src/ch/queries/format.ts | 6 +- .../query-engine/src/ch/queries/infra.test.ts | 7 +- packages/query-engine/src/ch/queries/infra.ts | 42 ++--- .../src/ch/queries/liveness.test.ts | 7 +- .../query-engine/src/ch/queries/liveness.ts | 20 +-- .../query-engine/src/ch/queries/logs.test.ts | 7 +- packages/query-engine/src/ch/queries/logs.ts | 70 ++++---- .../src/ch/queries/metrics.test.ts | 5 +- .../query-engine/src/ch/queries/metrics.ts | 69 ++++---- .../ch/queries/product-events-explore.test.ts | 5 +- .../src/ch/queries/product-events-explore.ts | 12 +- .../ch/queries/product-events-paths.test.ts | 9 +- .../src/ch/queries/product-events-paths.ts | 23 +-- .../src/ch/queries/product-events.test.ts | 9 +- .../src/ch/queries/product-events.ts | 37 +++-- .../src/ch/queries/query-helpers.ts | 16 +- .../src/ch/queries/releases.test.ts | 5 +- .../query-engine/src/ch/queries/releases.ts | 12 +- .../src/ch/queries/rollup-splice.test.ts | 11 +- .../src/ch/queries/rollup-splice.ts | 6 +- .../src/ch/queries/series-cap.test.ts | 5 +- .../query-engine/src/ch/queries/series-cap.ts | 13 +- .../src/ch/queries/service-endpoints.test.ts | 5 +- .../src/ch/queries/service-infra.test.ts | 3 +- .../src/ch/queries/service-infra.ts | 19 +-- .../src/ch/queries/service-map-rollup.test.ts | 15 +- .../src/ch/queries/service-map-rollup.ts | 32 ++-- .../src/ch/queries/service-map.test.ts | 20 ++- .../src/ch/queries/service-map.ts | 56 +++---- .../src/ch/queries/service-operations.test.ts | 9 +- .../src/ch/queries/service-operations.ts | 28 ++-- .../src/ch/queries/services.test.ts | 7 +- .../query-engine/src/ch/queries/services.ts | 21 +-- .../src/ch/queries/session-events.test.ts | 5 +- .../src/ch/queries/session-events.ts | 69 ++++---- .../src/ch/queries/session-replays.test.ts | 7 +- .../src/ch/queries/session-replays.ts | 48 +++--- .../src/ch/queries/session-tags.ts | 8 +- .../src/ch/queries/signal-presence.test.ts | 5 +- .../src/ch/queries/signal-presence.ts | 16 +- .../src/ch/queries/top-operations.ts | 20 +-- .../src/ch/queries/traces.test.ts | 5 +- .../query-engine/src/ch/queries/traces.ts | 42 +++-- .../src/ch/queries/web-analytics-ai.test.ts | 9 +- .../src/ch/queries/web-analytics-ai.ts | 12 +- .../src/ch/queries/web-analytics.ts | 14 +- packages/query-engine/src/ch/raw-sql.ts | 7 +- packages/query-engine/src/ch/tables.ts | 22 +-- packages/query-engine/src/ch/user-agent.ts | 2 +- .../src/dashboard-variables/interpolate.ts | 2 +- .../src/execution/executor.test.ts | 2 +- .../query-engine/src/execution/executor.ts | 2 +- .../query-engine/src/execution/fingerprint.ts | 2 +- packages/query-engine/src/execution/ports.ts | 2 +- packages/query-engine/src/local.ts | 7 + .../src/observability/WarehouseExecutor.ts | 3 +- .../src/observability/error-detail.test.ts | 3 +- .../observability/explore-attributes.test.ts | 3 +- .../observability/find-slow-traces.test.ts | 5 +- .../src/observability/inspect-trace.test.ts | 3 +- .../src/observability/inspect-trace.ts | 2 +- .../src/observability/search-logs.test.ts | 2 +- .../src/observability/session-replays.test.ts | 5 +- .../src/observability/span-detail.test.ts | 3 +- .../src/profiles/query-profile.ts | 2 +- packages/query-engine/src/registry/logs.ts | 5 +- .../src/registry/product-events.ts | 23 +-- packages/query-engine/src/registry/queries.ts | 153 +++++++++--------- .../src/registry/query-definition.ts | 5 +- .../src/runtime/query-definition-runner.ts | 5 +- .../query-engine/src/runtime/query-engine.ts | 11 +- .../query-engine/src/runtime/raw-sql.test.ts | 7 +- packages/query-engine/src/runtime/raw-sql.ts | 5 +- packages/query-engine/src/traces-shared.ts | 4 +- 157 files changed, 1300 insertions(+), 1058 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7313878f3a..9d1f496969 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,7 +10,7 @@ backend, ClickHouse/Tinybird as the warehouse, PlanetScale Postgres for relation - `lib/*`: zero Maple knowledge, could ship as a standalone OSS library. A `lib/` package that imports `@maple/domain` must move to `packages/`. New packages default to `packages/`. -`@maple-dev/effect-clickhouse` lives in the separate effect-clickhouse repo; change it there. +`@maple-dev/effect-orm` lives in the separate effect-orm repo; change it there. ## Commands diff --git a/apps/api/package.json b/apps/api/package.json index b8615b141a..a57f5655ee 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -21,7 +21,7 @@ }, "dependencies": { "@effect/platform-bun": "catalog:effect", - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", "@maple-dev/effect-sdk": "workspace:*", "@maple/ai-model-catalog": "workspace:*", "@maple/auth": "workspace:*", diff --git a/apps/api/scripts/bench-queries.ts b/apps/api/scripts/bench-queries.ts index 8bd83123cc..cc55cb2a51 100644 --- a/apps/api/scripts/bench-queries.ts +++ b/apps/api/scripts/bench-queries.ts @@ -6,11 +6,12 @@ import { Clock, Config, Console, Context, Effect, FileSystem, Layer, Option, Red import { Command, Flag } from "effect/cli" import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http" import { BunRuntime, BunServices } from "@effect/platform-bun" +import { OrgId } from "@maple/domain" import { CH } from "@maple/query-engine" import * as Integrations from "@maple/query-engine-integrations" import * as Bench from "@maple/query-engine/benchmark" import { collectWarehouseQueryCatalog } from "./query-bench/catalog" -import { runCli } from "@maple-dev/effect-clickhouse/benchmark/cli" +import { runCli } from "@maple-dev/effect-orm/benchmark/cli" // Errors @@ -310,7 +311,7 @@ const fetchHandler = Effect.fn("bench.fetch")(function* (config: FetchConfig) { const orgId = yield* Option.match(config.org, { onNone: () => tinybird.internalOrgId, onSome: (o) => Effect.succeed(o), - }) + }).pipe(Effect.flatMap(Schema.decodeUnknownEffect(OrgId))) const host = yield* tinybird.host const compiled = yield* CH.compile( diff --git a/apps/api/scripts/bench-suites/services.ts b/apps/api/scripts/bench-suites/services.ts index 919ac8c955..91089e9677 100644 --- a/apps/api/scripts/bench-suites/services.ts +++ b/apps/api/scripts/bench-suites/services.ts @@ -1,6 +1,6 @@ /** Edit the inputs to a representative fixed window in a populated snapshot. */ import * as CH from "@maple/query-engine/ch" -import * as Bench from "@maple-dev/effect-clickhouse/benchmark" +import * as Bench from "@maple-dev/effect-orm/benchmark" export default Bench.defineSuite({ name: "services-workload", diff --git a/apps/api/scripts/query-bench/catalog.clickhouse.e2e.test.ts b/apps/api/scripts/query-bench/catalog.clickhouse.e2e.test.ts index 54ad9fe057..c8b9ce92c9 100644 --- a/apps/api/scripts/query-bench/catalog.clickhouse.e2e.test.ts +++ b/apps/api/scripts/query-bench/catalog.clickhouse.e2e.test.ts @@ -21,7 +21,7 @@ import { mkdir, readFile, writeFile } from "node:fs/promises" import { fileURLToPath } from "node:url" import { join } from "node:path" import * as CH from "@maple/query-engine/ch" -import type { CompiledQueryDecodeError } from "@maple-dev/effect-clickhouse" +import type { CompiledQueryDecodeError } from "@maple-dev/effect-orm/clickhouse" import { caseFromCompiled, RunOutput, Suite } from "@maple/query-engine/benchmark" import { collectWarehouseQueryCatalog } from "./catalog" import { normalizeSqlForClickHouseClient } from "@maple/query-engine/execution" @@ -78,7 +78,7 @@ const benchScript = fileURLToPath(new URL("../bench-queries.ts", import.meta.url const runCli = async (label: string, args: ReadonlyArray) => { const result = await new Promise<{ code: number; output: string }>((resolve, reject) => { const nodeBin = fileURLToPath( - new URL("bin.mjs", import.meta.resolve("@maple-dev/effect-clickhouse/benchmark/cli")), + new URL("bin.mjs", import.meta.resolve("@maple-dev/effect-orm/benchmark/cli")), ) const child = spawn( label === "baseline" ? "node" : "bun", @@ -340,7 +340,7 @@ describe.skipIf(!clickhouseE2eEnabled)("Query benchmark catalog analyzer sweep", candidateSuite, ` import * as CH from "@maple/query-engine/ch" -import * as Bench from "@maple-dev/effect-clickhouse/benchmark" +import * as Bench from "@maple-dev/effect-orm/benchmark" export default Bench.defineSuite({ name: "populated-maple-e2e", dataset: ${JSON.stringify(database)}, cases: [ Bench.query({ id: "traces/overview/5m", inputs: ${JSON.stringify({ ...inputs, ...options })}, compile: (values) => CH.compile(CH.tracesTimeseriesQuery({ ...values, overviewTiers: "minute" }), values), results: "unordered" }), Bench.query({ id: "services/facets", inputs: ${JSON.stringify(inputs)}, compile: (values) => CH.compileUnion(CH.servicesFacetsQuery(), values), results: "unordered" }), diff --git a/apps/api/src/routes/queries.ts b/apps/api/src/routes/queries.ts index 40b5bda9ef..d8a1836f52 100644 --- a/apps/api/src/routes/queries.ts +++ b/apps/api/src/routes/queries.ts @@ -36,6 +36,7 @@ import { workloadMetricSpec, } from "@maple/backend/queries/query-helpers" import { traceCacheTtlSeconds } from "@maple/backend/services/warehouse/trace-detail-cache" +import type { OrgId } from "@maple/domain" // App-side queries depend on integrations or API-only helpers, so moving them // into the core registry would invert dependencies. Entries own query inputs; @@ -45,7 +46,7 @@ const cloudflareInfraZoneCounters = defineQuery({ id: "cloudflareInfraZoneCounters", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZonesRequest, orgId: string) => { + compile: (payload: CloudflareInfraZonesRequest, orgId: OrgId) => { const params = { orgId: orgId, startTime: payload.startTime, @@ -63,7 +64,7 @@ const cloudflareInfraZoneLatency = defineQuery({ id: "cloudflareInfraZoneLatency", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZonesRequest, orgId: string) => { + compile: (payload: CloudflareInfraZonesRequest, orgId: OrgId) => { const params = { orgId: orgId, startTime: payload.startTime, @@ -81,7 +82,7 @@ const cloudflareInfraZoneFirewallTimeseries = defineQuery({ id: "cloudflareInfraZoneFirewallTimeseries", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZoneSecurityRequest, orgId: string) => { + compile: (payload: CloudflareInfraZoneSecurityRequest, orgId: OrgId) => { const params = { orgId: orgId, serviceName: payload.serviceName, @@ -100,7 +101,7 @@ const cloudflareInfraZoneFirewallTop = defineQuery({ id: "cloudflareInfraZoneFirewallTop", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZoneSecurityRequest, orgId: string) => { + compile: (payload: CloudflareInfraZoneSecurityRequest, orgId: OrgId) => { const params = { orgId: orgId, serviceName: payload.serviceName, @@ -116,7 +117,7 @@ const cloudflareInfraZoneDnsTimeseries = defineQuery({ id: "cloudflareInfraZoneDnsTimeseries", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZoneDnsRequest, orgId: string) => { + compile: (payload: CloudflareInfraZoneDnsRequest, orgId: OrgId) => { const params = { orgId: orgId, serviceName: payload.serviceName, @@ -135,7 +136,7 @@ const cloudflareInfraZoneDnsBreakdown = defineQuery({ id: "cloudflareInfraZoneDnsBreakdown", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZoneDnsRequest, orgId: string) => { + compile: (payload: CloudflareInfraZoneDnsRequest, orgId: OrgId) => { const params = { orgId: orgId, serviceName: payload.serviceName, @@ -151,7 +152,7 @@ const cloudflareInfraWorkerCounters = defineQuery({ id: "cloudflareInfraWorkerCounters", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraWorkersRequest, orgId: string) => { + compile: (payload: CloudflareInfraWorkersRequest, orgId: OrgId) => { const params = { orgId: orgId, startTime: payload.startTime, @@ -165,7 +166,7 @@ const cloudflareInfraWorkerLatency = defineQuery({ id: "cloudflareInfraWorkerLatency", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraWorkersRequest, orgId: string) => { + compile: (payload: CloudflareInfraWorkersRequest, orgId: OrgId) => { const params = { orgId: orgId, startTime: payload.startTime, @@ -179,7 +180,7 @@ const cloudflareInfraQueueGauges = defineQuery({ id: "cloudflareInfraQueueGauges", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraPlatformResourcesRequest, orgId: string) => { + compile: (payload: CloudflareInfraPlatformResourcesRequest, orgId: OrgId) => { const params = { orgId: orgId, startTime: payload.startTime, @@ -193,7 +194,7 @@ const cloudflareInfraDurableObjects = defineQuery({ id: "cloudflareInfraDurableObjects", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraPlatformResourcesRequest, orgId: string) => { + compile: (payload: CloudflareInfraPlatformResourcesRequest, orgId: OrgId) => { const params = { orgId: orgId, startTime: payload.startTime, @@ -207,7 +208,7 @@ const cloudflareServiceCounters = defineQuery({ id: "cloudflareServiceCounters", profile: "aggregation", cache: 15, - compile: (payload: ServiceCloudflareStatsRequest, orgId: string) => { + compile: (payload: ServiceCloudflareStatsRequest, orgId: OrgId) => { const params = { orgId: orgId, startTime: payload.startTime, @@ -225,7 +226,7 @@ const cloudflareServiceLatency = defineQuery({ id: "cloudflareServiceLatency", profile: "aggregation", cache: 15, - compile: (payload: ServiceCloudflareStatsRequest, orgId: string) => { + compile: (payload: ServiceCloudflareStatsRequest, orgId: OrgId) => { const params = { orgId: orgId, startTime: payload.startTime, @@ -243,7 +244,7 @@ const planetscaleInfraTimeseries = defineQuery({ id: "planetscaleInfraTimeseries", profile: "aggregation", cache: 15, - compile: (payload: PlanetScaleInfraTimeseriesRequest, orgId: string) => { + compile: (payload: PlanetScaleInfraTimeseriesRequest, orgId: OrgId) => { const base = { orgId: orgId, startTime: payload.startTime, @@ -264,7 +265,7 @@ const railwayInfraServices = defineQuery({ id: "railwayInfraServices", profile: "aggregation", cache: 15, - compile: (payload: RailwayInfraServicesRequest, orgId: string) => + compile: (payload: RailwayInfraServicesRequest, orgId: OrgId) => CH.compile(Integrations.railwayServicesSQL(), { orgId, startTime: payload.startTime, @@ -276,7 +277,7 @@ const railwayInfraServiceTimeseries = defineQuery({ id: "railwayInfraServiceTimeseries", profile: "aggregation", cache: 15, - compile: (payload: RailwayInfraServiceTimeseriesRequest, orgId: string) => + compile: (payload: RailwayInfraServiceTimeseriesRequest, orgId: OrgId) => CH.compile(Integrations.railwayServiceTimeseriesSQL(), { orgId, startTime: payload.startTime, @@ -287,7 +288,7 @@ const railwayInfraServiceTimeseries = defineQuery({ }), }) -const zoneDetailParams = (payload: CloudflareInfraZoneDetailRequest, orgId: string) => ({ +const zoneDetailParams = (payload: CloudflareInfraZoneDetailRequest, orgId: OrgId) => ({ orgId, serviceName: payload.serviceName, startTime: payload.startTime, @@ -299,7 +300,7 @@ const cloudflareInfraZoneDetailStatus = defineQuery({ id: "cloudflareInfraZoneDetailStatus", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZoneDetailRequest, orgId: string) => + compile: (payload: CloudflareInfraZoneDetailRequest, orgId: OrgId) => CH.compile( Integrations.cloudflareZoneStatusTimeseriesSQL(toCloudflareFilters(payload)), zoneDetailParams(payload, orgId), @@ -310,7 +311,7 @@ const cloudflareInfraZoneDetailCache = defineQuery({ id: "cloudflareInfraZoneDetailCache", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZoneDetailRequest, orgId: string) => + compile: (payload: CloudflareInfraZoneDetailRequest, orgId: OrgId) => CH.compile( Integrations.cloudflareZoneCacheTimeseriesSQL(toCloudflareFilters(payload)), zoneDetailParams(payload, orgId), @@ -322,7 +323,7 @@ const cloudflareInfraZoneDetailLatency = defineQuery({ id: "cloudflareInfraZoneDetailLatency", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZoneDetailRequest, orgId: string) => + compile: (payload: CloudflareInfraZoneDetailRequest, orgId: OrgId) => CH.compile(Integrations.cloudflareZoneLatencyTimeseriesSQL(), zoneDetailParams(payload, orgId)), }) @@ -330,21 +331,20 @@ const cloudflareInfraZoneDetailLatency = defineQuery({ // whether a database was requested. The branch lives inside `compile` so the id, // profile and row schema stay one decision per sub-query rather than two. -const planetscaleStatsParams = (payload: ServicePlanetScaleStatsRequest, orgId: string) => ({ +const planetscaleStatsParams = (payload: ServicePlanetScaleStatsRequest, orgId: OrgId) => ({ orgId, startTime: payload.startTime, endTime: payload.endTime, - ...(payload.database !== undefined ? { database: payload.database } : undefined), }) const planetscaleServiceGauges = defineQuery({ id: "planetscaleServiceGauges", profile: "aggregation", cache: 15, - compile: (payload: ServicePlanetScaleStatsRequest, orgId: string) => { + compile: (payload: ServicePlanetScaleStatsRequest, orgId: OrgId) => { const params = planetscaleStatsParams(payload, orgId) return payload.database !== undefined - ? CH.compile(Integrations.planetscaleBranchGaugesSQL(), params) + ? CH.compile(Integrations.planetscaleBranchGaugesSQL(), { ...params, database: payload.database }) : CH.compile(Integrations.planetscaleGaugesSQL(), params) }, }) @@ -353,10 +353,13 @@ const planetscaleServiceConnections = defineQuery({ id: "planetscaleServiceConnections", profile: "aggregation", cache: 15, - compile: (payload: ServicePlanetScaleStatsRequest, orgId: string) => { + compile: (payload: ServicePlanetScaleStatsRequest, orgId: OrgId) => { const params = planetscaleStatsParams(payload, orgId) return payload.database !== undefined - ? CH.compile(Integrations.planetscaleBranchConnectionsSQL(), params) + ? CH.compile(Integrations.planetscaleBranchConnectionsSQL(), { + ...params, + database: payload.database, + }) : CH.compile(Integrations.planetscaleConnectionsSQL(), params) }, }) @@ -365,12 +368,16 @@ const planetscaleServiceStorage = defineQuery({ id: "planetscaleServiceStorage", profile: "aggregation", cache: 15, - compile: (payload: ServicePlanetScaleStatsRequest, orgId: string) => { + compile: (payload: ServicePlanetScaleStatsRequest, orgId: OrgId) => { const params = planetscaleStatsParams(payload, orgId) return payload.database !== undefined - ? CH.compile(Integrations.planetscaleBranchStorageSQL(), params, { - rowSchema: Integrations.planetscaleBranchStorageRowSchema, - }) + ? CH.compile( + Integrations.planetscaleBranchStorageSQL(), + { ...params, database: payload.database }, + { + rowSchema: Integrations.planetscaleBranchStorageRowSchema, + }, + ) : CH.compile(Integrations.planetscaleStorageSQL(), params, { rowSchema: Integrations.planetscaleStorageRowSchema, }) @@ -387,7 +394,7 @@ const cloudflareInfraZoneFacets = defineQuery({ profile: "discovery", settings: { maxThreads: 4 }, cache: 60, - compile: (payload: CloudflareInfraZoneFacetsRequest, orgId: string) => + compile: (payload: CloudflareInfraZoneFacetsRequest, orgId: OrgId) => CH.compileUnion(Integrations.cloudflareZoneFacetsQuery(toCloudflareFilters(payload)), { orgId, serviceName: payload.serviceName, @@ -406,7 +413,7 @@ const hostInfraNetworkTimeseries = defineQuery({ id: "hostInfraTimeseries", profile: "aggregation", cache: 15, - compile: (payload: HostInfraTimeseriesRequest, orgId: string) => + compile: (payload: HostInfraTimeseriesRequest, orgId: OrgId) => CH.compile(CH.hostNetworkTimeseriesQuery({ hostName: payload.hostName }), { orgId, startTime: payload.startTime, @@ -419,7 +426,7 @@ const hostInfraGaugeTimeseries = defineQuery({ id: "hostInfraTimeseries", profile: "aggregation", cache: 15, - compile: (payload: HostInfraTimeseriesRequest, orgId: string) => { + compile: (payload: HostInfraTimeseriesRequest, orgId: OrgId) => { const spec = hostMetricSpec(payload.metric) return CH.compile( CH.hostGaugeTimeseriesQuery({ @@ -445,7 +452,7 @@ const containerInfraSumTimeseries = defineQuery({ id: "containerInfraTimeseries", profile: "aggregation", cache: 15, - compile: (payload: ContainerInfraTimeseriesRequest, orgId: string) => { + compile: (payload: ContainerInfraTimeseriesRequest, orgId: OrgId) => { const spec = containerMetricSpec(payload.metric) return CH.compile( CH.containerSumTimeseriesQuery({ @@ -470,7 +477,7 @@ const containerInfraGaugeTimeseries = defineQuery({ id: "containerInfraTimeseries", profile: "aggregation", cache: 15, - compile: (payload: ContainerInfraTimeseriesRequest, orgId: string) => { + compile: (payload: ContainerInfraTimeseriesRequest, orgId: OrgId) => { const spec = containerMetricSpec(payload.metric) return CH.compile( CH.containerGaugeTimeseriesQuery({ @@ -489,7 +496,7 @@ const containerInfraGaugeTimeseries = defineQuery({ }, }) -const zoneBreakdownParams = (payload: CloudflareInfraZoneBreakdownRequest, orgId: string) => ({ +const zoneBreakdownParams = (payload: CloudflareInfraZoneBreakdownRequest, orgId: OrgId) => ({ orgId, serviceName: payload.serviceName, startTime: payload.startTime, @@ -500,7 +507,7 @@ const cloudflareInfraZoneBreakdownTotals = defineQuery({ id: "cloudflareInfraZoneBreakdownTotals", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZoneBreakdownRequest, orgId: string) => { + compile: (payload: CloudflareInfraZoneBreakdownRequest, orgId: OrgId) => { return CH.compile( Integrations.cloudflareZoneBreakdownTotalsSQL( payload.dimension, @@ -521,7 +528,7 @@ const cloudflareInfraZoneBreakdownCoverage = defineQuery({ id: "cloudflareInfraZoneBreakdownCoverage", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZoneBreakdownRequest, orgId: string) => + compile: (payload: CloudflareInfraZoneBreakdownRequest, orgId: OrgId) => CH.compile( Integrations.cloudflareZoneBreakdownCoverageSQL(payload.dimension), zoneBreakdownParams(payload, orgId), @@ -532,7 +539,7 @@ const cloudflareInfraZoneBreakdownZoneTotal = defineQuery({ id: "cloudflareInfraZoneBreakdownZoneTotal", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZoneBreakdownRequest, orgId: string) => + compile: (payload: CloudflareInfraZoneBreakdownRequest, orgId: OrgId) => CH.compile( Integrations.cloudflareZoneCountersSQL(toCloudflareFilters(payload)), zoneBreakdownParams(payload, orgId), @@ -556,7 +563,7 @@ const cloudflareInfraZoneBreakdownTimeseries = defineQuery({ cache: 15, compile: ( payload: CloudflareInfraZoneBreakdownRequest & { readonly topKeys: ReadonlyArray }, - orgId: string, + orgId: OrgId, ) => CH.compile( Integrations.cloudflareZoneBreakdownTimeseriesSQL( @@ -580,7 +587,7 @@ export const Queries = { id: "getLog", profile: "list", cache: 15, - compile: (payload: GetLogRequest, orgId: string) => { + compile: (payload: GetLogRequest, orgId: OrgId) => { const { startTime, endTime } = partitionWindowAround(payload.timestamp) return CH.compile( CH.getLogByKeyQuery({ @@ -602,7 +609,7 @@ export const Queries = { id: "spanDetail", profile: "discovery", cache: (payload: SpanDetailRequest, nowMs: number) => traceCacheTtlSeconds(payload.endTime, nowMs), - compile: (payload: SpanDetailRequest, orgId: string) => { + compile: (payload: SpanDetailRequest, orgId: OrgId) => { // Without both bounds there is no window to narrow to, and passing a // half-open range would widen the scan rather than prune it. const narrowByTime = payload.startTime != null && payload.endTime != null @@ -621,7 +628,7 @@ export const Queries = { id: "fleetUtilizationTimeseries", profile: "aggregation", cache: 15, - compile: (payload: FleetUtilizationTimeseriesRequest, orgId: string) => + compile: (payload: FleetUtilizationTimeseriesRequest, orgId: OrgId) => CH.compile(CH.fleetUtilizationTimeseriesQuery(), { orgId, startTime: payload.startTime, @@ -634,7 +641,7 @@ export const Queries = { id: "podInfraTimeseries", profile: "aggregation", cache: 15, - compile: (payload: PodInfraTimeseriesRequest, orgId: string) => + compile: (payload: PodInfraTimeseriesRequest, orgId: OrgId) => CH.compile( CH.podGaugeTimeseriesQuery({ podName: payload.podName, @@ -654,7 +661,7 @@ export const Queries = { id: "nodeInfraTimeseries", profile: "aggregation", cache: 15, - compile: (payload: NodeInfraTimeseriesRequest, orgId: string) => + compile: (payload: NodeInfraTimeseriesRequest, orgId: OrgId) => CH.compile( CH.nodeGaugeTimeseriesQuery({ nodeName: payload.nodeName, @@ -673,7 +680,7 @@ export const Queries = { id: "workloadInfraTimeseries", profile: "aggregation", cache: 15, - compile: (payload: WorkloadInfraTimeseriesRequest, orgId: string) => + compile: (payload: WorkloadInfraTimeseriesRequest, orgId: OrgId) => CH.compile( CH.workloadGaugeTimeseriesQuery({ kind: payload.kind, @@ -695,7 +702,7 @@ export const Queries = { id: "cloudflareInfraZoneTimeseries", profile: "aggregation", cache: 15, - compile: (payload: CloudflareInfraZoneTimeseriesRequest, orgId: string) => + compile: (payload: CloudflareInfraZoneTimeseriesRequest, orgId: OrgId) => CH.compile(Integrations.cloudflareZoneTimeseriesSQL(toCloudflareFilters(payload)), { orgId, startTime: payload.startTime, diff --git a/apps/cli/src/server/service-map-rollup.ts b/apps/cli/src/server/service-map-rollup.ts index 258ad1fa00..64a73ae461 100644 --- a/apps/cli/src/server/service-map-rollup.ts +++ b/apps/cli/src/server/service-map-rollup.ts @@ -6,8 +6,8 @@ import * as CH from "@maple/query-engine/ch" import { Clock, Duration, Effect, Schema } from "effect" import type { Chdb } from "./chdb" import { decodeJsonEachRow } from "./chdb-rows" +import { LOCAL_ORG_ID } from "@maple/query-engine/local" -const ORG_ID = "local" const HOUR_MS = CH.SERVICE_MAP_ROLLUP_HOUR_MS /** Lets exporters flush spans buffered while the server was down before hours seal. */ @@ -112,7 +112,7 @@ const firstTraceHourMs = (db: LocalServiceMapRollupDeps["db"], sinceMs: number, const sql = yield* selectBody( CH.compileUnion( CH.signalPresenceQuery(), - CH.serviceMapRollupWindowParams(ORG_ID, sinceMs, nowMs), + CH.serviceMapRollupWindowParams(LOCAL_ORG_ID, sinceMs, nowMs), ), "signal presence", ) @@ -131,7 +131,7 @@ const sealedHours = (db: LocalServiceMapRollupDeps["db"], oldestHourMs: number, Effect.gen(function* () { const sql = yield* selectBody( CH.serviceMapEdgesExistingHoursSQL( - CH.serviceMapRollupWindowParams(ORG_ID, oldestHourMs, currentHourMs), + CH.serviceMapRollupWindowParams(LOCAL_ORG_ID, oldestHourMs, currentHourMs), ), "existing hours", ) @@ -142,7 +142,7 @@ const sealedHours = (db: LocalServiceMapRollupDeps["db"], oldestHourMs: number, /** The INSERTs that seal one hour from raw spans: resolutions, then edges (the seal). */ export const serviceMapRollupInserts = (hourMs: number) => Effect.gen(function* () { - const params = CH.serviceMapRollupHourParams(ORG_ID, hourMs) + const params = CH.serviceMapRollupHourParams(LOCAL_ORG_ID, hourMs) const resolutions = yield* selectBody(CH.serviceMapResolutionsRollupSQL(params), "resolutions") const edges = yield* selectBody(CH.serviceMapEdgesRollupSQL(params), "edges") return [ diff --git a/apps/clickhouse-builder-docs/package.json b/apps/clickhouse-builder-docs/package.json index e87f225d06..3de670cd5f 100644 --- a/apps/clickhouse-builder-docs/package.json +++ b/apps/clickhouse-builder-docs/package.json @@ -12,7 +12,7 @@ }, "dependencies": { "@base-ui/react": "^1.8.0", - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", "effect": "catalog:effect", "fumadocs-core": "^16.15.15", "fumadocs-mdx": "^15.4.5", diff --git a/apps/clickhouse-builder-docs/scripts/sync-content.ts b/apps/clickhouse-builder-docs/scripts/sync-content.ts index 5670ce54bd..a333c12e04 100644 --- a/apps/clickhouse-builder-docs/scripts/sync-content.ts +++ b/apps/clickhouse-builder-docs/scripts/sync-content.ts @@ -6,7 +6,7 @@ import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises" // Use a reviewed local docs directory for documentation-only releases. const source = process.env.EFFECT_CLICKHOUSE_DOCS_DIR ? pathToFileURL(resolve(process.env.EFFECT_CLICKHOUSE_DOCS_DIR) + "/") - : new URL("../docs/", import.meta.resolve("@maple-dev/effect-clickhouse")) + : new URL("../docs/", import.meta.resolve("@maple-dev/effect-orm/clickhouse")) const output = new URL("../content/", import.meta.url) const repository = "https://github.com/MapleTechLabs/effect-clickhouse/blob/v0.1.0/" const sections = [ diff --git a/apps/local-ui/src/lib/constants.ts b/apps/local-ui/src/lib/constants.ts index b4c88f0ec3..92d2830cba 100644 --- a/apps/local-ui/src/lib/constants.ts +++ b/apps/local-ui/src/lib/constants.ts @@ -1,7 +1,4 @@ -// Local mode pins a single synthetic tenant. The Rust ingest binary writes -// every decoded span/log/metric under this `OrgId`, and every `CH.compile(...)` -// call must pass the same constant so the WHERE `OrgId = 'local'` filter matches. -export const LOCAL_ORG_ID = "local" +export { LOCAL_ORG_ID } from "@maple/query-engine/local" // Default OTLP/HTTP + query port for `maple start`. export const DEFAULT_LOCAL_PORT = "4318" diff --git a/apps/local-ui/src/lib/query.ts b/apps/local-ui/src/lib/query.ts index d8d9738070..9928864c1a 100644 --- a/apps/local-ui/src/lib/query.ts +++ b/apps/local-ui/src/lib/query.ts @@ -43,7 +43,7 @@ export async function executeLocalCompiledFirstRow( } /** The org + window params nearly every local query compiles with. */ -export function localParams(bounds: TimeBounds): { orgId: string; startTime: string; endTime: string } { +export function localParams(bounds: TimeBounds) { return { orgId: LOCAL_ORG_ID, startTime: bounds.startTime, endTime: bounds.endTime } } diff --git a/bun.lock b/bun.lock index 8ca0f4fc4b..30c6577bb2 100644 --- a/bun.lock +++ b/bun.lock @@ -80,7 +80,7 @@ "name": "@maple/api", "dependencies": { "@effect/platform-bun": "catalog:effect", - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", "@maple-dev/effect-sdk": "workspace:*", "@maple/ai-model-catalog": "workspace:*", "@maple/auth": "workspace:*", @@ -155,7 +155,7 @@ "name": "@maple/clickhouse-builder-docs", "dependencies": { "@base-ui/react": "^1.8.0", - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", "effect": "catalog:effect", "fumadocs-core": "^16.15.15", "fumadocs-mdx": "^15.4.5", @@ -614,8 +614,8 @@ "@distilled.cloud/cloudflare": "1.0.0-rc.13", "@distilled.cloud/core": "1.0.0-rc.13", "@distilled.cloud/railway": "1.0.0-rc.13", - "@maple-dev/effect-clickhouse": "0.2.0", "@maple-dev/effect-clickhouse-http": "workspace:*", + "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", "@maple/alerting-core": "workspace:*", "@maple/auth": "workspace:*", "@maple/cache": "workspace:*", @@ -755,7 +755,7 @@ "name": "@maple/domain", "version": "0.0.0", "dependencies": { - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", "@maple/primitives": "workspace:*", "@maple/query-model": "workspace:*", "@maple/widgets": "workspace:*", @@ -850,7 +850,7 @@ "packages/query-engine": { "name": "@maple/query-engine", "dependencies": { - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", "@maple/cache": "workspace:*", "@maple/domain": "workspace:*", "@maple/query-model": "workspace:*", @@ -867,7 +867,7 @@ "packages/query-engine-integrations": { "name": "@maple/query-engine-integrations", "dependencies": { - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", "@maple/domain": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect", @@ -1551,10 +1551,10 @@ "@maple-dev/browser": ["@maple-dev/browser@workspace:packages/browser"], - "@maple-dev/effect-clickhouse": ["@maple-dev/effect-clickhouse@0.2.0", "", { "peerDependencies": { "effect": "^4.0.0" }, "bin": { "ch-bench": "dist/benchmark/bin.mjs" } }, "sha512-pFTekfGfuzJyiD0pQp8HSTNs1K10Z8K1i5Ozqexih9utHyWWan+tfAEsqWfCOCLiDdCP5Phun6r/bVZ683otvQ=="], - "@maple-dev/effect-clickhouse-http": ["@maple-dev/effect-clickhouse-http@workspace:lib/effect-clickhouse-http"], + "@maple-dev/effect-orm": ["@maple-dev/effect-orm@link:@maple-dev/effect-orm", { "bin": { "ch-bench": "./dist/benchmark/bin.mjs", "effect-orm": "./dist/kit/bin.mjs" } }], + "@maple-dev/effect-sdk": ["@maple-dev/effect-sdk@workspace:packages/effect-sdk"], "@maple-examples/alchemy": ["@maple-examples/alchemy@workspace:examples/alchemy-maple"], diff --git a/docs/query-benchmarking.md b/docs/query-benchmarking.md index 11c090643e..83d5559e80 100644 --- a/docs/query-benchmarking.md +++ b/docs/query-benchmarking.md @@ -4,7 +4,7 @@ Use `bun run bench:queries` from the repository root. The CLI compiles the real query catalog, runs fixed workloads against ClickHouse, saves evidence, and compares implementations by **case ID**, so changing SQL does not lose the baseline. -The reusable, driver-free API lives at `@maple-dev/effect-clickhouse/benchmark`. +The reusable, driver-free API lives at `@maple-dev/effect-orm/benchmark`. `@maple/query-engine/benchmark` composes it with Maple catalog fixtures. `apps/api/scripts/bench-queries.ts` owns Maple catalog export and trace mining; execution, HTTP, files and printing use the published `ch-bench` CLI. See the diff --git a/packages/backend/package.json b/packages/backend/package.json index 8fc88db8a2..f22f5a24df 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -23,7 +23,7 @@ "@distilled.cloud/cloudflare": "1.0.0-rc.13", "@distilled.cloud/core": "1.0.0-rc.13", "@distilled.cloud/railway": "1.0.0-rc.13", - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", "@maple-dev/effect-clickhouse-http": "workspace:*", "@maple/alerting-core": "workspace:*", "@maple/auth": "workspace:*", diff --git a/packages/backend/src/services/ai-sessions/ai-session-reads.ts b/packages/backend/src/services/ai-sessions/ai-session-reads.ts index dff7b8ecbd..6ad53326af 100644 --- a/packages/backend/src/services/ai-sessions/ai-session-reads.ts +++ b/packages/backend/src/services/ai-sessions/ai-session-reads.ts @@ -491,27 +491,33 @@ export const readAiSessionSummary = Effect.fn("aiSessions.summary")(function* ( // capped, and a session grouping into more turns than the cap // must still report exact totals — those come from the ungrouped // read, which no cap touches. - const params = + const base = { orgId: tenant.orgId, ...window } + const turnsSchema = { rowSchema: Integrations.aiSessionSummaryRowSchema } + const totalsSchema = { rowSchema: Integrations.aiSessionTotalsRowSchema } + // Compiled per branch: each query pairs with its own params. + const compiled = traceId === undefined - ? { orgId: tenant.orgId, sessionId: payload.sessionId, ...window } - : { orgId: tenant.orgId, traceId, ...window } - const turnsQuery = - traceId === undefined ? Integrations.aiSessionSummaryQuery() : Integrations.aiTraceSummaryQuery() - const totalsQuery = - traceId === undefined ? Integrations.aiSessionTotalsQuery() : Integrations.aiTraceTotalsQuery() + ? { + turns: CH.compile( + Integrations.aiSessionSummaryQuery(), + { ...base, sessionId: payload.sessionId }, + turnsSchema, + ), + totals: CH.compile( + Integrations.aiSessionTotalsQuery(), + { ...base, sessionId: payload.sessionId }, + totalsSchema, + ), + } + : { + turns: CH.compile(Integrations.aiTraceSummaryQuery(), { ...base, traceId }, turnsSchema), + totals: CH.compile(Integrations.aiTraceTotalsQuery(), { ...base, traceId }, totalsSchema), + } const kind = traceId === undefined ? "aiSession" : "aiTrace" const [rows, totals] = yield* Effect.all( [ - warehouse.compiledQuery( - tenant, - CH.compile(turnsQuery, params, { rowSchema: Integrations.aiSessionSummaryRowSchema }), - { context: `${kind}Summary` }, - ), - warehouse.compiledQuery( - tenant, - CH.compile(totalsQuery, params, { rowSchema: Integrations.aiSessionTotalsRowSchema }), - { context: `${kind}Totals` }, - ), + warehouse.compiledQuery(tenant, compiled.turns, { context: `${kind}Summary` }), + warehouse.compiledQuery(tenant, compiled.totals, { context: `${kind}Totals` }), ], { concurrency: 2 }, ) diff --git a/packages/backend/src/services/auth/tinybird-jwt.ts b/packages/backend/src/services/auth/tinybird-jwt.ts index baa39d5614..ed3dc34fa8 100644 --- a/packages/backend/src/services/auth/tinybird-jwt.ts +++ b/packages/backend/src/services/auth/tinybird-jwt.ts @@ -1,5 +1,5 @@ import { createHash, createHmac } from "node:crypto" -import { escapeClickHouseString } from "@maple-dev/effect-clickhouse/sql" +import { escapeClickHouseString } from "@maple-dev/effect-orm/sql" // Tinybird per-org read JWT minting. // diff --git a/packages/backend/src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts b/packages/backend/src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts index c89a78277f..cb9bea8981 100644 --- a/packages/backend/src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts +++ b/packages/backend/src/services/warehouse/WarehouseQueryService.clickhouse.e2e.test.ts @@ -4,7 +4,7 @@ import { FetchHttpClient } from "effect/http" import { clickHouseVersionAtLeast } from "@maple/domain/clickhouse" import { OrgId, RawSqlValidationError, UserId } from "@maple/domain/http" import { prepareRawSql } from "@maple/query-engine/runtime" -import { parseStatement } from "@maple-dev/effect-clickhouse/sql" +import { parseStatement } from "@maple-dev/effect-orm/sql" import { EdgeCacheService, MemoryCacheBackendLive } from "@maple/cache" import { OrgClickHouseSettingsService } from "@maple/backend/services/org/OrgClickHouseSettingsService" import { TinybirdOrgTokenService } from "@maple/backend/services/integrations/TinybirdOrgTokenService" diff --git a/packages/backend/src/services/warehouse/WarehouseQueryService.test.ts b/packages/backend/src/services/warehouse/WarehouseQueryService.test.ts index d666dc2c7b..23de7869fa 100644 --- a/packages/backend/src/services/warehouse/WarehouseQueryService.test.ts +++ b/packages/backend/src/services/warehouse/WarehouseQueryService.test.ts @@ -19,7 +19,7 @@ import { import { FetchHttpClient } from "effect/http" import { TestClock } from "effect/testing" import { rawCompiledQuery } from "@maple/query-engine/ch" -import { parseStatement, type ClickHouseStatement } from "@maple-dev/effect-clickhouse/sql" +import { parseStatement, type ClickHouseStatement } from "@maple-dev/effect-orm/sql" import { EdgeCacheService, MemoryCacheBackendLive } from "@maple/cache" import { makeWarehouseExecutor, diff --git a/packages/backend/src/services/warehouse/ai-tools.clickhouse.e2e.test.ts b/packages/backend/src/services/warehouse/ai-tools.clickhouse.e2e.test.ts index babe0b48dd..c23e35c688 100644 --- a/packages/backend/src/services/warehouse/ai-tools.clickhouse.e2e.test.ts +++ b/packages/backend/src/services/warehouse/ai-tools.clickhouse.e2e.test.ts @@ -29,7 +29,7 @@ import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" import { Array as Arr, Effect } from "effect" -import { compileUnionUnsafe, compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnionUnsafe, compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { MAPLE_AI_SESSION_ID_ATTR, MAPLE_AI_VENDOR_ID_ATTR } from "@maple/domain/gen-ai" import * as Integrations from "@maple/query-engine-integrations" import { normalizeSqlForClickHouseClient } from "@maple/query-engine/execution" diff --git a/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts b/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts index f328761f1e..ab4e875d66 100644 --- a/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts +++ b/packages/backend/src/services/warehouse/ai-trace-index-materialization.clickhouse.e2e.test.ts @@ -17,7 +17,7 @@ import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" import { Effect } from "effect" -import { compileUnionUnsafe, compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnionUnsafe, compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { MAPLE_AI_SESSION_ID_ATTR, MAPLE_AI_TRACE_SESSION_PREFIX, diff --git a/packages/backend/src/services/warehouse/series-cap.clickhouse.e2e.test.ts b/packages/backend/src/services/warehouse/series-cap.clickhouse.e2e.test.ts index e669b525c6..4d515a2510 100644 --- a/packages/backend/src/services/warehouse/series-cap.clickhouse.e2e.test.ts +++ b/packages/backend/src/services/warehouse/series-cap.clickhouse.e2e.test.ts @@ -1,15 +1,19 @@ // SAFETY-FILE: JSON is emitted by the isolated ClickHouse fixture and decoded by the real query schema. import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest" import { Schema } from "effect" -import { compileUnsafe, from, param, table } from "@maple-dev/effect-clickhouse" -import * as T from "@maple-dev/effect-clickhouse/types" +import { compileUnsafe, from, param, table } from "@maple-dev/effect-orm/clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" import { finalizeTimeseries } from "../../../../query-engine/src/ch/queries/series-cap" import { normalizeSqlForClickHouseClient } from "@maple/query-engine/execution" import { clickhouseE2eEnabled, clickhouseExec, uniqueDatabase } from "./clickhouse-e2e-support" const database = uniqueDatabase("maple_series_cap_e2e") const outputColumns = { bucket: T.string, groupName: T.string, value: T.nullable(T.float64), flag: T.bool } -const inputs = table("series_inputs", { OrgId: T.string, ...outputColumns }, { tenantColumn: "OrgId" }) +const inputs = table("series_inputs", { + external: true, + columns: { OrgId: T.string, ...outputColumns }, + tenantColumn: "OrgId", +}) const inner = from(inputs) .select(($) => ({ bucket: $.bucket, groupName: $.groupName, value: $.value, flag: $.flag })) .where(($) => [$.OrgId.eq(param.string("orgId"))]) diff --git a/packages/domain/package.json b/packages/domain/package.json index ee4ec46c65..288e9e507a 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -59,7 +59,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", "@maple/primitives": "workspace:*", "@maple/query-model": "workspace:*", "@maple/widgets": "workspace:*", diff --git a/packages/domain/src/raw-sql.ts b/packages/domain/src/raw-sql.ts index 918b941b3f..ce84b62c30 100644 --- a/packages/domain/src/raw-sql.ts +++ b/packages/domain/src/raw-sql.ts @@ -1,5 +1,5 @@ import { Schema } from "effect" -import { maskLiteralsAndComments, splitTerminalClauses } from "@maple-dev/effect-clickhouse/sql" +import { maskLiteralsAndComments, splitTerminalClauses } from "@maple-dev/effect-orm/sql" // The static half of raw-SQL validation: everything that can be decided from the // query text alone, without an org, a time window, or a granularity. diff --git a/packages/domain/src/tinybird/ai-crawler-columns.ts b/packages/domain/src/tinybird/ai-crawler-columns.ts index 39a69d8d68..6ceeb12b6f 100644 --- a/packages/domain/src/tinybird/ai-crawler-columns.ts +++ b/packages/domain/src/tinybird/ai-crawler-columns.ts @@ -2,9 +2,9 @@ // fetches, and the request facts it keeps from them. Compiled from the catalog // in `../ai-traffic.ts` so the write filter and the read-side labels share one list. -import type { Expr } from "@maple-dev/effect-clickhouse/expr" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { compile } from "@maple-dev/effect-clickhouse/sql" +import type { Expr } from "@maple-dev/effect-orm/expr" +import * as CH from "@maple-dev/effect-orm/expr" +import { compile } from "@maple-dev/effect-orm/sql" import { AI_CRAWLERS } from "../ai-traffic" interface MapColumnLike { diff --git a/packages/domain/src/tinybird/gen-ai-columns.ts b/packages/domain/src/tinybird/gen-ai-columns.ts index 8a47670f88..a79137a781 100644 --- a/packages/domain/src/tinybird/gen-ai-columns.ts +++ b/packages/domain/src/tinybird/gen-ai-columns.ts @@ -13,10 +13,10 @@ // The one computation left here is generic: the failure fingerprint, hashed // through the redaction chain `error_events` fingerprints messages with. -import type { Expr } from "@maple-dev/effect-clickhouse/expr" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { compile } from "@maple-dev/effect-clickhouse/sql" -import * as T from "@maple-dev/effect-clickhouse/types" +import type { Expr } from "@maple-dev/effect-orm/expr" +import * as CH from "@maple-dev/effect-orm/expr" +import { compile } from "@maple-dev/effect-orm/sql" +import * as T from "@maple-dev/effect-orm/clickhouse" import { applyRedactions, chRedactChain, MSG_TEXT_REDACTIONS } from "./fingerprint" import { MAPLE_AI_STAMP_ATTRS } from "../gen-ai" diff --git a/packages/domain/src/tinybird/semconv-renames.test.ts b/packages/domain/src/tinybird/semconv-renames.test.ts index d04a437d0f..87b299c8e1 100644 --- a/packages/domain/src/tinybird/semconv-renames.test.ts +++ b/packages/domain/src/tinybird/semconv-renames.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { compile } from "@maple-dev/effect-clickhouse/sql" +import * as CH from "@maple-dev/effect-orm/expr" +import { compile } from "@maple-dev/effect-orm/sql" import { DEPLOYMENT_ENV_SQL, MESSAGING_DESTINATION_SQL, diff --git a/packages/domain/src/tinybird/semconv-renames.ts b/packages/domain/src/tinybird/semconv-renames.ts index c8236804c2..c9b546753c 100644 --- a/packages/domain/src/tinybird/semconv-renames.ts +++ b/packages/domain/src/tinybird/semconv-renames.ts @@ -11,9 +11,9 @@ // Renames whose keys are only ever surfaced (never keyed on) live in the // recommendation dictionary in `../recommendations.ts` instead. -import type { Expr } from "@maple-dev/effect-clickhouse/expr" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { compile } from "@maple-dev/effect-clickhouse/sql" +import type { Expr } from "@maple-dev/effect-orm/expr" +import * as CH from "@maple-dev/effect-orm/expr" +import { compile } from "@maple-dev/effect-orm/sql" /** * The shape every caller shares: a query builder's `$.ResourceAttributes` / diff --git a/packages/domain/src/tinybird/span-display-name.ts b/packages/domain/src/tinybird/span-display-name.ts index 5163357e63..2e9119ea5c 100644 --- a/packages/domain/src/tinybird/span-display-name.ts +++ b/packages/domain/src/tinybird/span-display-name.ts @@ -1,6 +1,6 @@ -import type { Expr } from "@maple-dev/effect-clickhouse/expr" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { compile } from "@maple-dev/effect-clickhouse/sql" +import type { Expr } from "@maple-dev/effect-orm/expr" +import * as CH from "@maple-dev/effect-orm/expr" +import { compile } from "@maple-dev/effect-orm/sql" /** * Canonical operation-name expression shared by runtime queries and generated diff --git a/packages/query-engine-integrations/package.json b/packages/query-engine-integrations/package.json index 9a2465d1f0..92856e20c0 100644 --- a/packages/query-engine-integrations/package.json +++ b/packages/query-engine-integrations/package.json @@ -15,7 +15,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", "@maple/domain": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect" diff --git a/packages/query-engine-integrations/src/ai/ai-sessions.test.ts b/packages/query-engine-integrations/src/ai/ai-sessions.test.ts index 34ebdc9cbc..dfa020324e 100644 --- a/packages/query-engine-integrations/src/ai/ai-sessions.test.ts +++ b/packages/query-engine-integrations/src/ai/ai-sessions.test.ts @@ -5,7 +5,7 @@ import { compileUnionUnsafe, QueryBuilderDefect, type CompiledQuery, -} from "@maple-dev/effect-clickhouse" +} from "@maple-dev/effect-orm/clickhouse" import { aiSessionDetailsQuery, aiSessionDetailsSlices, @@ -27,9 +27,10 @@ import { aiTraceTotalsQuery, aiTraceWindowQuery, } from "./ai-sessions" +import { OrgId } from "@maple/domain" const params = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-08-18 00:00:00", endTime: "2026-08-19 23:59:59", } @@ -247,9 +248,9 @@ describe("aiSessionPageQuery", () => { expect(compiled.tenantScope).toBe("single-tenant") expect(orgPredicateCount(compiled.sql)).toBe(1) - expect(compileUnsafe(aiSessionPageQuery(), { ...params, orgId: "org'evil" }).sql).toContain( - "OrgId = 'org\\'evil'", - ) + expect( + compileUnsafe(aiSessionPageQuery(), { ...params, orgId: OrgId.make("org'evil") }).sql, + ).toContain("OrgId = 'org\\'evil'") }) it("bounds the read by the caller's window, unpadded", () => { @@ -733,7 +734,7 @@ describe("aiSessionDetailsQuery", () => { it("escapes an org id carrying a quote", () => { const { sql } = compileUnsafe(aiSessionDetailsQuery(listOpts), { ...listParams, - orgId: "org'evil", + orgId: OrgId.make("org'evil"), }) expect(sql).toContain("OrgId = 'org\\'evil'") @@ -1450,7 +1451,7 @@ describe("aiTraceSpansQuery — a turn's traces", () => { it("reads the named traces and nothing else, with no detection level", () => { const { sql } = compileUnsafe( aiTraceSpansQuery({ traceIds: [TRACE_ID, "0123456789abcdef0123456789abcdef"], scope: "app" }), - { orgId: "org_1", startTime: "2026-08-18 00:00:00", endTime: "2026-08-19 23:59:59" }, + { orgId: OrgId.make("org_1"), startTime: "2026-08-18 00:00:00", endTime: "2026-08-19 23:59:59" }, ) expect(sql).toContain(`TraceId IN ('${TRACE_ID}', '0123456789abcdef0123456789abcdef')`) diff --git a/packages/query-engine-integrations/src/ai/ai-sessions.ts b/packages/query-engine-integrations/src/ai/ai-sessions.ts index db00dbdafc..ea6835f297 100644 --- a/packages/query-engine-integrations/src/ai/ai-sessions.ts +++ b/packages/query-engine-integrations/src/ai/ai-sessions.ts @@ -121,9 +121,9 @@ // and inherits `org` scope from it. import { Schema } from "effect" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import * as T from "@maple-dev/effect-clickhouse/types" -import { compile } from "@maple-dev/effect-clickhouse/sql" +import * as CH from "@maple-dev/effect-orm/expr" +import * as T from "@maple-dev/effect-orm/clickhouse" +import { compile } from "@maple-dev/effect-orm/sql" import { compileFnCall, from, @@ -135,8 +135,8 @@ import { type CHUnionQuery, type ColumnAccessor, type CompiledQueryRowSchema, -} from "@maple-dev/effect-clickhouse" -import { AiTraceIndex, TraceDetailSpans, Traces } from "@maple/query-engine/ch/tables" +} from "@maple-dev/effect-orm/clickhouse" +import { AiTraceIndex, TraceDetailSpans, Traces, orgIdParam } from "@maple/query-engine/ch/tables" import { CHNumber } from "@maple/query-engine/ch/schema" import { AI_SESSION_SPANS_MAX_SPANS, @@ -303,7 +303,7 @@ const deepestFailureCount = (failedSpans: string, kind: "tool" | "turn"): CH.Exp * * The predicate is built from the lambda's key parameter, so it can use every * condition the DSL has (`in_`, `like`, `or`, …); values are not inspected. - * Lives here until `@maple-dev/effect-clickhouse` ships a `mapFilter`. + * Lives here until `@maple-dev/effect-orm` ships a `mapFilter`. */ const mapFilterKeys = ( mapExpr: CH.Expr>, @@ -437,6 +437,12 @@ export interface AiSessionDetailsOpts extends AiSessionFilterOpts { * (`startTime`/`endTime`) or the page's (`fanOutStart`/`fanOutEnd`). */ type IndexBounds = "window" | "page" +/** The param names of each pair of bounds, so a read's params name only its own pair. */ +const boundParams = { + window: { start: "startTime", end: "endTime" }, + page: { start: "fanOutStart", end: "fanOutEnd" }, +} as const + /** What the index cannot answer about a session: the facts of its traces' * other spans. Each replaces the page row's agent-only figure of the same name. */ export interface AiSessionDetailsOutput { @@ -535,9 +541,9 @@ const traceRankColumns = ($: IndexColumns) => ({ /** The agent traces of the window (or of a ranked page's bounds, and then of * its sessions alone), one row each, with whatever `select` measures — the * rows and the filters `indexTraces` documents. */ -const indexTracesOf = }>( +const indexTracesOf = }>( opts: AiSessionFilterOpts, - bounds: IndexBounds, + bounds: Bounds, sessionIds: readonly string[] | undefined, select: ($: IndexColumns) => Row, ) => { @@ -547,9 +553,9 @@ const indexTracesOf = }>( return from(AiTraceIndex) .select(select) .where(($) => [ - $.OrgId.eq(param.string("orgId")), - $.Timestamp.gte(param.dateTimeString(bounds === "window" ? "startTime" : "fanOutStart")), - $.Timestamp.lte(param.dateTimeString(bounds === "window" ? "endTime" : "fanOutEnd")), + $.OrgId.eq(orgIdParam), + $.Timestamp.gte(param.dateTimeString(boundParams[bounds].start)), + $.Timestamp.lte(param.dateTimeString(boundParams[bounds].end)), // A ranked page's rows are the window's: `fanOutEnd` is where the // page's last span ENDED, and a span that started past the window // but before that is one the ranking never counted. @@ -575,7 +581,11 @@ const indexTracesOf = }>( ]) } -const indexTraces = (opts: AiSessionFilterOpts, bounds: IndexBounds, sessionIds?: readonly string[]) => +const indexTraces = ( + opts: AiSessionFilterOpts, + bounds: Bounds, + sessionIds?: readonly string[], +) => indexTracesOf(opts, bounds, sessionIds, ($) => { // Ranks the trace's spans for the agent-name `argMin`: a span that // names an agent sorts at its own timestamp, one that does not sorts @@ -691,8 +701,12 @@ const sessionRankColumns = ($: { * netting reads — the session level of the page and of the distributions, * so a session measures the same in the row and in the histogram above it. */ -const indexSessions = (opts: AiSessionFilterOpts, sessionIds?: readonly string[]) => - fromQuery(indexTraces(opts, sessionIds ? "page" : "window", sessionIds), "index_traces") +const indexSessions = ( + opts: AiSessionFilterOpts, + bounds: Bounds, + sessionIds?: readonly string[], +) => + fromQuery(indexTraces(opts, bounds, sessionIds), "index_traces") .select(($) => ({ ...sessionRankColumns($), // Across traces the same ordering resolves the session's vendor: its @@ -880,9 +894,9 @@ export function aiSessionPageQuery(opts: AiSessionPageOpts = {}) { } // A page already ranked is read as it is: its sessions passed the filters // and the cut when `aiSessionRankQuery` made them. - const sessions = opts.sessionIds - ? indexSessions(opts, opts.sessionIds) - : indexSessions(opts).having(sessionFilters) + const bounds: IndexBounds = opts.sessionIds ? "page" : "window" + const indexed = indexSessions(opts, bounds, opts.sessionIds) + const sessions = opts.sessionIds ? indexed : indexed.having(sessionFilters) // A String order, and a correct one: the literal is fixed-width // `YYYY-MM-DD hh:mm:ss.nnnnnnnnn`, so it sorts as the instant does. const cutsOnSession = !opts.sessionIds && ranksOnSession(order) @@ -1016,7 +1030,7 @@ export function aiSessionDetailsQuery(opts: AiSessionDetailsOpts) { } }) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("spansStart")), $.Timestamp.lte(param.dateTimeString("spansEnd")), inSubquery($.TraceId, pageTraceIds), @@ -1207,7 +1221,7 @@ export function aiSessionFacetsQuery(): CHUnionQuery { .where(($) => [ // Every UNION ALL branch reads a table, so every branch carries the org // predicate itself — see this file's header. - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), ]) @@ -1295,7 +1309,7 @@ const DISTRIBUTION_BUCKET_FLOORS = { */ export function aiSessionDistributionsQuery() { // The page's netting and sums, less every column the histograms do not read. - const netted = fromQuery(indexSessions({}), "window_sessions").select(($) => ({ + const netted = fromQuery(indexSessions({}, "window"), "window_sessions").select(($) => ({ agentDurationMs: $.agentDurationMs, toolCalls: $.toolCalls, netted: nettedReportersExpr("reporters"), @@ -1371,7 +1385,7 @@ export function aiSessionWindowQuery() { spanCount: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), // Same presence guard as `aiSessionSpansQuery`, for the same reason: a // missing Map key reads back as `''`, so equality alone would resolve a // blank id to the bounds of every span in the org that lacks the key. @@ -1401,7 +1415,7 @@ export function aiTraceWindowQuery() { endTime: CH.toString_(CH.intervalAdd(CH.max_($.Timestamp), WINDOW_PAD_SECONDS)), spanCount: CH.count(), })) - .where(($) => [$.OrgId.eq(param.string("orgId")), $.TraceId.eq(param.string("traceId"))]) + .where(($) => [$.OrgId.eq(orgIdParam), $.TraceId.eq(param.string("traceId"))]) .format("JSON") } @@ -1527,7 +1541,7 @@ export function aiSessionSpansQuery(opts: AiSessionSpansOpts = {}) { from(TraceDetailSpans) .select(spanProjection) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), inSubquery($.TraceId, sessionTraceIds), @@ -1552,7 +1566,7 @@ const sessionTraceIdsSubquery = () => from(Traces) .select(($) => ({ TraceId: $.TraceId })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), // The presence guard is what stops an empty `sessionId` param from @@ -1600,7 +1614,7 @@ export function aiTraceSpansQuery(opts: AiTraceSpansOpts = {}) { return from(TraceDetailSpans) .select(spanProjection) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), opts.traceIds === undefined @@ -1890,7 +1904,7 @@ export function aiSessionSummaryQuery() { return from(TraceDetailSpans) .select(summaryProjection) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), inSubquery($.TraceId, sessionTraceIdsSubquery()), @@ -1906,7 +1920,7 @@ export function aiTraceSummaryQuery() { return from(TraceDetailSpans) .select(summaryProjection) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), $.TraceId.eq(param.string("traceId")), @@ -1922,7 +1936,7 @@ export function aiSessionTotalsQuery() { return from(TraceDetailSpans) .select(totalsProjection) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), inSubquery($.TraceId, sessionTraceIdsSubquery()), @@ -1935,7 +1949,7 @@ export function aiTraceTotalsQuery() { return from(TraceDetailSpans) .select(totalsProjection) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), $.TraceId.eq(param.string("traceId")), diff --git a/packages/query-engine-integrations/src/ai/ai-span-columns.test.ts b/packages/query-engine-integrations/src/ai/ai-span-columns.test.ts index 0a54311a8b..5b43ba7df1 100644 --- a/packages/query-engine-integrations/src/ai/ai-span-columns.test.ts +++ b/packages/query-engine-integrations/src/ai/ai-span-columns.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import * as T from "@maple-dev/effect-clickhouse/types" -import { compile } from "@maple-dev/effect-clickhouse/sql" +import * as CH from "@maple-dev/effect-orm/expr" +import * as T from "@maple-dev/effect-orm/clickhouse" +import { compile } from "@maple-dev/effect-orm/sql" import { nettedReportersExpr, sessionLlmCalls, diff --git a/packages/query-engine-integrations/src/ai/ai-span-columns.ts b/packages/query-engine-integrations/src/ai/ai-span-columns.ts index 392f9eb30f..8088ee1b58 100644 --- a/packages/query-engine-integrations/src/ai/ai-span-columns.ts +++ b/packages/query-engine-integrations/src/ai/ai-span-columns.ts @@ -29,10 +29,10 @@ // Reporters without an id are counted as they are — the page does not // guess. -import type { Expr } from "@maple-dev/effect-clickhouse/expr" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import * as T from "@maple-dev/effect-clickhouse/types" -import { compile } from "@maple-dev/effect-clickhouse/sql" +import type { Expr } from "@maple-dev/effect-orm/expr" +import * as CH from "@maple-dev/effect-orm/expr" +import * as T from "@maple-dev/effect-orm/clickhouse" +import { compile } from "@maple-dev/effect-orm/sql" import { AI_SESSION_SPANS_MAX_SPANS } from "@maple/domain/http" /** diff --git a/packages/query-engine-integrations/src/ai/ai-tools.test.ts b/packages/query-engine-integrations/src/ai/ai-tools.test.ts index 6ac4655f0e..dc3c68c7ca 100644 --- a/packages/query-engine-integrations/src/ai/ai-tools.test.ts +++ b/packages/query-engine-integrations/src/ai/ai-tools.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { Array as Arr, Effect } from "effect" -import { compileUnionUnsafe, compileUnsafe, type CompiledQuery } from "@maple-dev/effect-clickhouse" +import { compileUnionUnsafe, compileUnsafe, type CompiledQuery } from "@maple-dev/effect-orm/clickhouse" import { aiToolDescriptionQuery, aiToolDescriptionRowSchema, @@ -30,9 +30,10 @@ import { type AiToolErrorCallKey, } from "./ai-tools" import { AI_TOOLS_OTHER_SERIES_KEY } from "@maple/domain/http" +import { OrgId } from "@maple/domain" const params = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-08-18 00:00:00", endTime: "2026-08-19 23:59:59", bucketSeconds: 300, @@ -636,7 +637,7 @@ describe("aiToolErrorPayloadsQuery", () => { ] const compiled = compileUnsafe( aiToolErrorPayloadsQuery(calls), - { orgId: "org_1", ...aiToolErrorPayloadSlice(calls) }, + { orgId: OrgId.make("org_1"), ...aiToolErrorPayloadSlice(calls) }, { rowSchema: aiToolErrorPayloadsRowSchema }, ) diff --git a/packages/query-engine-integrations/src/ai/ai-tools.ts b/packages/query-engine-integrations/src/ai/ai-tools.ts index dfbc3b656e..3a53303a6a 100644 --- a/packages/query-engine-integrations/src/ai/ai-tools.ts +++ b/packages/query-engine-integrations/src/ai/ai-tools.ts @@ -65,14 +65,21 @@ // (structured-output pseudo-tools emitted by several SDKs) and are NOT filtered // out: they are calls, and dropping them would move every percentile. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import * as T from "@maple-dev/effect-clickhouse/types" -import { compile } from "@maple-dev/effect-clickhouse/sql" -import { from, fromQuery, inSubquery, param, unionAll, type CHUnionQuery } from "@maple-dev/effect-clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" +import * as T from "@maple-dev/effect-orm/clickhouse" +import { compile } from "@maple-dev/effect-orm/sql" +import { + from, + fromQuery, + inSubquery, + param, + unionAll, + type CHUnionQuery, +} from "@maple-dev/effect-orm/clickhouse" import { AI_TOOLS_BREAKDOWN_MAX, AI_TOOLS_OTHER_SERIES_KEY, type AiToolsPeriod } from "@maple/domain/http" import { Array as Arr, Schema } from "effect" -import type { CompiledQueryRowSchema } from "@maple-dev/effect-clickhouse" -import { AiTraceIndex, TraceDetailSpans } from "@maple/query-engine/ch/tables" +import { QueryBuilderDefect, type CompiledQueryRowSchema } from "@maple-dev/effect-orm/clickhouse" +import { AiTraceIndex, TraceDetailSpans, orgIdParam } from "@maple/query-engine/ch/tables" import { finiteOrZero, isoBucket } from "@maple/query-engine/ch/format" import { CHNumber } from "@maple/query-engine/ch/schema" import { aiToolCallPayload } from "./ai-integrations" @@ -129,10 +136,14 @@ const likeLiteral = (value: string): string => value.replace(/[\\%_]/g, (c) => ` */ export type AiToolsWindow = "current" | "previous" -const startParam = (window: AiToolsWindow) => - param.dateTimeString(window === "current" ? "startTime" : "prevStartTime") -const endParam = (window: AiToolsWindow) => - param.dateTimeString(window === "current" ? "endTime" : "prevEndTime") +/** The param names of each window, so a read's params name only its own pair. */ +const windowParams = { + current: { start: "startTime", end: "endTime" }, + previous: { start: "prevStartTime", end: "prevEndTime" }, +} as const + +const startParam = (window: W) => param.dateTimeString(windowParams[window].start) +const endParam = (window: W) => param.dateTimeString(windowParams[window].end) /** Series a bucketed read returns before the rest collapse into * {@link AI_TOOLS_OTHER_SERIES_KEY}. */ @@ -148,7 +159,7 @@ export const AI_TOOLS_BREAKDOWN_LIMIT = AI_TOOLS_BREAKDOWN_MAX * would name as its parent. Left-joined, so a tool span whose parent is a * workflow node (or whose parent was never exported) still produces a row. */ -const parentModels = (window: AiToolsWindow) => +const parentModels = (window: W) => from(AiTraceIndex) .select(($) => ({ TraceId: $.TraceId, @@ -158,7 +169,7 @@ const parentModels = (window: AiToolsWindow) => parentModel: CH.anyIf($.Model, $.Model.neq("")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(startParam(window)), $.Timestamp.lte(endParam(window)), // The join key is `(TraceId, ParentSpanId)`, so rows with no model are @@ -195,7 +206,7 @@ const parentModels = (window: AiToolsWindow) => * Sessions tile counts that list's population. A trace holding a tool call * always passes it, so no tool call is lost to the join. */ -const traceFacts = (window: AiToolsWindow) => +const traceFacts = (window: W) => from(AiTraceIndex) .select(($) => ({ TraceId: $.TraceId, @@ -211,7 +222,7 @@ const traceFacts = (window: AiToolsWindow) => ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(startParam(window)), $.Timestamp.lte(endParam(window)), ]) @@ -258,7 +269,7 @@ interface ToolCallColumns { /** The index rows of the window with their trace's facts beside them — the half * of {@link toolCalls} both its shapes share. */ -const toolCallIndex = (window: AiToolsWindow) => +const toolCallIndex = (window: W) => from(AiTraceIndex).innerJoinQuery(traceFacts(window), "trace", (call, trace) => call.TraceId.eq(trace.TraceId), ) @@ -290,9 +301,9 @@ const seriesParentModelNeeded = (opts: AiToolsFilterOpts): boolean => * derived column of the same name, and an aggregate over the shadowed name * becomes a cyclic alias rather than the aggregate the author meant. */ -const toolCalls = ( +const toolCalls = ( opts: AiToolsFilterOpts, - window: AiToolsWindow = "current", + window: W, withParentModel: boolean = parentModelNeeded(opts), ) => { const columns = ($: ToolCallSource, model: CH.Expr) => ({ @@ -312,7 +323,7 @@ const toolCalls = ( durationNs: $.Duration, }) const filters = ($: ToolCallSource, model: CH.Expr) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(startParam(window)), $.Timestamp.lte(endParam(window)), // The whole population, and the only predicate that is not a filter: @@ -517,7 +528,7 @@ export function aiToolsTotalsQuery( opts: AiToolsFilterOpts = {}, periods: ReadonlyArray = AI_TOOLS_TOTALS_PERIODS, ): CHUnionQuery { - const branch = (window: AiToolsWindow, period: AiToolsPeriod) => + const branch = (window: W, period: AiToolsPeriod) => fromQuery(toolCalls(opts, window), `tool_calls_${period}`).select(($) => ({ period: CH.lit(period), ...measures($), @@ -548,7 +559,11 @@ export function aiToolsTotalsQuery( const branches = AI_TOOLS_TOTALS_PERIODS.filter((period) => periods.includes(period)).map((period) => period === "window" ? allSessions : branch(period, period), ) - return unionAll(...branches).format("JSON") + const [first, ...rest] = branches + if (first === undefined) { + throw new QueryBuilderDefect({ message: "aiToolsTotalsQuery needs at least one period" }) + } + return unionAll(first, ...rest).format("JSON") } export interface AiToolsBreakdownsOutput { @@ -591,7 +606,7 @@ export const aiToolsBreakdownsRowSchema: CompiledQueryRowSchema ({ key: $.toolName, ...measures($), @@ -736,7 +751,10 @@ export const aiToolErrorsRowSchema: CompiledQueryRowSchema = export function aiToolErrorsQuery(opts: AiToolErrorsOpts = {}) { // Column names differ from the aliases below (`callErrorType`, `failureMessage`): // an aggregate aliased to its own input's name would read itself. - const numbered = fromQuery(toolCalls({ ...opts, failingOnly: undefined }), "tool_calls").select(($) => ({ + const numbered = fromQuery( + toolCalls({ ...opts, failingOnly: undefined }, "current"), + "tool_calls", + ).select(($) => ({ ts: $.ts, bucket: isoBucket($.ts), sessionKey: $.sessionKey, @@ -1015,7 +1033,7 @@ export function aiToolErrorPayloadsQuery(calls: Arr.NonEmptyReadonlyArray [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("sliceStart")), $.Timestamp.lte(param.dateTimeString("sliceEnd")), CH.inExprList( @@ -1123,7 +1141,7 @@ export function aiToolDescriptionQuery() { return from(AiTraceIndex) .select(($) => ({ description: CH.argMax($.ToolDescription, $.Timestamp) })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), $.IsToolCall.eq(1), diff --git a/packages/query-engine-integrations/src/benchmark/catalog.test.ts b/packages/query-engine-integrations/src/benchmark/catalog.test.ts index 581532c092..c4e5517fe9 100644 --- a/packages/query-engine-integrations/src/benchmark/catalog.test.ts +++ b/packages/query-engine-integrations/src/benchmark/catalog.test.ts @@ -27,7 +27,7 @@ describe("integration sql catalog", () => { }) // Structural, not a substring check on the SQL — see the tenant-scope work in - // @maple-dev/effect-clickhouse. No integration query reads across tenants. + // @maple-dev/effect-orm. No integration query reads across tenants. it("scopes every query to an org", () => { for (const entry of entries) { expect(entry.compiled.tenantScope, `${entry.id} tenant scope`).toBe("single-tenant") diff --git a/packages/query-engine-integrations/src/benchmark/index.ts b/packages/query-engine-integrations/src/benchmark/index.ts index 632dc771e1..d97df8f6f7 100644 --- a/packages/query-engine-integrations/src/benchmark/index.ts +++ b/packages/query-engine-integrations/src/benchmark/index.ts @@ -14,6 +14,7 @@ import { MAPLE_AI_TRACE_SESSION_PREFIX } from "@maple/domain/gen-ai" import * as CH from "../index" import { BenchmarkError, type CatalogEntry } from "@maple/query-engine/benchmark" import { fingerprintSql } from "@maple/query-engine/execution" +import { OrgId } from "@maple/domain" export interface IntegrationFixture { /** Source module basename, e.g. `"cloudflare-infra"`. */ @@ -25,7 +26,7 @@ export interface IntegrationFixture { readonly compile: () => CompiledQuery } -const ORG_ID = "org_sql_catalog" +const ORG_ID = OrgId.make("org_sql_catalog") const START_TIME = "2026-01-01 10:30:00" const END_TIME = "2026-01-03 14:15:00" diff --git a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-breakdowns.test.ts b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-breakdowns.test.ts index 099755b90c..12ca27818e 100644 --- a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-breakdowns.test.ts +++ b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-breakdowns.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" -import { compileUnsafe, compileUnionUnsafe, type CompiledQuery } from "@maple-dev/effect-clickhouse" +import { compileUnsafe, compileUnionUnsafe, type CompiledQuery } from "@maple-dev/effect-orm/clickhouse" import { CLOUDFLARE_BREAKDOWN_DIMENSIONS, CLOUDFLARE_BREAKDOWN_OTHER_KEY, @@ -10,9 +10,10 @@ import { cloudflareZoneBreakdownTotalsSQL, cloudflareZoneFacetsQuery, } from "./cloudflare-infra-breakdowns" +import { OrgId } from "@maple/domain" const zoneParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), serviceName: "cloudflare/example.com", startTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", diff --git a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-breakdowns.ts b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-breakdowns.ts index 12397b1d53..23f02f7df3 100644 --- a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-breakdowns.ts +++ b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-breakdowns.ts @@ -9,9 +9,9 @@ // metric emit literal 0 for those columns, so the response schema and the UI // table stay generic. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { from, param, unionAll, type CHUnionQuery } from "@maple-dev/effect-clickhouse" -import { MetricsSum } from "@maple/query-engine/ch/tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { from, param, unionAll, type CHUnionQuery } from "@maple-dev/effect-orm/clickhouse" +import { MetricsSum, orgIdParam } from "@maple/query-engine/ch/tables" import { ISO_Z_FORMAT, isoBucket } from "@maple/query-engine/ch/format" import { CF_ATTR, @@ -180,7 +180,7 @@ export function cloudflareZoneBreakdownTotalsSQL( return from(MetricsSum) .select(($) => ({ key: keyExpr($, dimension), ...measures($, spec) })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.MetricName.in_(...cloudflareBreakdownMetrics(dimension)), $.TimeUnix.gte(param.dateTimeString("startTime")), @@ -220,7 +220,7 @@ export function cloudflareZoneBreakdownTimeseriesSQL( requests: CH.sum($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.MetricName.eq(spec.requestsMetric), $.TimeUnix.gte(param.dateTimeString("startTime")), @@ -245,7 +245,7 @@ export function cloudflareZoneBreakdownCoverageSQL(dimension: CloudflareBreakdow attributedRequests: CH.sum($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.MetricName.eq(spec.requestsMetric), $.TimeUnix.gte(param.dateTimeString("startTime")), @@ -290,7 +290,7 @@ const makeCfFacet = ( facetType: CH.lit(facetType), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.MetricName.eq(metricName), $.TimeUnix.gte(param.dateTimeString("startTime")), diff --git a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-extended.test.ts b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-extended.test.ts index 3498dc8c0e..9ede1ce575 100644 --- a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-extended.test.ts +++ b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-extended.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" -import { compileUnsafe, type CompiledQuery } from "@maple-dev/effect-clickhouse" +import { compileUnsafe, type CompiledQuery } from "@maple-dev/effect-orm/clickhouse" import { cloudflareDurableObjectCountersSQL, cloudflareQueueGaugesSQL, @@ -9,9 +9,10 @@ import { cloudflareZoneFirewallTimeseriesSQL, cloudflareZoneFirewallTopSQL, } from "./cloudflare-infra-extended" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", } diff --git a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-extended.ts b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-extended.ts index 7064f87210..e5ab5c9067 100644 --- a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-extended.ts +++ b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-extended.ts @@ -11,9 +11,9 @@ // numeric output through CHNumber so BYO-ClickHouse string-encoded aggregates // decode identically to Tinybird numbers. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { from, param, type ColumnAccessor } from "@maple-dev/effect-clickhouse" -import { MetricsGauge, MetricsSum } from "@maple/query-engine/ch/tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { from, param, type ColumnAccessor } from "@maple-dev/effect-orm/clickhouse" +import { MetricsGauge, MetricsSum, orgIdParam } from "@maple/query-engine/ch/tables" import { avgWhere, isoBucket } from "@maple/query-engine/ch/format" import { CF_FILTERABLE, @@ -51,7 +51,7 @@ export function cloudflareZoneFirewallTimeseriesSQL(opts: CloudflareFilterOpts = events: CH.sum($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.MetricName.eq("cloudflare.firewall.events"), $.TimeUnix.gte(param.dateTimeString("startTime")), @@ -74,7 +74,7 @@ export function cloudflareZoneFirewallTopSQL(opts: CloudflareFilterOpts = {}) { events: CH.sum($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.MetricName.eq("cloudflare.firewall.events"), $.TimeUnix.gte(param.dateTimeString("startTime")), @@ -112,7 +112,7 @@ export function cloudflareZoneDnsTimeseriesSQL(opts: CloudflareFilterOpts = {}) queries: CH.sum($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.MetricName.eq("cloudflare.dns.queries"), $.TimeUnix.gte(param.dateTimeString("startTime")), @@ -133,7 +133,7 @@ export function cloudflareZoneDnsBreakdownSQL(opts: CloudflareFilterOpts = {}) { nxdomain: CH.sumIf($.Value, $.Attributes.get("dns.response_code").eq("NXDOMAIN")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.MetricName.eq("cloudflare.dns.queries"), $.TimeUnix.gte(param.dateTimeString("startTime")), @@ -188,7 +188,7 @@ export function cloudflareQueueGaugesSQL() { consumerConcurrency: avgWhere($.Value, $.MetricName.eq("cloudflare.queue.consumer.concurrency")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...QUEUE_GAUGE_METRIC_NAMES), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), @@ -208,7 +208,7 @@ export function cloudflareDurableObjectCountersSQL() { errors: CH.sumIf($.Value, $.MetricName.eq("cloudflare.durable_object.errors")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_("cloudflare.durable_object.requests", "cloudflare.durable_object.errors"), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), diff --git a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-filters.test.ts b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-filters.test.ts index fe28b9ce8b..776f22fd8c 100644 --- a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-filters.test.ts +++ b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-filters.test.ts @@ -1,12 +1,13 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { cloudflareZoneCacheTimeseriesSQL } from "./cloudflare-infra" import { cloudflareZoneBreakdownTotalsSQL } from "./cloudflare-infra-breakdowns" import { cloudflareZoneFirewallTopSQL } from "./cloudflare-infra-extended" import { CF_METRIC, cloudflareIgnoredFilters, cloudflareIgnoredFiltersFor } from "./cloudflare-infra-filters" +import { OrgId } from "@maple/domain" const zoneParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), serviceName: "cloudflare/example.com", startTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", diff --git a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-filters.ts b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-filters.ts index 11cf3db435..b1d2f1fdcf 100644 --- a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-filters.ts +++ b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra-filters.ts @@ -13,8 +13,8 @@ // dropped here, and the handler reports them back as `ignoredFilters` so the UI // can mark that panel zone-wide rather than silently lying about its scope. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { type ColumnAccessor } from "@maple-dev/effect-clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" +import { type ColumnAccessor } from "@maple-dev/effect-orm/clickhouse" import type { MetricsGauge, MetricsSum } from "@maple/query-engine/ch/tables" /** Filter key → the metric attribute it lives on. */ diff --git a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra.test.ts b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra.test.ts index a2def27a55..44546f0ad4 100644 --- a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra.test.ts +++ b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { cloudflareWorkerCountersSQL, cloudflareWorkerLatencySQL, @@ -10,9 +10,10 @@ import { cloudflareZoneStatusTimeseriesSQL, cloudflareZoneTimeseriesSQL, } from "./cloudflare-infra" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", } @@ -38,7 +39,10 @@ describe("cloudflareZoneCountersSQL", () => { }) it("escapes single quotes in orgId", () => { - const { sql } = compileUnsafe(cloudflareZoneCountersSQL(), { ...baseParams, orgId: "org'evil" }) + const { sql } = compileUnsafe(cloudflareZoneCountersSQL(), { + ...baseParams, + orgId: OrgId.make("org'evil"), + }) expect(sql).toContain("OrgId = 'org\\'evil'") }) }) diff --git a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra.ts b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra.ts index 376f6593ce..1f51c47093 100644 --- a/packages/query-engine-integrations/src/cloudflare/cloudflare-infra.ts +++ b/packages/query-engine-integrations/src/cloudflare/cloudflare-infra.ts @@ -12,9 +12,9 @@ // and pre-computed percentiles live in `metrics_gauge` (one row per // `quantile`). The API handlers merge the two by ServiceName. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { from, param, type ColumnAccessor } from "@maple-dev/effect-clickhouse" -import { MetricsGauge, MetricsSum } from "@maple/query-engine/ch/tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { from, param, type ColumnAccessor } from "@maple-dev/effect-orm/clickhouse" +import { MetricsGauge, MetricsSum, orgIdParam } from "@maple/query-engine/ch/tables" import { avgWhere, isoBucket } from "@maple/query-engine/ch/format" import { CF_FILTERABLE, @@ -123,7 +123,7 @@ export function cloudflareZoneCountersSQL(opts: CloudflareFilterOpts = {}) { ...zoneCounterColumns($), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...ZONE_COUNTER_METRIC_NAMES), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), @@ -160,7 +160,7 @@ export function cloudflareZoneLatencySQL() { originP99Ms: quantileAvg("cloudflare.http.origin.duration", "0.99")($), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...ZONE_GAUGE_METRIC_NAMES), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), @@ -179,7 +179,7 @@ export function cloudflareZoneTimeseriesSQL(opts: CloudflareFilterOpts = {}) { ...zoneCounterColumns($), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...ZONE_COUNTER_METRIC_NAMES), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), @@ -230,7 +230,7 @@ export function cloudflareZoneStatusTimeseriesSQL(opts: CloudflareFilterOpts = { requests: CH.sum($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.MetricName.eq("cloudflare.http.requests"), $.TimeUnix.gte(param.dateTimeString("startTime")), @@ -251,7 +251,7 @@ export function cloudflareZoneCacheTimeseriesSQL(opts: CloudflareFilterOpts = {} requests: CH.sum($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.MetricName.eq("cloudflare.http.requests"), $.TimeUnix.gte(param.dateTimeString("startTime")), @@ -284,7 +284,7 @@ export function cloudflareZoneLatencyTimeseriesSQL() { originP99Ms: quantileAvg("cloudflare.http.origin.duration", "0.99")($), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.MetricName.in_(...ZONE_GAUGE_METRIC_NAMES), $.TimeUnix.gte(param.dateTimeString("startTime")), @@ -325,7 +325,7 @@ export function cloudflareWorkerCountersSQL() { subrequests: CH.sumIf($.Value, $.MetricName.eq("cloudflare.worker.subrequests")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...WORKER_COUNTER_METRIC_NAMES), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), @@ -363,7 +363,7 @@ export function cloudflareWorkerLatencySQL() { ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...WORKER_GAUGE_METRIC_NAMES), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), diff --git a/packages/query-engine-integrations/src/cloudflare/cloudflare-map.test.ts b/packages/query-engine-integrations/src/cloudflare/cloudflare-map.test.ts index 0cf462d3e0..7deddb38ad 100644 --- a/packages/query-engine-integrations/src/cloudflare/cloudflare-map.test.ts +++ b/packages/query-engine-integrations/src/cloudflare/cloudflare-map.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { cloudflareServiceCountersSQL, cloudflareServiceLatencySQL } from "./cloudflare-map" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", } @@ -28,7 +29,10 @@ describe("cloudflareServiceCountersSQL", () => { }) it("escapes single quotes in orgId", () => { - const { sql } = compileUnsafe(cloudflareServiceCountersSQL(), { ...baseParams, orgId: "org'evil" }) + const { sql } = compileUnsafe(cloudflareServiceCountersSQL(), { + ...baseParams, + orgId: OrgId.make("org'evil"), + }) expect(sql).toContain("OrgId = 'org\\'evil'") }) }) diff --git a/packages/query-engine-integrations/src/cloudflare/cloudflare-map.ts b/packages/query-engine-integrations/src/cloudflare/cloudflare-map.ts index 5eae4826ca..b44eb7b3d2 100644 --- a/packages/query-engine-integrations/src/cloudflare/cloudflare-map.ts +++ b/packages/query-engine-integrations/src/cloudflare/cloudflare-map.ts @@ -14,9 +14,9 @@ // the pre-computed percentiles live in `metrics_gauge` (one row per quantile). // The caller (the `serviceCloudflareStats` handler) merges both by ServiceName. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { from, param } from "@maple-dev/effect-clickhouse" -import { MetricsGauge, MetricsSum } from "@maple/query-engine/ch/tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { from, param } from "@maple-dev/effect-orm/clickhouse" +import { MetricsGauge, MetricsSum, orgIdParam } from "@maple/query-engine/ch/tables" import { avgWhere } from "@maple/query-engine/ch/format" /** Counter metrics the poller emits for Workers (all in `metrics_sum`). */ @@ -50,7 +50,7 @@ export function cloudflareServiceCountersSQL() { errorCount: CH.sumIf($.Value, $.MetricName.eq("cloudflare.worker.errors")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...COUNTER_METRIC_NAMES), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), @@ -84,7 +84,7 @@ export function cloudflareServiceLatencySQL() { ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...GAUGE_METRIC_NAMES), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), diff --git a/packages/query-engine-integrations/src/cloudflare/cloudflare-usage.test.ts b/packages/query-engine-integrations/src/cloudflare/cloudflare-usage.test.ts index 61cfff880b..2c6417132e 100644 --- a/packages/query-engine-integrations/src/cloudflare/cloudflare-usage.test.ts +++ b/packages/query-engine-integrations/src/cloudflare/cloudflare-usage.test.ts @@ -1,10 +1,11 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { cloudflareUsageQuery, cloudflareUsageStatsQuery } from "./cloudflare-usage" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), bucketSeconds: 3600, startTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", @@ -30,13 +31,16 @@ describe("cloudflareUsageQuery", () => { }) it("escapes single quotes in orgId", () => { - const { sql } = compileUnsafe(cloudflareUsageQuery(), { ...baseParams, orgId: "org'evil" }) + const { sql } = compileUnsafe(cloudflareUsageQuery(), { + ...baseParams, + orgId: OrgId.make("org'evil"), + }) expect(sql).toContain("OrgId = 'org\\'evil'") }) }) const statsParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), prevStartTime: "2026-07-01 00:00:00.000", currentStartTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", @@ -68,7 +72,10 @@ describe("cloudflareUsageStatsQuery", () => { }) it("escapes single quotes in orgId", () => { - const { sql } = compileUnsafe(cloudflareUsageStatsQuery(), { ...statsParams, orgId: "org'evil" }) + const { sql } = compileUnsafe(cloudflareUsageStatsQuery(), { + ...statsParams, + orgId: OrgId.make("org'evil"), + }) expect(sql).toContain("OrgId = 'org\\'evil'") }) diff --git a/packages/query-engine-integrations/src/cloudflare/cloudflare-usage.ts b/packages/query-engine-integrations/src/cloudflare/cloudflare-usage.ts index eb5f3c1e0d..cb89a79d15 100644 --- a/packages/query-engine-integrations/src/cloudflare/cloudflare-usage.ts +++ b/packages/query-engine-integrations/src/cloudflare/cloudflare-usage.ts @@ -5,9 +5,9 @@ // The request-count metrics are 5-min delta sums written by the analytics // poller, so sum(Value) per bucket is the true request count. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { from, param } from "@maple-dev/effect-clickhouse" -import { MetricsSum } from "@maple/query-engine/ch/tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { from, param } from "@maple-dev/effect-orm/clickhouse" +import { MetricsSum, orgIdParam } from "@maple/query-engine/ch/tables" import { ISO_Z_FORMAT, isoBucket } from "@maple/query-engine/ch/format" /** @@ -65,7 +65,7 @@ export function cloudflareUsageStatsQuery() { ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...CLOUDFLARE_USAGE_METRIC_NAMES, "cloudflare.firewall.events"), $.TimeUnix.gte(param.dateTimeString("prevStartTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), @@ -83,7 +83,7 @@ export function cloudflareUsageQuery() { lastTimeUnix: CH.formatDateTime(CH.max_($.TimeUnix), ISO_Z_FORMAT), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...CLOUDFLARE_USAGE_METRIC_NAMES), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), diff --git a/packages/query-engine-integrations/src/planetscale/planetscale-infra.test.ts b/packages/query-engine-integrations/src/planetscale/planetscale-infra.test.ts index d20252f60d..fe4e4fb346 100644 --- a/packages/query-engine-integrations/src/planetscale/planetscale-infra.test.ts +++ b/packages/query-engine-integrations/src/planetscale/planetscale-infra.test.ts @@ -1,12 +1,13 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { planetscaleBranchInfraTimeseriesSQL, planetscaleInfraTimeseriesSQL } from "./planetscale-infra" +import { OrgId } from "@maple/domain" describe("planetscaleInfraTimeseriesSQL", () => { it("buckets per-timestamp totals for one database", () => { const { sql } = compileUnsafe(planetscaleInfraTimeseriesSQL(), { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", bucketSeconds: 300, @@ -28,7 +29,7 @@ describe("planetscaleInfraTimeseriesSQL", () => { it("decodes ClickHouse numeric strings through the row schema", () => { const compiled = compileUnsafe(planetscaleInfraTimeseriesSQL(), { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", bucketSeconds: 300, @@ -64,7 +65,7 @@ describe("planetscaleInfraTimeseriesSQL", () => { it("aggregates volume gauges without counting unsampled buckets as full", () => { const { sql } = compileUnsafe(planetscaleInfraTimeseriesSQL(), { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", bucketSeconds: 300, @@ -83,7 +84,7 @@ describe("planetscaleInfraTimeseriesSQL", () => { describe("planetscaleBranchInfraTimeseriesSQL", () => { it("scopes the same rollup to a single branch", () => { const { sql } = compileUnsafe(planetscaleBranchInfraTimeseriesSQL(), { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", bucketSeconds: 300, diff --git a/packages/query-engine-integrations/src/planetscale/planetscale-infra.ts b/packages/query-engine-integrations/src/planetscale/planetscale-infra.ts index 1239497385..45a05e7f20 100644 --- a/packages/query-engine-integrations/src/planetscale/planetscale-infra.ts +++ b/packages/query-engine-integrations/src/planetscale/planetscale-infra.ts @@ -12,9 +12,9 @@ // Scoping to a branch is what makes the chart mean anything. import { finiteOrZero } from "@maple/query-engine/ch/format" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { from, fromQuery, param } from "@maple-dev/effect-clickhouse" -import { MetricsGauge } from "@maple/query-engine/ch/tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { from, fromQuery, param } from "@maple-dev/effect-orm/clickhouse" +import { MetricsGauge, orgIdParam } from "@maple/query-engine/ch/tables" import { CONNECTION_METRIC_NAMES, CPU_METRIC_NAMES, @@ -94,7 +94,7 @@ const bucketOuter = (inner: ReturnType) => export function planetscaleInfraTimeseriesSQL() { const inner = timeseriesInner() .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...ALL_METRIC_NAMES), CH.coalesce( CH.nullIf($.Attributes.get("planetscale_database_name"), ""), @@ -112,7 +112,7 @@ export function planetscaleInfraTimeseriesSQL() { export function planetscaleBranchInfraTimeseriesSQL() { const inner = timeseriesInner() .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...ALL_METRIC_NAMES), CH.coalesce( CH.nullIf($.Attributes.get("planetscale_database_name"), ""), diff --git a/packages/query-engine-integrations/src/planetscale/planetscale-map.test.ts b/packages/query-engine-integrations/src/planetscale/planetscale-map.test.ts index 76120a1c7e..ff434ebb85 100644 --- a/packages/query-engine-integrations/src/planetscale/planetscale-map.test.ts +++ b/packages/query-engine-integrations/src/planetscale/planetscale-map.test.ts @@ -1,15 +1,16 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { planetscaleBranchConnectionsSQL, planetscaleBranchGaugesSQL, planetscaleConnectionsSQL, planetscaleGaugesSQL, } from "./planetscale-map" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", } @@ -52,7 +53,10 @@ describe("planetscaleGaugesSQL", () => { }) it("escapes single quotes in orgId", () => { - const { sql } = compileUnsafe(planetscaleGaugesSQL(), { ...baseParams, orgId: "org'evil" }) + const { sql } = compileUnsafe(planetscaleGaugesSQL(), { + ...baseParams, + orgId: OrgId.make("org'evil"), + }) expect(sql).toContain("OrgId = 'org\\'evil'") }) }) diff --git a/packages/query-engine-integrations/src/planetscale/planetscale-map.ts b/packages/query-engine-integrations/src/planetscale/planetscale-map.ts index fedb6df57b..4a3556faff 100644 --- a/packages/query-engine-integrations/src/planetscale/planetscale-map.ts +++ b/packages/query-engine-integrations/src/planetscale/planetscale-map.ts @@ -19,10 +19,10 @@ import { finiteOrZero } from "@maple/query-engine/ch/format" import { Schema } from "effect" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { from, fromQuery, param, type CompiledQueryRowSchema } from "@maple-dev/effect-clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" +import { from, fromQuery, param, type CompiledQueryRowSchema } from "@maple-dev/effect-orm/clickhouse" import { CHNumber } from "@maple/query-engine/ch/schema" -import { MetricsGauge } from "@maple/query-engine/ch/tables" +import { MetricsGauge, orgIdParam } from "@maple/query-engine/ch/tables" /** Active connections (gauge) — one series per edge region/branch; summed per timestamp. */ export const CONNECTION_METRIC_NAMES = [ @@ -148,7 +148,7 @@ export function planetscaleGaugesSQL() { replicaLagMaxSeconds: CH.maxIf($.Value, $.MetricName.in_(...REPLICA_LAG_METRIC_NAMES)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...GAUGE_METRIC_NAMES), CH.coalesce( CH.nullIf($.Attributes.get("planetscale_database_name"), ""), @@ -179,7 +179,7 @@ export function planetscaleBranchGaugesSQL() { replicaLagMaxSeconds: CH.maxIf($.Value, $.MetricName.in_(...REPLICA_LAG_METRIC_NAMES)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...GAUGE_METRIC_NAMES), CH.coalesce( CH.nullIf($.Attributes.get("planetscale_database_name"), ""), @@ -243,7 +243,7 @@ export function planetscaleStorageSQL() { samples: CH.countIf($.MetricName.in_(...STORAGE_AVAILABLE_METRIC_NAMES)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...STORAGE_METRIC_NAMES), CH.coalesce( CH.nullIf($.Attributes.get("planetscale_database_name"), ""), @@ -282,7 +282,7 @@ export function planetscaleBranchStorageSQL() { samples: CH.countIf($.MetricName.in_(...STORAGE_AVAILABLE_METRIC_NAMES)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...STORAGE_METRIC_NAMES), CH.coalesce( CH.nullIf($.Attributes.get("planetscale_database_name"), ""), @@ -323,7 +323,7 @@ export function planetscaleConnectionsSQL() { totalConnections: CH.sum($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...CONNECTION_METRIC_NAMES), CH.coalesce( CH.nullIf($.Attributes.get("planetscale_database_name"), ""), @@ -361,7 +361,7 @@ export function planetscaleBranchConnectionsSQL() { totalConnections: CH.sum($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...CONNECTION_METRIC_NAMES), CH.coalesce( CH.nullIf($.Attributes.get("planetscale_database_name"), ""), diff --git a/packages/query-engine-integrations/src/planetscale/planetscale-storage.test.ts b/packages/query-engine-integrations/src/planetscale/planetscale-storage.test.ts index 72f33f2440..b38757acd8 100644 --- a/packages/query-engine-integrations/src/planetscale/planetscale-storage.test.ts +++ b/packages/query-engine-integrations/src/planetscale/planetscale-storage.test.ts @@ -1,15 +1,16 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { planetscaleBranchStorageRowSchema, planetscaleBranchStorageSQL, planetscaleStorageRowSchema, planetscaleStorageSQL, } from "./planetscale-map" +import { OrgId } from "@maple/domain" const params = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-07-02 00:00:00.000", endTime: "2026-07-03 00:00:00.000", } diff --git a/packages/query-engine-integrations/src/product/billing-usage.test.ts b/packages/query-engine-integrations/src/product/billing-usage.test.ts index fdaed1e941..ef87fac8bf 100644 --- a/packages/query-engine-integrations/src/product/billing-usage.test.ts +++ b/packages/query-engine-integrations/src/product/billing-usage.test.ts @@ -1,10 +1,11 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" -import { compileUnsafe, type CompiledQuery } from "@maple-dev/effect-clickhouse" +import { compileUnsafe, type CompiledQuery } from "@maple-dev/effect-orm/clickhouse" import { dailyProductEventCountQuery, dailySessionCountQuery, dailySignalVolumeQuery } from "./billing-usage" +import { OrgId } from "@maple/domain" const params = { - orgId: "org_123", + orgId: OrgId.make("org_123"), startTime: "2026-07-01 00:00:00", endTime: "2026-07-31 23:59:59", } diff --git a/packages/query-engine-integrations/src/product/billing-usage.ts b/packages/query-engine-integrations/src/product/billing-usage.ts index 7996dee8bd..84d45eaaee 100644 --- a/packages/query-engine-integrations/src/product/billing-usage.ts +++ b/packages/query-engine-integrations/src/product/billing-usage.ts @@ -19,9 +19,9 @@ // schemas are built from `CHNumber`. import { Schema } from "effect" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { from, param, type CompiledQueryRowSchema } from "@maple-dev/effect-clickhouse" -import { ProductEvents, ServiceUsage, SessionReplays } from "@maple/query-engine/ch/tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { from, param, type CompiledQueryRowSchema } from "@maple-dev/effect-orm/clickhouse" +import { ProductEvents, ServiceUsage, SessionReplays, orgIdParam } from "@maple/query-engine/ch/tables" import { CHNumber } from "@maple/query-engine/ch/schema" import { hourFloor } from "@maple/query-engine/ch/query-helpers" @@ -58,7 +58,7 @@ export function dailySignalVolumeQuery() { .add(CH.sum($.ExpHistogramMetricSizeBytes)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(hourFloor("startTime")), $.Hour.lte(hourFloor("endTime")), ]) @@ -90,7 +90,7 @@ export function dailySessionCountQuery() { sessions: CH.uniqExact($.SessionId), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.StartTime.gte(CH.toDateTime(param.dateTimeString("startTime"))), $.StartTime.lte(CH.toDateTime(param.dateTimeString("endTime"))), ]) @@ -127,7 +127,7 @@ export function dailyProductEventCountQuery() { events: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(CH.toDateTime(param.dateTimeString("startTime"))), $.Timestamp.lte(CH.toDateTime(param.dateTimeString("endTime"))), $.Kind.neq("navigation"), diff --git a/packages/query-engine-integrations/src/product/internal.ts b/packages/query-engine-integrations/src/product/internal.ts index a120056c98..76a3a5a169 100644 --- a/packages/query-engine-integrations/src/product/internal.ts +++ b/packages/query-engine-integrations/src/product/internal.ts @@ -6,10 +6,10 @@ // their scope explicit. import { finiteOrZero } from "@maple/query-engine/ch/format" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import { from } from "@maple-dev/effect-clickhouse" -import { Traces } from "@maple/query-engine/ch/tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { from } from "@maple-dev/effect-orm/clickhouse" +import { Traces, orgIdParam } from "@maple/query-engine/ch/tables" // db.statement samples @@ -74,7 +74,7 @@ export function dbStatementSamplesQuery(opts: DbStatementSamplesOpts) { maxDurationMs: CH.max_($.Duration).div(1000000), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.SpanName.eq("WarehouseQueryService.executeSql"), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), diff --git a/packages/query-engine-integrations/src/product/setup-audit.test.ts b/packages/query-engine-integrations/src/product/setup-audit.test.ts index 2f6c1eca19..7801241548 100644 --- a/packages/query-engine-integrations/src/product/setup-audit.test.ts +++ b/packages/query-engine-integrations/src/product/setup-audit.test.ts @@ -1,7 +1,7 @@ // BOUNDARY: Test doubles mirror intentionally untyped external callbacks. import { describe, expect, it } from "vitest" import { Schema, Effect } from "effect" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { auditAttributeKeyInventoryQuery, auditAttributeKeyInventoryRowSchema, @@ -23,9 +23,10 @@ import { auditSpanProfileByServiceQuery, auditSpanProfileRowSchema, } from "./setup-audit" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", } @@ -156,7 +157,7 @@ describe("auditTraceSampleModulus", () => { describe("trace-completeness joins", () => { const window = { - orgId: "org_1", + orgId: OrgId.make("org_1"), childStart: "2024-01-01 10:00:00", childEnd: "2024-01-01 11:00:00", parentStart: "2024-01-01 09:00:00", @@ -220,7 +221,7 @@ describe("trace-completeness joins", () => { it("escapes the org id so an embedded quote cannot terminate the literal", () => { const { sql } = Effect.runSync( - auditOrphanSpansSQL({ ...window, orgId: "org_'; DROP TABLE traces; --" }), + auditOrphanSpansSQL({ ...window, orgId: OrgId.make("org_'; DROP TABLE traces; --") }), ) // The injected quote is escaped, so the whole payload stays inside one string literal. expect(sql).toContain("OrgId = 'org_\\'\\x3B DROP TABLE traces\\x3B --'") diff --git a/packages/query-engine-integrations/src/product/setup-audit.ts b/packages/query-engine-integrations/src/product/setup-audit.ts index bff750e7ee..95c7395665 100644 --- a/packages/query-engine-integrations/src/product/setup-audit.ts +++ b/packages/query-engine-integrations/src/product/setup-audit.ts @@ -14,8 +14,8 @@ // `CHNumber` in the caller's `rowSchema` — ClickHouse serializes 64-bit integers as JSON strings, so // a BYO-ClickHouse org otherwise fails to decode. -import * as CH from "@maple-dev/effect-clickhouse" -import { compile, from, fromQuery, param, type CompiledQuery } from "@maple-dev/effect-clickhouse" +import * as CH from "@maple-dev/effect-orm/clickhouse" +import { compile, from, fromQuery, param, type CompiledQuery } from "@maple-dev/effect-orm/clickhouse" import { Schema, Effect } from "effect" import { AttributeKeysHourly, @@ -28,11 +28,13 @@ import { ServiceOverviewHourly, TraceListMv, TracesAggregatesHourly, + orgIdParam, } from "@maple/query-engine/ch/tables" import { CHNumber } from "@maple/query-engine/ch/schema" import { hourFloor } from "@maple/query-engine/ch/query-helpers" -import * as T from "@maple-dev/effect-clickhouse/types" -import type { QueryBuilderError } from "@maple-dev/effect-clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" +import type { QueryBuilderError } from "@maple-dev/effect-orm/clickhouse" +import type { OrgId } from "@maple/domain" /** Snaps a window bound to its hour floor so any overlapping hour of an hourly MV contributes. */ @@ -72,7 +74,7 @@ export function auditAttributeKeyInventoryQuery(opts: { limit?: number } = {}) { usageCount: CH.sum($.UsageCount), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), CH.inList($.AttributeScope, ["span", "resource", "log", "metric"]), $.Hour.gte(param.dateTimeString("startTime")), $.Hour.lte(param.dateTimeString("endTime")), @@ -143,7 +145,7 @@ export function auditSpanProfileByServiceQuery(opts: { limit?: number } = {}) { ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(hourFloor("startTime")), $.Hour.lte(hourFloor("endTime")), ]) @@ -186,7 +188,7 @@ export function auditSamplingByServiceQuery(opts: { limit?: number } = {}) { commitTaggedSpanCount: CH.sumIf($.SpanCount, $.CommitSha.neq("")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(hourFloor("startTime")), $.Hour.lte(hourFloor("endTime")), ]) @@ -222,7 +224,7 @@ export function auditLogSeverityByServiceQuery(opts: { limit?: number } = {}) { logCount: CH.sum($.Count), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(hourFloor("startTime")), $.Hour.lte(hourFloor("endTime")), ]) @@ -262,7 +264,7 @@ export function auditMetricLabelCardinalityQuery(opts: { limit?: number } = {}) usageCount: CH.sum($.UsageCount), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.AttributeScope.eq("metric"), $.Hour.gte(param.dateTimeString("startTime")), $.Hour.lte(param.dateTimeString("endTime")), @@ -311,7 +313,7 @@ export function auditPeerValueInventoryQuery(opts: { limit?: number } = {}) { usageCount: CH.sum($.UsageCount), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.AttributeScope.eq("span"), CH.inList($.AttributeKey, [...AUDIT_PEER_KEYS]), $.Hour.gte(param.dateTimeString("startTime")), @@ -353,7 +355,7 @@ export function auditDbEdgeIdentityQuery(opts: { limit?: number } = {}) { unknownNamespaceCallCount: CH.sumIf($.CallCount, $.DbNamespace.eq("")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(hourFloor("startTime")), $.Hour.lte(hourFloor("endTime")), $.DbSystem.neq(""), @@ -405,7 +407,7 @@ export function auditLogCorrelationQuery() { uncorrelatedErrorCount: CH.countIf($.TraceId.eq("").and(isErrorSeverity)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimestampTime.gte(param.dateTimeString("startTime")), $.TimestampTime.lte(param.dateTimeString("endTime")), $.Timestamp.gte(param.dateTimeString("startTime")), @@ -468,7 +470,7 @@ export interface AuditOrphanSpanRow { } export interface AuditTraceWindowParams { - readonly orgId: string + readonly orgId: OrgId /** Children considered, half-open: `[childStart, childEnd)`. */ readonly childStart: string readonly childEnd: string @@ -505,7 +507,7 @@ export function auditOrphanSpansSQL( TraceState: $.TraceState, })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("childStart")), $.Timestamp.lt(param.dateTimeString("childEnd")), $.ParentSpanId.neq(""), @@ -515,7 +517,7 @@ export function auditOrphanSpansSQL( const parents = from(ServiceMapSpans) .select(($) => ({ TraceId: $.TraceId, SpanId: $.SpanId })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("parentStart")), $.Timestamp.lt(param.dateTimeString("childEnd")), modulusFilter($.TraceId, modulus), @@ -584,7 +586,7 @@ export function auditRootlessTracesSQL( isSampled: CH.max(CH.match($.TraceState, SAMPLED_TRACE_STATE_PATTERN)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("childStart")), $.Timestamp.lt(param.dateTimeString("childEnd")), modulusFilter($.TraceId, modulus), @@ -594,7 +596,7 @@ export function auditRootlessTracesSQL( const roots = from(TraceListMv) .select(($) => ({ TraceId: $.TraceId })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("parentStart")), $.Timestamp.lt(param.dateTimeString("childEnd")), modulusFilter($.TraceId, modulus), diff --git a/packages/query-engine-integrations/src/railway/railway-infra.ts b/packages/query-engine-integrations/src/railway/railway-infra.ts index 64976e4ecb..566f4d19d1 100644 --- a/packages/query-engine-integrations/src/railway/railway-infra.ts +++ b/packages/query-engine-integrations/src/railway/railway-infra.ts @@ -5,9 +5,9 @@ // (environment, service) pair. Replicas are summed per timestamp before bucketing. import { finiteOrZero } from "@maple/query-engine/ch/format" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { from, fromQuery, param } from "@maple-dev/effect-clickhouse" -import { MetricsGauge } from "@maple/query-engine/ch/tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { from, fromQuery, param } from "@maple-dev/effect-orm/clickhouse" +import { MetricsGauge, orgIdParam } from "@maple/query-engine/ch/tables" export const RAILWAY_CPU_USAGE = "railway.cpu.usage" export const RAILWAY_CPU_LIMIT = "railway.cpu.limit" @@ -61,7 +61,7 @@ export function railwayServicesSQL() { ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...RAILWAY_METRIC_NAMES), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), @@ -143,7 +143,7 @@ export function railwayServiceTimeseriesSQL() { ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.in_(...RAILWAY_METRIC_NAMES), $.ResourceAttributes.get("railway.environment.id").eq(param.string("environmentId")), $.ResourceAttributes.get("railway.service.id").eq(param.string("serviceId")), diff --git a/packages/query-engine/package.json b/packages/query-engine/package.json index fce1d7f0ac..cd1ca8c8af 100644 --- a/packages/query-engine/package.json +++ b/packages/query-engine/package.json @@ -28,7 +28,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", "@maple/cache": "workspace:*", "@maple/domain": "workspace:*", "@maple/query-model": "workspace:*", diff --git a/packages/query-engine/src/benchmark/builders.ts b/packages/query-engine/src/benchmark/builders.ts index ceb7516d4a..78d4a8109d 100644 --- a/packages/query-engine/src/benchmark/builders.ts +++ b/packages/query-engine/src/benchmark/builders.ts @@ -13,10 +13,11 @@ // only reachable from ErrorsService/telemetry. Remaining modules live on the // exemption list and shrink batch by batch. -import type { CompiledQuery } from "@maple-dev/effect-clickhouse" +import type { CompiledQuery } from "@maple-dev/effect-orm/clickhouse" import * as CH from "../ch" import { Effect } from "effect" -import type { QueryBuilderError } from "@maple-dev/effect-clickhouse" +import type { QueryBuilderError } from "@maple-dev/effect-orm/clickhouse" +import { OrgId } from "@maple/domain" /** * Run a compile that is now Effect-returning, throwing on failure. @@ -54,7 +55,7 @@ export interface BuilderFixture { readonly sampleValues?: Readonly> } -const ORG_ID = "org_sql_catalog" +const ORG_ID = OrgId.make("org_sql_catalog") const START_TIME = "2026-01-01 10:30:00" const END_TIME = "2026-01-03 14:15:00" const SESSION_ID = "sess_0af7651916cd43dd" @@ -903,7 +904,10 @@ export const builderFixtures: ReadonlyArray = [ name: "errorFingerprintOccurrencesQuery", label: "default", compile: () => - CH.compileUnsafe(CH.errorFingerprintOccurrencesQuery({ fingerprintHashes: [FINGERPRINT] }), window), + CH.compileUnsafe( + CH.errorFingerprintOccurrencesQuery({ fingerprintHashes: [FINGERPRINT] }), + window, + ), // The branded TraceId/SpanId columns reject the synthetic "" the sweep would use. sampleValues: { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "b7ad6b7169203331" }, }, diff --git a/packages/query-engine/src/benchmark/catalog.test.ts b/packages/query-engine/src/benchmark/catalog.test.ts index c7d3c827c6..e6e3cd6d70 100644 --- a/packages/query-engine/src/benchmark/catalog.test.ts +++ b/packages/query-engine/src/benchmark/catalog.test.ts @@ -340,6 +340,7 @@ const EXEMPT_BUILDERS: ReadonlySet = new Set([ // todo (dead export — removal tracked separately; zero consumers) "errors/traceTimeProbeQuery", + "errors/recentTraceTimeProbeQuery", // todo batch ② — alerting correctness (anomaly, alert-checks, setup-audit, activity, liveness, internal) "alert-checks/listRuleChecksQuery", diff --git a/packages/query-engine/src/benchmark/catalog.ts b/packages/query-engine/src/benchmark/catalog.ts index f36be0924e..1ef1ab784f 100644 --- a/packages/query-engine/src/benchmark/catalog.ts +++ b/packages/query-engine/src/benchmark/catalog.ts @@ -25,7 +25,7 @@ // executed. `assertRouteCoverage` below fails when a route is never exercised. import { Effect } from "effect" -import type { CompiledQuery } from "@maple-dev/effect-clickhouse" +import type { CompiledQuery } from "@maple-dev/effect-orm/clickhouse" import { DeploymentEnvironment, MetricName, @@ -43,7 +43,7 @@ import { compilePipeQuery } from "../ch/pipe-dispatch" import { compiledQueryOf } from "../execution/compiled-input" import { fingerprintSql } from "../execution/fingerprint" import { makeQueryEngineExecute, type QueryEngineWarehouse, type QueryTenant } from "../runtime" -import { BenchmarkError, validateSuite, type Suite } from "@maple-dev/effect-clickhouse/benchmark" +import { BenchmarkError, validateSuite, type Suite } from "@maple-dev/effect-orm/benchmark" // Fixture inputs diff --git a/packages/query-engine/src/benchmark/index.ts b/packages/query-engine/src/benchmark/index.ts index d5cef6468e..c803ef81d4 100644 --- a/packages/query-engine/src/benchmark/index.ts +++ b/packages/query-engine/src/benchmark/index.ts @@ -1,3 +1,3 @@ // Maple catalog plus the public, driver-free benchmark engine. -export * from "@maple-dev/effect-clickhouse/benchmark" +export * from "@maple-dev/effect-orm/benchmark" export * from "./catalog" diff --git a/packages/query-engine/src/ch/ch.test.ts b/packages/query-engine/src/ch/ch.test.ts index 8fadb9eac6..b60356a014 100644 --- a/packages/query-engine/src/ch/ch.test.ts +++ b/packages/query-engine/src/ch/ch.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import * as CH from "./index" -import { compileUnsafe, compileUnionUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe, compileUnionUnsafe } from "@maple-dev/effect-orm/clickhouse" import { tracesTimeseriesQuery, tracesBreakdownQuery, tracesListQuery } from "./queries/traces" import { logsFacetsQuery } from "./queries/logs" import { servicesFacetsQuery } from "./queries/services" @@ -11,18 +11,23 @@ import { spanHierarchyQuery, spanDetailQuery, traceTimeProbeQuery, + recentTraceTimeProbeQuery, } from "./queries/errors" -import { unionAll } from "@maple-dev/effect-clickhouse" -import * as T from "@maple-dev/effect-clickhouse/types" +import { unionAll } from "@maple-dev/effect-orm/clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" +import { OrgId } from "@maple/domain" // Core DSL tests describe("CH.from / select / where / compile", () => { const TestTable = CH.table("test_table", { - Id: CH.string, - Name: CH.string, - Value: CH.uint64, - Attrs: CH.map(CH.string, CH.string), + external: true, + columns: { + Id: CH.string, + Name: CH.string, + Value: CH.uint64, + Attrs: CH.map(CH.string, CH.string), + }, }) it("compiles a basic select", () => { @@ -50,7 +55,7 @@ describe("CH.from / select / where / compile", () => { .where(($) => [$.Id.eq(CH.param.string("orgId")), $.Name.eq("test")]) .groupBy("id") - const { sql } = compileUnsafe(q, { orgId: "org_123" }) + const { sql } = compileUnsafe(q, { orgId: OrgId.make("org_123") }) expect(sql).toContain("Id AS id") expect(sql).toContain("count() AS count") expect(sql).toContain("Id = 'org_123'") @@ -208,7 +213,7 @@ describe("CH.from / select / where / compile", () => { describe("tracesTimeseriesQuery", () => { const baseParams = { - orgId: "org_123", + orgId: OrgId.make("org_123"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", bucketSeconds: 3600, @@ -690,7 +695,7 @@ describe("tracesTimeseriesQuery", () => { describe("tracesBreakdownQuery", () => { const baseParams = { - orgId: "org_123", + orgId: OrgId.make("org_123"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", } @@ -891,7 +896,7 @@ describe("tracesBreakdownQuery", () => { describe("tracesListQuery", () => { const baseParams = { - orgId: "org_123", + orgId: OrgId.make("org_123"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", } @@ -1003,9 +1008,12 @@ describe("tracesListQuery", () => { describe("unionAll", () => { const TestTable = CH.table("test_table", { - Id: CH.string, - Name: CH.string, - Value: CH.uint64, + external: true, + columns: { + Id: CH.string, + Name: CH.string, + Value: CH.uint64, + }, }) it("compiles two queries with UNION ALL", () => { @@ -1061,9 +1069,12 @@ describe("unionAll", () => { describe("subquery support", () => { const TestTable = CH.table("test_table", { - Id: CH.string, - Name: CH.string, - Value: CH.uint64, + external: true, + columns: { + Id: CH.string, + Name: CH.string, + Value: CH.uint64, + }, }) it("compiles inSubquery", () => { @@ -1121,16 +1132,22 @@ describe("subquery support", () => { describe("type-safe joins", () => { const Users = CH.table("users", { - Id: CH.string, - Name: CH.string, - OrgId: CH.string, + external: true, + columns: { + Id: CH.string, + Name: CH.string, + OrgId: CH.string, + }, }) const Orders = CH.table("orders", { - Id: CH.string, - UserId: CH.string, - Amount: CH.uint64, - Status: CH.string, + external: true, + columns: { + Id: CH.string, + UserId: CH.string, + Amount: CH.uint64, + Status: CH.string, + }, }) it("compiles innerJoin with Table", () => { @@ -1234,7 +1251,10 @@ describe("type-safe joins", () => { }) it("compiles multiple chained joins", () => { - const Tags = CH.table("tags", { Id: CH.string, UserId: CH.string, Label: CH.string }) + const Tags = CH.table("tags", { + external: true, + columns: { Id: CH.string, UserId: CH.string, Label: CH.string }, + }) const q = CH.from(Users) .innerJoin(Orders, "o", (u, o) => u.Id.eq(o.UserId)) @@ -1270,10 +1290,13 @@ describe("type-safe joins", () => { describe("new expression functions", () => { const TestTable = CH.table("test_table", { - Id: CH.string, - Name: CH.string, - Value: CH.uint64, - Attrs: CH.map(CH.string, CH.string), + external: true, + columns: { + Id: CH.string, + Name: CH.string, + Value: CH.uint64, + Attrs: CH.map(CH.string, CH.string), + }, }) it("compiles uniq()", () => { @@ -1343,7 +1366,7 @@ describe("new expression functions", () => { // Converted queries — smoke tests describe("converted queries", () => { - const baseParams = { orgId: "org_1", startTime: "2024-01-01", endTime: "2024-01-02" } + const baseParams = { orgId: OrgId.make("org_1"), startTime: "2024-01-01", endTime: "2024-01-02" } it("logsFacetsQuery compiles UNION ALL", () => { const q = logsFacetsQuery({}) @@ -1409,7 +1432,7 @@ describe("converted queries", () => { it("spanHierarchyQuery projects only the trimmed tree attribute keys", () => { const q = spanHierarchyQuery({ traceId: "abc123" }) - const { sql } = compileUnsafe(q, { orgId: "org_1" }) + const { sql } = compileUnsafe(q, { orgId: OrgId.make("org_1") }) // Maps are trimmed to the keys the tree views render — never the full map. expect(sql).not.toContain("toJSONString(SpanAttributes)") expect(sql).not.toContain("toJSONString(ResourceAttributes)") @@ -1427,14 +1450,14 @@ describe("converted queries", () => { it("spanHierarchyQuery with spanId marks target", () => { const q = spanHierarchyQuery({ traceId: "abc", spanId: "span1" }) - const { sql } = compileUnsafe(q, { orgId: "org_1" }) + const { sql } = compileUnsafe(q, { orgId: OrgId.make("org_1") }) expect(sql).toContain("'target'") expect(sql).toContain("'related'") }) it("spanHierarchyQuery without narrowByTime omits Timestamp filter", () => { const q = spanHierarchyQuery({ traceId: "abc" }) - const { sql } = compileUnsafe(q, { orgId: "org_1" }) + const { sql } = compileUnsafe(q, { orgId: OrgId.make("org_1") }) expect(sql).not.toContain("Timestamp >=") expect(sql).not.toContain("Timestamp <=") }) @@ -1442,7 +1465,7 @@ describe("converted queries", () => { it("spanHierarchyQuery with narrowByTime adds Timestamp BETWEEN filter", () => { const q = spanHierarchyQuery({ traceId: "abc", narrowByTime: true }) const { sql } = compileUnsafe(q, { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-04-15 13:00:00", endTime: "2026-04-15 15:00:00", }) @@ -1452,7 +1475,7 @@ describe("converted queries", () => { it("spanDetailQuery is a point lookup returning the full attribute maps", () => { const q = spanDetailQuery({ traceId: "abc123", spanId: "span1" }) - const { sql } = compileUnsafe(q, { orgId: "org_1" }) + const { sql } = compileUnsafe(q, { orgId: OrgId.make("org_1") }) expect(sql).toContain("toJSONString(trace_detail_spans.SpanAttributes) AS spanAttributes") expect(sql).toContain("toJSONString(trace_detail_spans.ResourceAttributes) AS resourceAttributes") expect(sql).toContain("FROM trace_detail_spans") @@ -1467,7 +1490,7 @@ describe("converted queries", () => { it("spanDetailQuery with narrowByTime adds Timestamp filters", () => { const q = spanDetailQuery({ traceId: "abc", spanId: "s1", narrowByTime: true }) const { sql } = compileUnsafe(q, { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-04-15 13:00:00", endTime: "2026-04-15 15:00:00", }) @@ -1477,7 +1500,7 @@ describe("converted queries", () => { it("traceTimeProbeQuery is a cheap LIMIT-1 timestamp seek with no ORDER BY", () => { const q = traceTimeProbeQuery({ traceId: "abc123" }) - const { sql } = compileUnsafe(q, { orgId: "org_1" }) + const { sql } = compileUnsafe(q, { orgId: OrgId.make("org_1") }) expect(sql).toContain("Timestamp AS timestamp") expect(sql).toContain("FROM trace_detail_spans") expect(sql).toContain("TraceId = 'abc123'") @@ -1490,9 +1513,9 @@ describe("converted queries", () => { expect(sql).not.toContain("Timestamp >=") }) - it("traceTimeProbeQuery with narrowByTime adds a Timestamp lower bound for partition pruning", () => { - const q = traceTimeProbeQuery({ traceId: "abc123", narrowByTime: true }) - const { sql } = compileUnsafe(q, { orgId: "org_1", startTime: "2026-04-15 13:00:00" }) + it("recentTraceTimeProbeQuery adds a Timestamp lower bound for partition pruning", () => { + const q = recentTraceTimeProbeQuery({ traceId: "abc123" }) + const { sql } = compileUnsafe(q, { orgId: OrgId.make("org_1"), startTime: "2026-04-15 13:00:00" }) expect(sql).toContain("Timestamp >= '2026-04-15 13:00:00'") // Lower bound only — "recent" needs no upper bound. expect(sql).not.toContain("Timestamp <=") diff --git a/packages/query-engine/src/ch/endpoint-types.test-d.ts b/packages/query-engine/src/ch/endpoint-types.test-d.ts index 331c5853bf..2827cd7fb0 100644 --- a/packages/query-engine/src/ch/endpoint-types.test-d.ts +++ b/packages/query-engine/src/ch/endpoint-types.test-d.ts @@ -34,7 +34,7 @@ import type { SpanHierarchyOutput, TracesDurationStatsOutput, } from "@maple/domain/tinybird" -import type { InferQueryOutput } from "@maple-dev/effect-clickhouse" +import type { InferQueryOutput } from "@maple-dev/effect-orm/clickhouse" import { resourceAttributeValuesQuery, spanAttributeValuesQuery } from "./queries/attribute-keys" import { errorDetailTracesQuery, diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index cfebc36473..1865d1e3ed 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -1,13 +1,13 @@ // ClickHouse Query DSL — Maple facade // // The generic, reusable query builder now lives in the standalone -// @maple-dev/effect-clickhouse package. This module re-exports that public API +// @maple-dev/effect-orm package. This module re-exports that public API // and layers Maple's OpenTelemetry-specific table definitions, the named-query // ("pipe") registry, and the pre-built query templates on top of it. // Generic DSL — types, table, expressions, functions, params, query builder, // compilation, and unions — re-exported from the standalone library. -export * from "@maple-dev/effect-clickhouse" +export * from "@maple-dev/effect-orm/clickhouse" // Handwritten SQL. Shadows the builder's `rawCompiledQuery`, whose `reason` // is any string, with one that pins Maple's closed `RawSqlReason` union — the @@ -340,6 +340,7 @@ export { SPAN_HIERARCHY_MAX_SPANS, spanDetailQuery, traceTimeProbeQuery, + recentTraceTimeProbeQuery, tracesDurationStatsQuery, canUseTraceFacetsRollup, tracesFacetsQuery, diff --git a/packages/query-engine/src/ch/pipe-dispatch.ts b/packages/query-engine/src/ch/pipe-dispatch.ts index 6ecb1c572b..6dd4c245c0 100644 --- a/packages/query-engine/src/ch/pipe-dispatch.ts +++ b/packages/query-engine/src/ch/pipe-dispatch.ts @@ -15,10 +15,16 @@ import type { TracesMetric, AttributeFilter, MetricType } from "@maple/domain/query-engine" import { DEFAULT_ERROR_NAMESPACE_PREFIX, UNEXPECTED_IDENTITY_MARKERS } from "./queries/errors" import type { OrgId } from "@maple/domain" -import { compile, compileUnion, type CompiledQuery } from "@maple-dev/effect-clickhouse" +import { + compile, + compileUnion, + type CHQuery, + type CompiledQuery, + type NeedsSelect, +} from "@maple-dev/effect-orm/clickhouse" import { rawCompiledQuery } from "./raw-sql" import { Array as A, Effect, Match, Result, Schema } from "effect" -import type { QueryBuilderError } from "@maple-dev/effect-clickhouse" +import type { QueryBuilderError } from "@maple-dev/effect-orm/clickhouse" import { attributeIndexMode, baselineWarehouseCapabilities, @@ -72,7 +78,8 @@ import { type TracesTimeseriesOpts, } from "./queries/traces" -type CompileTarget = Parameters[0] +/** A query `compile` accepts, whatever it selects. */ +type CompileTarget> = CHQuery & NeedsSelect export type PipeCompiledQuery = CompiledQuery @@ -114,7 +121,7 @@ export function compilePipeQuery( params: PipeParams, capabilities: WarehouseCapabilities = baselineWarehouseCapabilities(), ): PipeCompiled | undefined { - const orgId = String(params.org_id) + const orgId = params.org_id const startTime = String(params.start_time ?? "2023-01-01 00:00:00") const endTime = String(params.end_time ?? "2099-12-31 23:59:59") const str = (key: string) => (params[key] != null ? String(params[key]) : undefined) @@ -147,9 +154,10 @@ export function compilePipeQuery( // up: a row schema decodes bytes off a socket, so it cannot ask for a service, // and a struct is service-free exactly when its fields are. const compileCompare = < + Output extends Record, Fields extends Schema.Struct.Fields & Record>, >( - query: CompileTarget, + query: CompileTarget, ranges: { currentStart: string currentEnd: string @@ -624,12 +632,15 @@ export function compilePipeQuery( const serviceName = str("service_name") if (serviceName) { return eraseType( - compile(serviceScopedAttributeKeysQuery({ scope: "span", limit: int("limit", 200) }), { - orgId, - startTime, - endTime, - serviceName, - }), + compile( + serviceScopedAttributeKeysQuery({ scope: "span", limit: int("limit", 200) }), + { + orgId, + startTime, + endTime, + serviceName, + }, + ), ) } return eraseType( @@ -645,12 +656,15 @@ export function compilePipeQuery( const serviceName = str("service_name") if (serviceName) { return eraseType( - compile(serviceScopedAttributeKeysQuery({ scope: "resource", limit: int("limit", 200) }), { - orgId, - startTime, - endTime, - serviceName, - }), + compile( + serviceScopedAttributeKeysQuery({ scope: "resource", limit: int("limit", 200) }), + { + orgId, + startTime, + endTime, + serviceName, + }, + ), ) } return eraseType( diff --git a/packages/query-engine/src/ch/queries/activity.test.ts b/packages/query-engine/src/ch/queries/activity.test.ts index d577e55feb..92b424d95e 100644 --- a/packages/query-engine/src/ch/queries/activity.test.ts +++ b/packages/query-engine/src/ch/queries/activity.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { activeOrgsByErrorEventsQuery, activeOrgsByLogsQuery, activeOrgsByTracesQuery } from "./activity" const params = { startTime: "2026-06-22 05:00:00" } diff --git a/packages/query-engine/src/ch/queries/activity.ts b/packages/query-engine/src/ch/queries/activity.ts index 923fa98aeb..3acedb3d22 100644 --- a/packages/query-engine/src/ch/queries/activity.ts +++ b/packages/query-engine/src/ch/queries/activity.ts @@ -17,7 +17,7 @@ // The discovery window must be a SUPERSET of the per-org scan window so no // active org is missed for the tick. -import { from, param } from "@maple-dev/effect-clickhouse" +import { from, param } from "@maple-dev/effect-orm/clickhouse" import type { OrgId } from "@maple/domain" import { ErrorEventsByTime, LogsAggregatesHourly, TracesAggregatesHourly } from "../tables" diff --git a/packages/query-engine/src/ch/queries/alert-checks.test.ts b/packages/query-engine/src/ch/queries/alert-checks.test.ts index 5efc87072d..0c50952a72 100644 --- a/packages/query-engine/src/ch/queries/alert-checks.test.ts +++ b/packages/query-engine/src/ch/queries/alert-checks.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from "@effect/vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { alertCheckGroupTotalsQuery, alertChecksSummaryQuery, listRuleChecksQuery } from "./alert-checks" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), ruleId: "rule_1", } @@ -57,7 +58,7 @@ describe("listRuleChecksQuery", () => { it("escapes single quotes in orgId", () => { const q = listRuleChecksQuery({ limit: 10 }) - const { sql } = compileUnsafe(q, { orgId: "org'evil", ruleId: "rule_1" }) + const { sql } = compileUnsafe(q, { orgId: OrgId.make("org'evil"), ruleId: "rule_1" }) expect(sql).toContain("OrgId = 'org\\'evil'") }) diff --git a/packages/query-engine/src/ch/queries/alert-checks.ts b/packages/query-engine/src/ch/queries/alert-checks.ts index 4c5693af5e..a6eba16829 100644 --- a/packages/query-engine/src/ch/queries/alert-checks.ts +++ b/packages/query-engine/src/ch/queries/alert-checks.ts @@ -3,12 +3,12 @@ // DSL-based query definitions for listing historical alert rule check rows // from the `alert_checks` datasource. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import { from } from "@maple-dev/effect-clickhouse" -import { AlertChecks } from "../tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { from } from "@maple-dev/effect-orm/clickhouse" +import { AlertChecks, orgIdParam } from "../tables" import { ISO_Z_FORMAT } from "./format" -import * as T from "@maple-dev/effect-clickhouse/types" +import * as T from "@maple-dev/effect-orm/clickhouse" export interface ListRuleChecksOpts { readonly groupKey?: string @@ -67,21 +67,21 @@ export function listRuleChecksQuery(opts: ListRuleChecksOpts) { skipReason: $.SkipReason, })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.RuleId.eq(param.string("ruleId")), - opts.groupKey != null && opts.groupKey !== "" - ? $.GroupKey.eq(param.string("groupKey")) - : undefined, - opts.status != null ? $.Status.eq(param.string("status")) : undefined, - opts.since != null ? $.Timestamp.gte(param.dateTimeString("since")) : undefined, - opts.until != null ? $.Timestamp.lte(param.dateTimeString("until")) : undefined, - opts.beforeTimestamp != null - ? $.Timestamp.lt(param.dateTimeString("beforeTimestamp")).or( - $.Timestamp.eq(param.dateTimeString("beforeTimestamp")).and( - $.GroupKey.gt(param.string("beforeGroupKey")), - ), - ) - : undefined, + CH.whenTrue(opts.groupKey != null && opts.groupKey !== "", () => + $.GroupKey.eq(param.string("groupKey")), + ), + CH.whenTrue(opts.status != null, () => $.Status.eq(param.string("status"))), + CH.whenTrue(opts.since != null, () => $.Timestamp.gte(param.dateTimeString("since"))), + CH.whenTrue(opts.until != null, () => $.Timestamp.lte(param.dateTimeString("until"))), + CH.whenTrue(opts.beforeTimestamp != null, () => + $.Timestamp.lt(param.dateTimeString("beforeTimestamp")).or( + $.Timestamp.eq(param.dateTimeString("beforeTimestamp")).and( + $.GroupKey.gt(param.string("beforeGroupKey")), + ), + ), + ), ]) .orderBy(["timestamp", "desc"], ["groupKey", "asc"]) .limit(opts.limit) @@ -111,7 +111,7 @@ export function alertCheckGroupTotalsQuery(opts: AlertCheckGroupTotalsOpts) { totalCount: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.RuleId.eq(param.string("ruleId")), $.Timestamp.gte(param.dateTimeString("since")), $.Timestamp.lte(param.dateTimeString("until")), @@ -158,7 +158,7 @@ export function alertChecksSummaryQuery(opts: AlertChecksSummaryOpts) { threshold: CH.avg($.Threshold), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.RuleId.eq(param.string("ruleId")), $.Timestamp.gte(param.dateTimeString("since")), $.Timestamp.lte(param.dateTimeString("until")), diff --git a/packages/query-engine/src/ch/queries/anomaly.test.ts b/packages/query-engine/src/ch/queries/anomaly.test.ts index 6edcb058b7..16dd8569bd 100644 --- a/packages/query-engine/src/ch/queries/anomaly.test.ts +++ b/packages/query-engine/src/ch/queries/anomaly.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { anomalyErrorSpikeBaselineQuery, anomalyErrorSpikeCurrentQuery, @@ -10,9 +10,10 @@ import { anomalyTraceSignalTimeseriesQuery, matchedHoursOfDay, } from "./anomaly" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-08 00:00:00", } diff --git a/packages/query-engine/src/ch/queries/anomaly.ts b/packages/query-engine/src/ch/queries/anomaly.ts index d02322a259..a7b720bb28 100644 --- a/packages/query-engine/src/ch/queries/anomaly.ts +++ b/packages/query-engine/src/ch/queries/anomaly.ts @@ -11,11 +11,11 @@ // observation) and ≤21 sealed same-hour-of-day samples (the baseline) in ONE // query; the caller splits rows on `hour === currentHourStart`. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import { from, fromQuery } from "@maple-dev/effect-clickhouse" -import { ErrorEventsByTime, LogsAggregatesHourly, TracesAggregatesHourly } from "../tables" -import * as T from "@maple-dev/effect-clickhouse/types" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { from, fromQuery } from "@maple-dev/effect-orm/clickhouse" +import { ErrorEventsByTime, LogsAggregatesHourly, TracesAggregatesHourly, orgIdParam } from "../tables" +import * as T from "@maple-dev/effect-orm/clickhouse" /** Hour-of-day values matching the current hour ±1, wrapping at midnight. */ export function matchedHoursOfDay(currentHourOfDay: number): readonly number[] { @@ -54,7 +54,7 @@ export function anomalyTraceSignalsQuery(opts: AnomalyTraceSignalsOpts) { ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.IsEntryPoint.eq(1), $.Hour.gte(param.dateTimeSeconds("startTime")), $.Hour.lte(param.dateTimeSeconds("endTime")), @@ -88,7 +88,7 @@ export function anomalyLogVolumeQuery(opts: AnomalyTraceSignalsOpts) { warnLogCount: CH.sumIf($.Count, CH.lower_($.SeverityText).in_(...WARN_SEVERITIES)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(param.dateTimeSeconds("startTime")), $.Hour.lte(param.dateTimeSeconds("endTime")), CH.toHour($.Hour).in_(...opts.hoursOfDay), @@ -120,7 +120,7 @@ export function anomalyErrorSpikeCurrentQuery(opts: { limit?: number }) { count: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), ]) @@ -150,7 +150,7 @@ export function anomalyErrorSpikeBaselineQuery(opts: { limit?: number }) { hourCount: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lt(param.dateTimeSeconds("endTime")), ]) @@ -195,7 +195,7 @@ export function anomalyTraceSignalTimeseriesQuery() { ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), // Must match the detector's series definition (entry-point spans only). $.IsEntryPoint.eq(1), $.ServiceName.eq(param.string("serviceName")), @@ -222,7 +222,7 @@ export function anomalyLogVolumeTimeseriesQuery() { errorLogCount: CH.sumIf($.Count, CH.lower_($.SeverityText).in_(...ERROR_SEVERITIES)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.DeploymentEnv.eq(param.string("deploymentEnv")), $.Hour.gte(param.dateTimeSeconds("startTime")), @@ -252,7 +252,7 @@ export function anomalyErrorSpikeTimeseriesQuery() { count: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.FingerprintHash.eq(CH.toUInt64(param.string("fingerprintHash"))), $.DeploymentEnv.eq(param.string("deploymentEnv")), $.Timestamp.gte(param.dateTimeSeconds("startTime")), @@ -280,7 +280,7 @@ export function anomalyErrorSpikeServiceTimeseriesQuery() { count: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.DeploymentEnv.eq(param.string("deploymentEnv")), $.Timestamp.gte(param.dateTimeSeconds("startTime")), diff --git a/packages/query-engine/src/ch/queries/attribute-keys.test.ts b/packages/query-engine/src/ch/queries/attribute-keys.test.ts index a20776e3e9..de84559939 100644 --- a/packages/query-engine/src/ch/queries/attribute-keys.test.ts +++ b/packages/query-engine/src/ch/queries/attribute-keys.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { attributeKeysQuery, logAttributeValuesQuery, @@ -11,9 +11,10 @@ import { serviceScopedAttributeValuesQuery, spanAttributeValuesQuery, } from "./attribute-keys" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", } diff --git a/packages/query-engine/src/ch/queries/attribute-keys.ts b/packages/query-engine/src/ch/queries/attribute-keys.ts index 79f2ff6694..d080e3d4c3 100644 --- a/packages/query-engine/src/ch/queries/attribute-keys.ts +++ b/packages/query-engine/src/ch/queries/attribute-keys.ts @@ -1,8 +1,8 @@ import type { MetricType } from "@maple/domain/query-engine" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import { from, fromQuery } from "@maple-dev/effect-clickhouse" -import { AttributeKeysHourly, AttributeValuesHourly, MetricsSum, Traces } from "../tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { from, fromQuery } from "@maple-dev/effect-orm/clickhouse" +import { AttributeKeysHourly, AttributeValuesHourly, MetricsSum, Traces, orgIdParam } from "../tables" import { resolveMetricTable } from "./query-helpers" export interface AttributeKeysQueryOpts { @@ -22,7 +22,7 @@ export function attributeKeysQuery(opts: AttributeKeysQueryOpts) { usageCount: CH.sum($.UsageCount), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(param.dateTimeSeconds("startTime")), $.Hour.lte(param.dateTimeSeconds("endTime")), $.AttributeScope.eq(opts.scope), @@ -52,7 +52,7 @@ export function spanAttributeValuesQuery(opts: AttributeValuesOpts) { usageCount: CH.sum($.UsageCount), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(param.dateTimeSeconds("startTime")), $.Hour.lte(param.dateTimeSeconds("endTime")), $.AttributeScope.eq("span"), @@ -71,7 +71,7 @@ export function resourceAttributeValuesQuery(opts: AttributeValuesOpts) { usageCount: CH.sum($.UsageCount), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(param.dateTimeSeconds("startTime")), $.Hour.lte(param.dateTimeSeconds("endTime")), $.AttributeScope.eq("resource"), @@ -90,7 +90,7 @@ export function logAttributeValuesQuery(opts: AttributeValuesOpts) { usageCount: CH.sum($.UsageCount), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(param.dateTimeSeconds("startTime")), $.Hour.lte(param.dateTimeSeconds("endTime")), $.AttributeScope.eq("log"), @@ -121,7 +121,7 @@ export function metricScopedAttributeKeysQuery(opts: MetricScopedAttributeKeysOp usageCount: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.eq(param.string("metricName")), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), @@ -148,7 +148,7 @@ export function metricScopedAttributeValuesQuery(opts: MetricScopedAttributeValu usageCount: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.eq(param.string("metricName")), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), @@ -168,7 +168,7 @@ export function metricAttributeValuesQuery(opts: AttributeValuesOpts) { usageCount: CH.sum($.UsageCount), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(param.dateTimeSeconds("startTime")), $.Hour.lte(param.dateTimeSeconds("endTime")), $.AttributeScope.eq("metric"), @@ -197,7 +197,7 @@ const serviceSpanSample = (scope: "span" | "resource") => attrs: scope === "resource" ? $.ResourceAttributes : $.SpanAttributes, })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), diff --git a/packages/query-engine/src/ch/queries/audit-log.ts b/packages/query-engine/src/ch/queries/audit-log.ts index 2b0dfd6e50..83c961529d 100644 --- a/packages/query-engine/src/ch/queries/audit-log.ts +++ b/packages/query-engine/src/ch/queries/audit-log.ts @@ -1,6 +1,6 @@ -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { from, param, paramPlaceholder } from "@maple-dev/effect-clickhouse" -import { AuditLog } from "../tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { from, param, paramPlaceholder } from "@maple-dev/effect-orm/clickhouse" +import { AuditLog, orgIdParam } from "../tables" /** * Which optional filters a listing applies. Every set flag binds a parameter of @@ -56,24 +56,24 @@ export function auditLogEntriesQuery(opts: AuditLogEntriesOpts) { originCountry: $.OriginCountry, })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), - opts.actorType ? $.ActorType.eq(param.string("actorType")) : undefined, - opts.userId ? $.UserId.eq(param.string("userId")) : undefined, - opts.apiKeyId ? $.ApiKeyId.eq(param.string("apiKeyId")) : undefined, - opts.actorId ? $.ActorId.eq(param.string("actorId")) : undefined, - opts.affectedUserId ? $.AffectedUserId.eq(param.string("affectedUserId")) : undefined, - opts.action ? $.Action.eq(param.string("action")) : undefined, - opts.outcome ? $.Outcome.eq(param.string("outcome")) : undefined, - opts.resourceType ? $.ResourceType.eq(param.string("resourceType")) : undefined, - opts.resourceId ? $.ResourceId.eq(param.string("resourceId")) : undefined, + $.OrgId.eq(orgIdParam), + CH.whenTrue(!!opts.actorType, () => $.ActorType.eq(param.string("actorType"))), + CH.whenTrue(!!opts.userId, () => $.UserId.eq(param.string("userId"))), + CH.whenTrue(!!opts.apiKeyId, () => $.ApiKeyId.eq(param.string("apiKeyId"))), + CH.whenTrue(!!opts.actorId, () => $.ActorId.eq(param.string("actorId"))), + CH.whenTrue(!!opts.affectedUserId, () => $.AffectedUserId.eq(param.string("affectedUserId"))), + CH.whenTrue(!!opts.action, () => $.Action.eq(param.string("action"))), + CH.whenTrue(!!opts.outcome, () => $.Outcome.eq(param.string("outcome"))), + CH.whenTrue(!!opts.resourceType, () => $.ResourceType.eq(param.string("resourceType"))), + CH.whenTrue(!!opts.resourceId, () => $.ResourceId.eq(param.string("resourceId"))), // Array membership has no builder verb yet; the placeholder keeps the // value parameterised exactly like the typed comparisons above. opts.changedField ? CH.rawCond(`has(ChangedFields, ${paramPlaceholder("string", "changedField")})`) : undefined, - opts.requestId ? $.RequestId.eq(param.string("requestId")) : undefined, - opts.since ? $.OccurredAt.gte(param.dateTimeString("since")) : undefined, - opts.until ? $.OccurredAt.lte(param.dateTimeString("until")) : undefined, + CH.whenTrue(!!opts.requestId, () => $.RequestId.eq(param.string("requestId"))), + CH.whenTrue(!!opts.since, () => $.OccurredAt.gte(param.dateTimeString("since"))), + CH.whenTrue(!!opts.until, () => $.OccurredAt.lte(param.dateTimeString("until"))), ]) .orderBy(["occurredAt", "desc"], ["id", "desc"]) .limit(opts.limit) diff --git a/packages/query-engine/src/ch/queries/containers.test.ts b/packages/query-engine/src/ch/queries/containers.test.ts index 02a4cc560d..00fa43addd 100644 --- a/packages/query-engine/src/ch/queries/containers.test.ts +++ b/packages/query-engine/src/ch/queries/containers.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" -import { compileUnionUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" +import { compileUnionUnsafe } from "@maple-dev/effect-orm/clickhouse" import { listContainersQuery, listContainersSummaryQuery, @@ -10,9 +10,10 @@ import { containerSumTimeseriesQuery, containerFacetsQuery, } from "./containers" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", bucketSeconds: 60, diff --git a/packages/query-engine/src/ch/queries/containers.ts b/packages/query-engine/src/ch/queries/containers.ts index 3f4468b73a..62b83deb2d 100644 --- a/packages/query-engine/src/ch/queries/containers.ts +++ b/packages/query-engine/src/ch/queries/containers.ts @@ -21,11 +21,11 @@ // they are excluded by requiring `k8s.pod.name = ''` (same trick as the node // queries). -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import { from, fromQuery, type ColumnAccessor } from "@maple-dev/effect-clickhouse" -import { unionAll, type CHUnionQuery } from "@maple-dev/effect-clickhouse" -import { MetricsGauge, MetricsSum } from "../tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { from, fromQuery, type ColumnAccessor } from "@maple-dev/effect-orm/clickhouse" +import { unionAll, type CHUnionQuery } from "@maple-dev/effect-orm/clickhouse" +import { MetricsGauge, MetricsSum, orgIdParam } from "../tables" import { containerRuntimeExpr, deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import { avgIfOrZero, facetAttrExpr, maxIfOrZero, type FacetOutput } from "./query-helpers" import type { SortDirection } from "./infra" @@ -116,7 +116,7 @@ const containerBaseConditions = ( $: ColumnAccessor, metricNames: ReadonlyArray = CONTAINER_METRIC_NAMES, ): Array => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("container.name").neq(""), @@ -325,7 +325,7 @@ export function containerDetailSummaryQuery(opts: ContainerDetailSummaryOpts) { uptimeSeconds: maxIfOrZero($.Value, $.MetricName.eq("container.uptime")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("container.name").eq(opts.containerName), @@ -372,7 +372,7 @@ export function containerCountersSummaryQuery(opts: ContainerCountersSummaryOpts pidsAvg: avgIfOrZero($.Value, $.MetricName.eq("container.pids.count")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("container.name").eq(opts.containerName), @@ -423,7 +423,7 @@ export function containerGaugeTimeseriesQuery(opts: ContainerGaugeTimeseriesOpts avgValue: opts.divideBy ? CH.avg($.Value).div(opts.divideBy) : CH.avg($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("container.name").eq(opts.containerName), @@ -475,7 +475,7 @@ export function containerSumTimeseriesQuery(opts: ContainerSumTimeseriesOpts) { sumValue: opts.average ? CH.avg($.Value) : CH.sum($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("container.name").eq(opts.containerName), diff --git a/packages/query-engine/src/ch/queries/count-semantics.test.ts b/packages/query-engine/src/ch/queries/count-semantics.test.ts index 5eea654bbc..0dc3b9e981 100644 --- a/packages/query-engine/src/ch/queries/count-semantics.test.ts +++ b/packages/query-engine/src/ch/queries/count-semantics.test.ts @@ -15,12 +15,13 @@ // reading 5.5B directly above a status donut totalling 26.6M. import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { tracesBreakdownQuery, tracesTimeseriesQuery } from "./traces" import type { TracesBaseWhereOpts } from "./query-helpers" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_123", + orgId: OrgId.make("org_123"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", bucketSeconds: 3600, @@ -170,7 +171,7 @@ describe("traces count is sample-weighted on every route", () => { it("weights count on the raw breakdown path", () => { const { sql } = compileUnsafe(tracesBreakdownQuery({ metric: "count", groupBy: "span_name" }), { - orgId: "org_123", + orgId: OrgId.make("org_123"), startTime: baseParams.startTime, endTime: baseParams.endTime, }) @@ -181,7 +182,7 @@ describe("traces count is sample-weighted on every route", () => { it("weights count on the MV breakdown path", () => { const { sql } = compileUnsafe( tracesBreakdownQuery({ metric: "count", groupBy: "service", rootOnly: true }), - { orgId: "org_123", startTime: baseParams.startTime, endTime: baseParams.endTime }, + { orgId: OrgId.make("org_123"), startTime: baseParams.startTime, endTime: baseParams.endTime }, ) expect(sourceTable(sql)).toBe("service_overview_spans") expectWeighted(sql) @@ -206,7 +207,7 @@ describe("a breakdown totals the same regardless of the dimension", () => { const compiled = BREAKDOWN_DIMENSIONS.map( (dim) => compileUnsafe(tracesBreakdownQuery({ metric: "count", ...dim, ...filters }), { - orgId: "org_123", + orgId: OrgId.make("org_123"), startTime: baseParams.startTime, endTime: baseParams.endTime, }).sql, @@ -241,7 +242,7 @@ describe("timeseries and breakdown agree for the same query", () => { const breakdownSql = (groupBy: string, opts: TracesBaseWhereOpts) => compileUnsafe(tracesBreakdownQuery({ metric: "count", groupBy, ...opts }), { - orgId: "org_123", + orgId: OrgId.make("org_123"), startTime: baseParams.startTime, endTime: baseParams.endTime, }).sql diff --git a/packages/query-engine/src/ch/queries/errors.test.ts b/packages/query-engine/src/ch/queries/errors.test.ts index 8bfa761a42..c4fc62cee9 100644 --- a/packages/query-engine/src/ch/queries/errors.test.ts +++ b/packages/query-engine/src/ch/queries/errors.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe, compileUnionUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe, compileUnionUnsafe } from "@maple-dev/effect-orm/clickhouse" import { errorsByTypeQuery, errorsTimeseriesQuery, @@ -14,9 +14,10 @@ import { tracesDurationStatsQuery, tracesFacetsQuery, } from "./errors" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", bucketSeconds: 3600, diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index 9ff34c0e21..47cd9ccf8b 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -4,15 +4,21 @@ import { finiteOrZero } from "./format" import { edgeCondition, interiorConditions } from "./rollup-splice" -import * as CH from "@maple-dev/effect-clickhouse/expr" +import * as CH from "@maple-dev/effect-orm/expr" // From the root, not `/expr`: these overloads take a `CHQuery`, keeping the // subquery's params, table names and column types checked. -import { exists, inSubquery } from "@maple-dev/effect-clickhouse" -import { param } from "@maple-dev/effect-clickhouse" -import { from, fromQuery, fromUnion, type CHQuery, type ColumnAccessor } from "@maple-dev/effect-clickhouse" -import type { ColumnDefs } from "@maple-dev/effect-clickhouse/types" -import * as T from "@maple-dev/effect-clickhouse/types" -import { unionAll, type CHUnionQuery } from "@maple-dev/effect-clickhouse" +import { exists, inSubquery } from "@maple-dev/effect-orm/clickhouse" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { + from, + fromQuery, + fromUnion, + type CHQuery, + type ColumnAccessor, +} from "@maple-dev/effect-orm/clickhouse" +import type { ColumnDefs } from "@maple-dev/effect-orm/clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" +import { unionAll, type CHUnionQuery } from "@maple-dev/effect-orm/clickhouse" import type { SpanId, TraceId } from "@maple/domain" import { Schema } from "effect" import { @@ -24,6 +30,7 @@ import { TraceFacetsHourly, TraceListMv, Traces, + orgIdParam, } from "../tables" import { buildProjectedMapExpr, @@ -203,7 +210,7 @@ export function errorsByTypeQuery(opts: ErrorsByTypeOpts) { lastSeen: CH.max_($.Timestamp), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), CH.whenTrue(!!opts.rootOnly, () => $.ParentSpanId.eq("")), @@ -245,7 +252,7 @@ export function errorsTimeseriesQuery(opts: ErrorsTimeseriesOpts) { count: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), fingerprintHashEq($.FingerprintHash, opts.fingerprintHash), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), @@ -288,7 +295,7 @@ export function errorsSparkQuery(opts: ErrorsSparkOpts) { count: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), fingerprintHashIn($.FingerprintHash, opts.fingerprintHashes), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), @@ -429,7 +436,7 @@ export function spanHierarchyQuery(opts: SpanHierarchyOpts) { }) .where(($) => [ $.TraceId.eq(opts.traceId), - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), CH.whenTrue(!!opts.narrowByTime, () => $.Timestamp.gte(param.dateTimeString("startTime"))), CH.whenTrue(!!opts.narrowByTime, () => $.Timestamp.lte(param.dateTimeString("endTime"))), ]) @@ -498,7 +505,7 @@ export function spanDetailQuery(opts: SpanDetailOpts) { .where(($) => [ $.TraceId.eq(opts.traceId), $.SpanId.eq(opts.spanId), - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), CH.whenTrue(!!opts.narrowByTime, () => $.Timestamp.gte(param.dateTimeString("startTime"))), CH.whenTrue(!!opts.narrowByTime, () => $.Timestamp.lte(param.dateTimeString("endTime"))), ]) @@ -526,18 +533,19 @@ export interface TraceTimeProbeOutput { * (pruning to the last few partitions) and only fall back to the unbounded * probe when the trace is older than that window. */ -export function traceTimeProbeQuery(opts: { traceId: string; narrowByTime?: boolean }) { +export function traceTimeProbeQuery(opts: { traceId: string }) { return from(TraceDetailSpans) .select(($) => ({ timestamp: $.Timestamp })) - .where(($) => [ - $.TraceId.eq(opts.traceId), - $.OrgId.eq(param.string("orgId")), - CH.whenTrue(!!opts.narrowByTime, () => $.Timestamp.gte(param.dateTimeString("startTime"))), - ]) + .where(($) => [$.TraceId.eq(opts.traceId), $.OrgId.eq(orgIdParam)]) .limit(1) .format("JSON") } +/** {@link traceTimeProbeQuery} bounded below by the `startTime` param. */ +export function recentTraceTimeProbeQuery(opts: { traceId: string }) { + return traceTimeProbeQuery(opts).where(($) => [$.Timestamp.gte(param.dateTimeString("startTime"))]) +} + // Traces duration stats and facets // // Both count root spans over the window. When `trace_facets_hourly` carries the @@ -640,7 +648,7 @@ export function canUseTraceFacetsRollup( /** `trace_list_mv` rows in the window, narrowed to the partial end hours when the rollup has the rest. */ function traceListWindowConditions($: ColumnAccessor, opts: TracesFacetsOpts) { return [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), ...traceFacetDimensionConditions($, opts), @@ -654,20 +662,7 @@ function traceFacetsHourlyInteriorConditions( $: ColumnAccessor, opts: TracesDurationStatsOpts, ) { - return [ - $.OrgId.eq(param.string("orgId")), - ...interiorConditions($.Hour), - ...traceFacetDimensionConditions($, opts), - ] -} - -/** The raw tier, plus the hourly interior when the rollup can answer it. */ -function traceFacetTiers>( - opts: TracesFacetsOpts, - raw: CHQuery, - hourly: () => CHQuery, -): CHUnionQuery { - return canUseTraceFacetsRollup(opts) ? unionAll(raw, hourly()) : unionAll(raw) + return [$.OrgId.eq(orgIdParam), ...interiorConditions($.Hour), ...traceFacetDimensionConditions($, opts)] } export interface TracesDurationStatsOutput { @@ -705,7 +700,10 @@ export function tracesDurationStatsQuery( .where(($) => traceFacetsHourlyInteriorConditions($, opts)) const quantiles = "quantilesTDigestMerge(0.5, 0.95)(durationQuantiles)" - return fromUnion(traceFacetTiers(opts, raw, hourly), "duration_tiers") + return fromUnion( + canUseTraceFacetsRollup(opts) ? unionAll(raw, hourly()) : unionAll(raw), + "duration_tiers", + ) .select(() => ({ minDurationMs: CH.rawExpr("minIf(durationMin, traceCount > 0) / 1000000", T.float64), maxDurationMs: CH.rawExpr("maxIf(durationMax, traceCount > 0) / 1000000", T.float64), @@ -741,7 +739,8 @@ export type TracesFacetsOutput = FacetOutput /** The String-typed columns of a table — a facet name is one of those, and a * facet over a `UInt64` would be a number in a field the wire calls a string. */ type StringColumn = { - [K in keyof Cols & string]: Cols[K] extends T.CHString ? K : never + // Both ways, so a branded id (`OrgId`) is not a plain string column. + [K in keyof Cols & string]: Cols[K] extends T.CHString ? (T.CHString extends Cols[K] ? K : never) : never }[keyof Cols & string] export function tracesFacetsQuery(opts: TracesFacetsOpts): CHUnionQuery { @@ -765,7 +764,7 @@ export function tracesFacetsQuery(opts: TracesFacetsOpts): CHUnionQuery [ CH.dynamicColumn("t_attr.TraceId").eq(CH.outerRef("TraceId")), // Typed so the compiler sees the subquery's tenant filter. - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), CH.dynamicColumn("t_attr.Timestamp").gte( param.dateTimeString("startTime"), ), @@ -791,7 +790,7 @@ export function tracesFacetsQuery(opts: TracesFacetsOpts): CHUnionQuery [ CH.dynamicColumn("t_res.TraceId").eq(CH.outerRef("TraceId")), // Typed so the compiler sees the subquery's tenant filter. - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), CH.dynamicColumn("t_res.Timestamp").gte( param.dateTimeString("startTime"), ), @@ -826,7 +825,10 @@ export function tracesFacetsQuery(opts: TracesFacetsOpts): CHUnionQuery $[colName].neq("")), ]) .groupBy("name") - return fromUnion(traceFacetTiers(opts, raw, hourly), `${facetType}_tiers`) + return fromUnion( + canUseTraceFacetsRollup(opts) ? unionAll(raw, hourly()) : unionAll(raw), + `${facetType}_tiers`, + ) .select(($) => ({ name: $.name, count: CH.sum($.count), @@ -845,7 +847,10 @@ export function tracesFacetsQuery(opts: TracesFacetsOpts): CHUnionQuery ({ count: CH.sum($.TraceCount) })) .where(($) => [...traceFacetsHourlyInteriorConditions($, opts), $.HasError.eq(1)]) - return fromUnion(traceFacetTiers(opts, raw, hourly), "errorCount_tiers").select(($) => ({ + return fromUnion( + canUseTraceFacetsRollup(opts) ? unionAll(raw, hourly()) : unionAll(raw), + "errorCount_tiers", + ).select(($) => ({ name: CH.lit("error"), count: CH.sum($.count), facetType: CH.lit("errorCount"), @@ -890,7 +895,7 @@ export function errorsFacetsQuery(opts: ErrorsFacetsOpts): CHUnionQuery ($: ColumnAccessor): Array => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), CH.whenTrue(!!opts.rootOnly, () => $.ParentSpanId.eq("")), @@ -984,7 +989,7 @@ export function errorsSummaryQuery(opts: ErrorsSummaryOpts) { affectedTracesCount: CH.uniq($.TraceId), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), CH.whenTrue(!!opts.rootOnly, () => $.ParentSpanId.eq("")), @@ -1015,7 +1020,7 @@ export function errorsSummaryQuery(opts: ErrorsSummaryOpts) { totalSpans: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), opts.services?.length ? CH.inList($.ServiceName, opts.services) : undefined, @@ -1032,7 +1037,7 @@ export function errorsSummaryQuery(opts: ErrorsSummaryOpts) { totalSpans: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), opts.services?.length ? CH.inList($.ServiceName, opts.services) : undefined, @@ -1049,7 +1054,7 @@ export function errorsSummaryQuery(opts: ErrorsSummaryOpts) { bucketSpans: CH.sum($.TraceCount), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), ...interiorConditions($.Hour), opts.services?.length ? CH.inList($.ServiceName, opts.services) : undefined, ]) @@ -1058,7 +1063,7 @@ export function errorsSummaryQuery(opts: ErrorsSummaryOpts) { bucketSpans: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), edgeCondition("Timestamp"), @@ -1112,7 +1117,7 @@ export function errorIssuesQuery(opts: ErrorIssuesOpts) { lastSeen: CH.max_($.Timestamp), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), opts.services?.length ? CH.inList($.ServiceName, opts.services) : undefined, @@ -1150,7 +1155,7 @@ export function errorTickIssuesQuery() { lastSeen: CH.max_($.LastSeen), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Minute.gte(param.dateTimeSeconds("startTime")), $.Minute.lt(param.dateTimeSeconds("endTime")), ]) @@ -1180,7 +1185,7 @@ export function errorTickBootstrapIssuesQuery() { lastSeen: CH.max_($.Timestamp), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lt(param.dateTimeSeconds("endTime")), ]) @@ -1213,7 +1218,7 @@ export function errorFingerprintsQuery(opts: ErrorFingerprintsOpts) { fingerprintHash: CH.toString_($.FingerprintHash), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), opts.services?.length ? CH.inList($.ServiceName, opts.services) : undefined, @@ -1238,7 +1243,7 @@ export function errorIssueTimeseriesQuery() { count: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.FingerprintHash.eq(CH.toUInt64(param.string("fingerprintHash"))), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), @@ -1272,7 +1277,7 @@ export function errorIssueSampleTracesQuery(opts: { limit?: number }) { durationMicros: CH.intDiv($.Duration, 1000), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.FingerprintHash.eq(CH.toUInt64(param.string("fingerprintHash"))), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), @@ -1310,7 +1315,7 @@ export function errorIssueVersionsSinceQuery(opts: { limit?: number } = {}) { count: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.FingerprintHash.eq(CH.toUInt64(param.string("fingerprintHash"))), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), @@ -1332,7 +1337,7 @@ export function errorIssueEnvironmentsQuery(opts: { limit?: number } = {}) { count: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.FingerprintHash.eq(CH.toUInt64(param.string("fingerprintHash"))), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), @@ -1399,7 +1404,7 @@ export function errorDetailTracesQuery(opts: ErrorDetailTracesOpts) { occurrenceExceptionMessage: CH.argMax($.ExceptionMessage, $.Timestamp), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), fingerprintHashEq($.FingerprintHash, opts.fingerprintHash), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), @@ -1450,7 +1455,7 @@ export function errorDetailTracesQuery(opts: ErrorDetailTracesOpts) { } }) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), inSubquery( $.TraceId, fromQuery(occurrences, "matching_traces").select(($$) => ({ TraceId: $$.TraceId })), @@ -1486,7 +1491,7 @@ export function errorFingerprintSummaryQuery(opts: ErrorFingerprintSummaryOpts) noExceptionCount: CH.countIf($.ExceptionType.eq("")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), fingerprintHashEq($.FingerprintHash, opts.fingerprintHash), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), @@ -1515,7 +1520,7 @@ export function errorCooccurringFingerprintsQuery(opts: ErrorCooccurringFingerpr count: CH.count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), opts.traceIds.length ? CH.inList($.TraceId, opts.traceIds) : CH.rawCond("1 = 0"), @@ -1538,7 +1543,7 @@ export function errorFingerprintOccurrencesQuery(opts: { fingerprintHashes: read spanId: CH.argMax($.SpanId, $.Timestamp), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), fingerprintHashIn($.FingerprintHash, opts.fingerprintHashes), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), @@ -1560,7 +1565,7 @@ export function errorOccurrenceSpansQuery(opts: { traceIds: readonly string[]; s ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), opts.traceIds.length ? CH.inList($.TraceId, opts.traceIds) : CH.rawCond("1 = 0"), opts.spanIds.length ? CH.inList($.SpanId, opts.spanIds) : CH.rawCond("1 = 0"), $.Timestamp.gte(param.dateTimeString("startTime")), @@ -1582,7 +1587,7 @@ export function errorsWindowTotalsQuery(opts: Omit [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), CH.whenTrue(!!opts.rootOnly, () => $.ParentSpanId.eq("")), diff --git a/packages/query-engine/src/ch/queries/format.ts b/packages/query-engine/src/ch/queries/format.ts index 12d8d73a66..4cd33fe894 100644 --- a/packages/query-engine/src/ch/queries/format.ts +++ b/packages/query-engine/src/ch/queries/format.ts @@ -4,9 +4,9 @@ // PlanetScale, usage and alert-check queries format buckets the same way, and // the format string has to match what the frontend parses. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { defineFn, param } from "@maple-dev/effect-clickhouse" -import * as T from "@maple-dev/effect-clickhouse/types" +import * as CH from "@maple-dev/effect-orm/expr" +import { defineFn, param } from "@maple-dev/effect-orm/clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" /** * ISO-8601 with a literal `Z`. ClickHouse `DateTime` has no zone, so the suffix diff --git a/packages/query-engine/src/ch/queries/infra.test.ts b/packages/query-engine/src/ch/queries/infra.test.ts index 1bcb52efdf..9d1c5f29a7 100644 --- a/packages/query-engine/src/ch/queries/infra.test.ts +++ b/packages/query-engine/src/ch/queries/infra.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" -import { compileUnionUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" +import { compileUnionUnsafe } from "@maple-dev/effect-orm/clickhouse" import { listHostsQuery, hostDetailSummaryQuery, @@ -20,9 +20,10 @@ import { workloadFacetsQuery, infraPresenceQuery, } from "./infra" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", bucketSeconds: 60, diff --git a/packages/query-engine/src/ch/queries/infra.ts b/packages/query-engine/src/ch/queries/infra.ts index 78260bbbe7..d490494ccd 100644 --- a/packages/query-engine/src/ch/queries/infra.ts +++ b/packages/query-engine/src/ch/queries/infra.ts @@ -11,11 +11,11 @@ // // Host identity is carried on the ResourceAttributes map under `host.name`. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import { from, fromQuery, type ColumnAccessor } from "@maple-dev/effect-clickhouse" -import { unionAll, type CHUnionQuery } from "@maple-dev/effect-clickhouse" -import { MetricsGauge, MetricsSum } from "../tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { from, fromQuery, type ColumnAccessor } from "@maple-dev/effect-orm/clickhouse" +import { unionAll, type CHUnionQuery } from "@maple-dev/effect-orm/clickhouse" +import { MetricsGauge, MetricsSum, orgIdParam } from "../tables" import { containerRuntimeExpr, deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import { avgIfOrZero, facetAttrExpr, maxIfOrZero, type FacetOutput } from "./query-helpers" @@ -69,7 +69,7 @@ export function listHostsQuery(opts: ListHostsOpts = {}) { load15: avgIfOrZero($.Value, $.MetricName.eq("system.cpu.load_average.15m")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("host.name").neq(""), @@ -130,7 +130,7 @@ export function hostDetailSummaryQuery(opts: HostDetailSummaryOpts) { load15: avgIfOrZero($.Value, $.MetricName.eq("system.cpu.load_average.15m")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("host.name").eq(opts.hostName), @@ -165,7 +165,7 @@ export function hostGaugeTimeseriesQuery(opts: HostGaugeTimeseriesOpts) { avgValue: CH.avg($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("host.name").eq(opts.hostName), @@ -219,7 +219,7 @@ export function fleetUtilizationTimeseriesQuery() { activeHosts: CH.uniq($.ResourceAttributes.get("host.name")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("host.name").neq(""), @@ -238,7 +238,7 @@ export function hostNetworkTimeseriesQuery(opts: HostNetworkTimeseriesOpts) { sumValue: CH.sum($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("host.name").eq(opts.hostName), @@ -396,7 +396,7 @@ const podBaseConditions = ( $: ColumnAccessor, metricNames: ReadonlyArray = POD_METRIC_NAMES, ): Array => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("k8s.pod.name").neq(""), @@ -700,7 +700,7 @@ export function podDetailSummaryQuery(opts: PodDetailSummaryOpts) { memoryRequestPct: avgIfOrZero($.Value, $.MetricName.eq("k8s.pod.memory_request_utilization")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("k8s.pod.name").eq(opts.podName), @@ -731,7 +731,7 @@ export function podGaugeTimeseriesQuery(opts: PodGaugeTimeseriesOpts) { avgValue: CH.avg($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("k8s.pod.name").eq(opts.podName), @@ -778,7 +778,7 @@ const nodeBaseConditions = ( $: ColumnAccessor, metricNames: ReadonlyArray = NODE_METRIC_NAMES, ): Array => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("k8s.node.name").neq(""), @@ -850,7 +850,7 @@ export function nodeDetailSummaryQuery(opts: NodeDetailSummaryOpts) { uptime: maxIfOrZero($.Value, $.MetricName.eq("k8s.node.uptime")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("k8s.node.name").eq(opts.nodeName), @@ -874,7 +874,7 @@ export function nodeGaugeTimeseriesQuery(opts: NodeGaugeTimeseriesOpts) { avgValue: CH.avg($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get("k8s.node.name").eq(opts.nodeName), @@ -953,7 +953,7 @@ export function listWorkloadsQuery(opts: ListWorkloadsOpts) { avgCpuUsage: avgIfOrZero($.Value, $.MetricName.eq("k8s.pod.cpu.usage")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get(attrKey).neq(""), @@ -999,7 +999,7 @@ export function workloadDetailSummaryQuery(opts: WorkloadDetailSummaryOpts) { avgCpuUsage: avgIfOrZero($.Value, $.MetricName.eq("k8s.pod.cpu.usage")), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get(attrKey).eq(opts.workloadName), @@ -1027,7 +1027,7 @@ export function workloadGaugeTimeseriesQuery(opts: WorkloadGaugeTimeseriesOpts) avgValue: CH.avg($.Value), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get(attrKey).eq(opts.workloadName), @@ -1121,7 +1121,7 @@ const makeWorkloadFacet = ( facetType: CH.lit(facetType), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.ResourceAttributes.get(ownerKey).neq(""), @@ -1189,7 +1189,7 @@ const presenceBranch = ( from(MetricsGauge) .select(() => ({ surface: CH.lit(surface) })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), $.MetricName.eq(metricName), diff --git a/packages/query-engine/src/ch/queries/liveness.test.ts b/packages/query-engine/src/ch/queries/liveness.test.ts index 4886b2f72a..4205de3bef 100644 --- a/packages/query-engine/src/ch/queries/liveness.test.ts +++ b/packages/query-engine/src/ch/queries/liveness.test.ts @@ -1,16 +1,17 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" -import { compileUnsafe, compileUnionUnsafe, type CompiledQuery } from "@maple-dev/effect-clickhouse" +import { compileUnsafe, compileUnionUnsafe, type CompiledQuery } from "@maple-dev/effect-orm/clickhouse" import { orgTelemetryPulseQuery, serviceLivenessQuery } from "./liveness" +import { OrgId } from "@maple/domain" const pulseParams = { - orgId: "org_123", + orgId: OrgId.make("org_123"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-01 00:10:00", } const livenessParams = { - orgId: "org_123", + orgId: OrgId.make("org_123"), serviceName: "checkout", startTime: "2024-01-01 00:00:00", endTime: "2024-01-01 00:30:00", diff --git a/packages/query-engine/src/ch/queries/liveness.ts b/packages/query-engine/src/ch/queries/liveness.ts index 192af0f1aa..83e837919d 100644 --- a/packages/query-engine/src/ch/queries/liveness.ts +++ b/packages/query-engine/src/ch/queries/liveness.ts @@ -26,16 +26,16 @@ // arithmetic over strings. import { Schema } from "effect" -import * as CH from "@maple-dev/effect-clickhouse/expr" +import * as CH from "@maple-dev/effect-orm/expr" import { from, param, unionAll, type CHUnionQuery, type CompiledQueryRowSchema, -} from "@maple-dev/effect-clickhouse" +} from "@maple-dev/effect-orm/clickhouse" import { CHNumber } from "../schema" -import { Logs, MetricCatalog, ServiceOperationsMinutely, ServiceOverviewSpans } from "../tables" +import { Logs, MetricCatalog, ServiceOperationsMinutely, ServiceOverviewSpans, orgIdParam } from "../tables" import { hourFloor } from "./query-helpers" export interface ServiceLivenessOutput { @@ -73,11 +73,11 @@ export function serviceLivenessQuery(opts: ServiceLivenessOpts = {}) { lastSeen: CH.toString_(CH.max_($.Minute)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.Minute.gte(param.dateTimeSeconds("startTime")), $.Minute.lte(param.dateTimeSeconds("endTime")), - opts.scopeToEnvironment ? $.DeploymentEnv.eq(param.string("deploymentEnv")) : undefined, + CH.whenTrue(!!opts.scopeToEnvironment, () => $.DeploymentEnv.eq(param.string("deploymentEnv"))), ]) .format("JSON") } @@ -114,7 +114,7 @@ export function orgTelemetryPulseQuery(): CHUnionQuery { lastSeen: CH.toString_(CH.max_($.Timestamp)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), ]) @@ -126,7 +126,7 @@ export function orgTelemetryPulseQuery(): CHUnionQuery { lastSeen: CH.toString_(CH.max_($.Timestamp)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimestampTime.gte(param.dateTimeSeconds("startTime")), $.TimestampTime.lte(param.dateTimeSeconds("endTime")), $.Timestamp.gte(param.dateTimeString("startTime")), @@ -151,7 +151,7 @@ export function ingestFreshnessQuery(): CHUnionQuery { lastSeen: CH.toString_(CH.max_($.Minute)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Minute.gte(param.dateTimeSeconds("startTime")), $.Minute.lte(param.dateTimeSeconds("endTime")), ]) @@ -173,7 +173,7 @@ export function ingestFreshnessQuery(): CHUnionQuery { ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(hourFloor("startTime")), $.Hour.lte(hourFloor("endTime")), $.LastSeen.gte(param.dateTimeSeconds("startTime")), @@ -192,7 +192,7 @@ export function logsFreshnessQuery() { lastSeen: CH.toString_(CH.toDateTime(CH.max_($.Timestamp))), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimestampTime.gte(param.dateTimeSeconds("startTime")), $.TimestampTime.lte(param.dateTimeSeconds("endTime")), $.Timestamp.gte(param.dateTimeString("startTime")), diff --git a/packages/query-engine/src/ch/queries/logs.test.ts b/packages/query-engine/src/ch/queries/logs.test.ts index e1da0bc11a..b951df0d11 100644 --- a/packages/query-engine/src/ch/queries/logs.test.ts +++ b/packages/query-engine/src/ch/queries/logs.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" -import { compile, compileUnsafe, compileUnionUnsafe } from "@maple-dev/effect-clickhouse" +import { compile, compileUnsafe, compileUnionUnsafe } from "@maple-dev/effect-orm/clickhouse" import { canUseLogsAggregatesHourly, logsTimeseriesQuery, @@ -11,9 +11,10 @@ import { errorRateByServiceQuery, logsFacetsQuery, } from "./logs" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", bucketSeconds: 3600, @@ -401,7 +402,7 @@ describe("logsBreakdownQuery", () => { expect(build).not.toThrow() const error = Effect.runSync(Effect.flip(compile(build(), baseParams))) - expect(error._tag).toBe("@maple-dev/effect-clickhouse/QueryBuilderError") + expect(error._tag).toBe("@maple-dev/effect-orm/QueryBuilderError") expect(error.message).toContain("Timestamp") }) diff --git a/packages/query-engine/src/ch/queries/logs.ts b/packages/query-engine/src/ch/queries/logs.ts index d740b49dc0..3826348297 100644 --- a/packages/query-engine/src/ch/queries/logs.ts +++ b/packages/query-engine/src/ch/queries/logs.ts @@ -3,14 +3,14 @@ // DSL-based query definitions for logs timeseries and breakdown. import { finiteOrZero } from "./format" -import { compileFnCall, subqueryExpr } from "@maple-dev/effect-clickhouse" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import { from, fromUnion, type CHQuery, type ColumnAccessor } from "@maple-dev/effect-clickhouse" -import type { ColumnDefs } from "@maple-dev/effect-clickhouse/types" -import * as T from "@maple-dev/effect-clickhouse/types" -import { unionAll, type CHUnionQuery } from "@maple-dev/effect-clickhouse" -import { Logs, LogsAggregatesHourly } from "../tables" +import { compileFnCall, subqueryExpr } from "@maple-dev/effect-orm/clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { from, fromUnion, type CHQuery, type ColumnAccessor } from "@maple-dev/effect-orm/clickhouse" +import type { ColumnDefs } from "@maple-dev/effect-orm/clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" +import { unionAll, type CHUnionQuery } from "@maple-dev/effect-orm/clickhouse" +import { Logs, LogsAggregatesHourly, orgIdParam } from "../tables" import { finalizeTimeseries } from "./series-cap" import type { AttributeFilter } from "@maple/domain/query-engine" import { deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" @@ -346,7 +346,7 @@ export function logsTimeseriesQuery(opts: LogsTimeseriesOpts): CHQuery [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(param.dateTimeSeconds("startTime")), // `param.dateTimeString("endTime")` substitutes as a quoted string literal; // `toStartOfHour` only accepts Date/DateTime, so wrap with `toDateTime`. @@ -369,7 +369,7 @@ export function logsTimeseriesQuery(opts: LogsTimeseriesOpts): CHQuery [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), // TimestampTime is the partition/index key; this filter unlocks // partition pruning. Timestamp filter retained for sub-second accuracy. $.TimestampTime.gte(param.dateTimeSeconds("startTime")), @@ -444,7 +444,7 @@ export function logsBreakdownQuery(opts: LogsBreakdownOpts): CHQuery [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), ...rawLogsTimeRange($), ...serviceSeverityConditions($, opts), opts.minSeverity !== undefined ? $.SeverityNumber.gte(opts.minSeverity) : undefined, @@ -466,7 +466,7 @@ export function logsBreakdownQuery(opts: LogsBreakdownOpts): CHQuery [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), ...rawLogsTimeRange($), rawLogEdgeCondition(), ...serviceSeverityConditions($, opts), @@ -484,11 +484,7 @@ export function logsBreakdownQuery(opts: LogsBreakdownOpts): CHQuery [ - $.OrgId.eq(param.string("orgId")), - ...interiorConditions($.Hour), - ...mvFacetConditions($, opts), - ]) + .where(($) => [$.OrgId.eq(orgIdParam), ...interiorConditions($.Hour), ...mvFacetConditions($, opts)]) .groupBy("name") const combined = fromUnion(unionAll(rawEdges, mvInterior), "breakdown") @@ -516,7 +512,7 @@ export function logsCountQuery(opts: LogsQueryOpts): CHQuery [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), ...rawLogsTimeRange($), ...serviceSeverityConditions($, opts), opts.minSeverity !== undefined ? $.SeverityNumber.gte(opts.minSeverity) : undefined, @@ -535,7 +531,7 @@ export function logsCountQuery(opts: LogsQueryOpts): CHQuery [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), ...rawLogsTimeRange($), rawLogEdgeCondition(), ...serviceSeverityConditions($, opts), @@ -546,11 +542,7 @@ export function logsCountQuery(opts: LogsQueryOpts): CHQuery ({ total: CH.sum($.Count), })) - .where(($) => [ - $.OrgId.eq(param.string("orgId")), - ...interiorConditions($.Hour), - ...mvFacetConditions($, opts), - ]) + .where(($) => [$.OrgId.eq(orgIdParam), ...interiorConditions($.Hour), ...mvFacetConditions($, opts)]) const combined = fromUnion(unionAll(rawEdges, mvInterior), "counts") .select(($) => ({ @@ -608,7 +600,7 @@ export function logsListQuery(opts: LogsListOpts) { const offset = opts.offset ?? 0 const baseWhere = ($: ColumnAccessor): Array => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimestampTime.gte(param.dateTimeSeconds("startTime")), $.TimestampTime.lte(param.dateTimeSeconds("endTime")), $.Timestamp.gte(param.dateTimeString("startTime")), @@ -710,7 +702,7 @@ export function getLogByKeyQuery(opts: LogByKeyOpts) { resourceAttributes: CH.toJSONString($.ResourceAttributes), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), // TimestampTime is the partition/index key; bounding it unlocks // partition pruning. Timestamp.eq pins the exact sub-second row. $.TimestampTime.gte(param.dateTimeSeconds("startTime")), @@ -742,7 +734,7 @@ export function errorRateByServiceQuery() { bucketErrorLogs: CH.countIf(CH.inList($.SeverityText, ["ERROR", "FATAL"])), errorRate: CH.lit(0), })) - .where(($) => [$.OrgId.eq(param.string("orgId")), ...rawLogsTimeRange($), rawLogEdgeCondition()]) + .where(($) => [$.OrgId.eq(orgIdParam), ...rawLogsTimeRange($), rawLogEdgeCondition()]) .groupBy("serviceName") const mvInterior = from(LogsAggregatesHourly) @@ -752,7 +744,7 @@ export function errorRateByServiceQuery() { bucketErrorLogs: CH.sumIf($.Count, CH.inList($.SeverityText, ["ERROR", "FATAL"])), errorRate: CH.lit(0), })) - .where(($) => [$.OrgId.eq(param.string("orgId")), ...interiorConditions($.Hour)]) + .where(($) => [$.OrgId.eq(orgIdParam), ...interiorConditions($.Hour)]) .groupBy("serviceName") return fromUnion(unionAll(rawEdges, mvInterior), "rates") @@ -803,7 +795,7 @@ function logsFacetsQueryFromMv( const baseWhere = ( $: ColumnAccessor, ): Array => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(param.dateTimeSeconds("startTime")), $.Hour.lte(param.dateTimeSeconds("endTime")), CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), @@ -866,11 +858,10 @@ function logsFacetsQueryFromMv( deploymentEnv: envQuery, namespace: namespaceQuery, } - const branches = facet ? [byFacet[facet]] : [severityQuery, serviceQuery, envQuery, namespaceQuery] - return unionAll(...branches) - .orderBy(["count", "desc"]) - .limit(500) - .format("JSON") + const union = facet + ? unionAll(byFacet[facet]) + : unionAll(severityQuery, serviceQuery, envQuery, namespaceQuery) + return union.orderBy(["count", "desc"]).limit(500).format("JSON") } function logsFacetsQueryFromRaw( @@ -878,7 +869,7 @@ function logsFacetsQueryFromRaw( facet?: LogsFacetDimension, ): CHUnionQuery { const baseWhere = ($: ColumnAccessor): Array => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimestampTime.gte(param.dateTimeSeconds("startTime")), $.TimestampTime.lte(param.dateTimeSeconds("endTime")), $.Timestamp.gte(param.dateTimeString("startTime")), @@ -943,9 +934,8 @@ function logsFacetsQueryFromRaw( deploymentEnv: envQuery, namespace: namespaceQuery, } - const branches = facet ? [byFacet[facet]] : [severityQuery, serviceQuery, envQuery, namespaceQuery] - return unionAll(...branches) - .orderBy(["count", "desc"]) - .limit(500) - .format("JSON") + const union = facet + ? unionAll(byFacet[facet]) + : unionAll(severityQuery, serviceQuery, envQuery, namespaceQuery) + return union.orderBy(["count", "desc"]).limit(500).format("JSON") } diff --git a/packages/query-engine/src/ch/queries/metrics.test.ts b/packages/query-engine/src/ch/queries/metrics.test.ts index 576bd7b8b2..3a688abdd5 100644 --- a/packages/query-engine/src/ch/queries/metrics.test.ts +++ b/packages/query-engine/src/ch/queries/metrics.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { metricsTimeseriesQuery, metricsTimeseriesRateQuery, @@ -8,9 +8,10 @@ import { listMetricsQuery, metricsSummaryQuery, } from "./metrics" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", bucketSeconds: 3600, diff --git a/packages/query-engine/src/ch/queries/metrics.ts b/packages/query-engine/src/ch/queries/metrics.ts index 37a8a9eb42..b7d140d957 100644 --- a/packages/query-engine/src/ch/queries/metrics.ts +++ b/packages/query-engine/src/ch/queries/metrics.ts @@ -5,12 +5,12 @@ import { finiteOrZero } from "./format" import type { AttributeFilter, MetricType } from "@maple/domain/query-engine" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import * as T from "@maple-dev/effect-clickhouse/types" -import { param } from "@maple-dev/effect-clickhouse" -import { from, type CHQuery } from "@maple-dev/effect-clickhouse" -import { table } from "@maple-dev/effect-clickhouse" -import { MetricsSum, MetricsGauge, MetricCatalog, SpanMetricsCallsHourly } from "../tables" +import * as CH from "@maple-dev/effect-orm/expr" +import * as T from "@maple-dev/effect-orm/clickhouse" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { from, type CHQuery } from "@maple-dev/effect-orm/clickhouse" +import { table } from "@maple-dev/effect-orm/clickhouse" +import { MetricsSum, MetricsGauge, MetricCatalog, SpanMetricsCallsHourly, orgIdParam } from "../tables" import { resolveMetricTable, metricsSelectExprs } from "./query-helpers" import { deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import { buildAttrFilterCondition } from "../../traces-shared" @@ -102,7 +102,7 @@ export function metricsTimeseriesQuery(opts: MetricsTimeseriesOpts) { })) .where(($) => [ $.MetricName.eq(param.string("metricName")), - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), @@ -224,7 +224,7 @@ function metricsTimeseriesRateFromSpanMetricsCallsHourly( Value: CH.argMaxMerge($.LastValue), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.MetricName.eq(param.string("metricName")), $.Hour.gte(previousBucket), $.Hour.lte(endBucket), @@ -244,14 +244,17 @@ function metricsTimeseriesRateFromSpanMetricsCallsHourly( ) const hourlyValues = table("hourly_values", { - Hour: T.dateTimeString, - ServiceName: T.string, - MetricName: T.string, - SpanKind: T.string, - AttrFingerprint: T.uint64, - ResourceFingerprint: T.uint64, - StartTimeUnix: T.dateTime64String, - Value: T.float64, + external: true, + columns: { + Hour: T.dateTimeString, + ServiceName: T.string, + MetricName: T.string, + SpanKind: T.string, + AttrFingerprint: T.uint64, + ResourceFingerprint: T.uint64, + StartTimeUnix: T.dateTime64String, + Value: T.float64, + }, }) const deltasQuery = from(hourlyValues) @@ -279,10 +282,13 @@ function metricsTimeseriesRateFromSpanMetricsCallsHourly( .where(($) => [$.Hour.gte(bucket)]) const deltas = table("with_deltas", { - Hour: T.dateTimeString, - ServiceName: T.string, - SpanKind: T.string, - delta: T.float64, + external: true, + columns: { + Hour: T.dateTimeString, + ServiceName: T.string, + SpanKind: T.string, + delta: T.float64, + }, }) const q = from(deltas) @@ -389,7 +395,7 @@ export function metricsTimeseriesRateQuery( opts.metricNames && opts.metricNames.length > 0 ? $.MetricName.in_(...opts.metricNames) : $.MetricName.eq(param.string("metricName")), - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte( CH.intervalSub( param.dateTimeString("startTime"), @@ -409,11 +415,14 @@ export function metricsTimeseriesRateQuery( // Outer query: aggregate deltas into rate/increase per bucket const cteTable = table("with_deltas", { - TimeUnix: T.dateTime64String, - ServiceName: T.string, - Attributes: T.map(T.string, T.string), - resourceAttributeValue: T.string, - delta: T.float64, + external: true, + columns: { + TimeUnix: T.dateTime64String, + ServiceName: T.string, + Attributes: T.map(T.string, T.string), + resourceAttributeValue: T.string, + delta: T.float64, + }, }) const q = from(cteTable) @@ -496,7 +505,7 @@ export function metricsSparklinesQuery(opts: MetricsSparklinesOpts) { }) .where(($) => [ $.MetricName.in_(...opts.metricNames), - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), ]) @@ -565,7 +574,7 @@ export function metricsBreakdownQuery(opts: MetricsBreakdownOpts) { }) .where(($) => [ $.MetricName.eq(param.string("metricName")), - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), // Drop datapoints missing the label so an empty bucket doesn't dominate. @@ -622,7 +631,7 @@ export function listMetricsQuery(opts: ListMetricsOpts) { isMonotonic: CH.any_($.IsMonotonic), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), // Floor the start bound to the hour so the oldest catalog bucket // (Hour is already hour-truncated) isn't dropped for mid-hour ranges. $.Hour.gte(CH.toStartOfInterval(CH.toDateTime(param.dateTimeString("startTime")), 3600)), @@ -658,7 +667,7 @@ export function metricsSummaryQuery(opts?: MetricsSummaryOpts) { dataPointCount: CH.sum($.DataPointCount), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(CH.toStartOfInterval(CH.toDateTime(param.dateTimeString("startTime")), 3600)), $.Hour.lte(param.dateTimeSeconds("endTime")), CH.when(opts?.serviceName, (v: string) => $.ServiceName.eq(v)), diff --git a/packages/query-engine/src/ch/queries/product-events-explore.test.ts b/packages/query-engine/src/ch/queries/product-events-explore.test.ts index 30e87b4693..1e9e772e1f 100644 --- a/packages/query-engine/src/ch/queries/product-events-explore.test.ts +++ b/packages/query-engine/src/ch/queries/product-events-explore.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { productEventAttributeKeysQuery, productEventAttributeValuesQuery, @@ -7,9 +7,10 @@ import { productEventsListQuery, productEventsTimeseriesQuery, } from "./product-events-explore" +import { OrgId } from "@maple/domain" const params = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2026-06-24 04:00:00", endTime: "2026-06-25 06:00:00", bucketSeconds: 3600, diff --git a/packages/query-engine/src/ch/queries/product-events-explore.ts b/packages/query-engine/src/ch/queries/product-events-explore.ts index 2429895794..730ecf8695 100644 --- a/packages/query-engine/src/ch/queries/product-events-explore.ts +++ b/packages/query-engine/src/ch/queries/product-events-explore.ts @@ -4,12 +4,12 @@ // side of the same table, and it keeps the `/analytics` session filters by // reusing `replaysWhere` so the two agree on what a filtered population is. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param, from, inSubquery } from "@maple-dev/effect-clickhouse" -import type { CHQuery, ColumnAccessor, ColumnDefs } from "@maple-dev/effect-clickhouse" -import * as T from "@maple-dev/effect-clickhouse/types" +import * as CH from "@maple-dev/effect-orm/expr" +import { param, from, inSubquery } from "@maple-dev/effect-orm/clickhouse" +import type { CHQuery, ColumnAccessor, ColumnDefs } from "@maple-dev/effect-orm/clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" import type { AttributeFilter, ProductEventsGroupBy, ProductEventsMetric } from "@maple/domain/query-engine" -import { ProductEvents, SessionReplays } from "../tables" +import { ProductEvents, SessionReplays, orgIdParam } from "../tables" import { buildAttrFilterCondition } from "../../traces-shared" import { finalizeTimeseries } from "./series-cap" import { soleValue } from "./query-helpers" @@ -136,7 +136,7 @@ function sessionSemiJoin($: EventsAccessor, opts: ProductEventsQueryOpts): CH.Co function eventConditions($: EventsAccessor, opts: ProductEventsQueryOpts): Array { return [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), inListOpt($.EventName, opts.eventNames), diff --git a/packages/query-engine/src/ch/queries/product-events-paths.test.ts b/packages/query-engine/src/ch/queries/product-events-paths.test.ts index 1c43aad5b5..5a88952e04 100644 --- a/packages/query-engine/src/ch/queries/product-events-paths.test.ts +++ b/packages/query-engine/src/ch/queries/product-events-paths.test.ts @@ -1,9 +1,14 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { ProductEventsFunnelError } from "./product-events" import { productEventsPathsQuery, type ProductEventsPathsOpts } from "./product-events-paths" +import { OrgId } from "@maple/domain" -const params = { orgId: "org_1", startTime: "2026-06-24 04:00:00", endTime: "2026-06-25 06:00:00" } +const params = { + orgId: OrgId.make("org_1"), + startTime: "2026-06-24 04:00:00", + endTime: "2026-06-25 06:00:00", +} const base: ProductEventsPathsOpts = { anchor: { kind: "event", eventName: "signup_completed" }, diff --git a/packages/query-engine/src/ch/queries/product-events-paths.ts b/packages/query-engine/src/ch/queries/product-events-paths.ts index 688a738530..116b0e62bf 100644 --- a/packages/query-engine/src/ch/queries/product-events-paths.ts +++ b/packages/query-engine/src/ch/queries/product-events-paths.ts @@ -17,12 +17,12 @@ // user-supplied value (the anchor, the excluded names) enters through a typed // condition, never a string. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param, from, fromQuery, inSubquery, table } from "@maple-dev/effect-clickhouse" -import type { CHQuery } from "@maple-dev/effect-clickhouse" -import * as T from "@maple-dev/effect-clickhouse/types" +import * as CH from "@maple-dev/effect-orm/expr" +import { param, from, fromQuery, inSubquery, table } from "@maple-dev/effect-orm/clickhouse" +import type { CHQuery } from "@maple-dev/effect-orm/clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" import { Schema } from "effect" -import { ProductEvents } from "../tables" +import { ProductEvents, orgIdParam } from "../tables" import { CHNumber } from "../schema" import { LINK_ALIAS, @@ -147,7 +147,7 @@ function pathEventsBranch(opts: ProductEventsPathsOpts, filters: ProductEventsFi const keyOf = ($: OpenJoinAccessor) => personKey(keyBy, $, keyBy === "person" ? $[LINK_ALIAS] : undefined) const inRange = ($: OpenJoinAccessor) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), keyOf($).neq(""), @@ -267,10 +267,13 @@ export function productEventsPathsQuery( // The hop rows are read three times below (ranking, and both sides of the // mapping join); a CTE evaluates the per-person sequencing once. const flat = table("path_hops", { - key: T.string, - hop: T.uint32, - fromNode: T.string, - toNode: T.string, + external: true, + columns: { + key: T.string, + hop: T.uint32, + fromNode: T.string, + toNode: T.string, + }, }) // The named nodes: per column (a `toNode` at `hop`), the `branches` most diff --git a/packages/query-engine/src/ch/queries/product-events.test.ts b/packages/query-engine/src/ch/queries/product-events.test.ts index 4cb5ed2b14..38bda022ae 100644 --- a/packages/query-engine/src/ch/queries/product-events.test.ts +++ b/packages/query-engine/src/ch/queries/product-events.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { productEventsFunnelQuery, productEventsFunnelBreakdownQuery, @@ -9,8 +9,13 @@ import { ProductEventsFunnelError, type FunnelStep, } from "./product-events" +import { OrgId } from "@maple/domain" -const params = { orgId: "org_1", startTime: "2026-06-24 04:00:00", endTime: "2026-06-25 06:00:00" } +const params = { + orgId: OrgId.make("org_1"), + startTime: "2026-06-24 04:00:00", + endTime: "2026-06-25 06:00:00", +} const STEPS: ReadonlyArray = [ { kind: "page", pagePath: "/pricing", host: "maple.dev" }, diff --git a/packages/query-engine/src/ch/queries/product-events.ts b/packages/query-engine/src/ch/queries/product-events.ts index d3782478a0..b199847da1 100644 --- a/packages/query-engine/src/ch/queries/product-events.ts +++ b/packages/query-engine/src/ch/queries/product-events.ts @@ -10,7 +10,7 @@ // The page-view queries themselves live in `web-analytics.ts` and read the // same table (`useProductEvents`); this module owns everything funnel-shaped. -import * as CH from "@maple-dev/effect-clickhouse/expr" +import * as CH from "@maple-dev/effect-orm/expr" import { param, from, @@ -19,13 +19,18 @@ import { unionAll, inSubquery, compileFnCall, -} from "@maple-dev/effect-clickhouse" -import type { CHQuery, ColumnAccessor, ColumnDefs, JoinedColumnAccessor } from "@maple-dev/effect-clickhouse" +} from "@maple-dev/effect-orm/clickhouse" +import type { + CHQuery, + ColumnAccessor, + ColumnDefs, + JoinedColumnAccessor, +} from "@maple-dev/effect-orm/clickhouse" import { Schema } from "effect" -import { ProductEvents, IdentityLinks, SessionReplays } from "../tables" +import { ProductEvents, IdentityLinks, SessionReplays, orgIdParam } from "../tables" import { CHNumber } from "../schema" import { replaysWhere, needsSessionSemiJoin, type ProductEventsFilters } from "./web-analytics" -import * as T from "@maple-dev/effect-clickhouse/types" +import * as T from "@maple-dev/effect-orm/clickhouse" export type { ProductEventsFilters } from "./web-analytics" @@ -36,7 +41,7 @@ export type { ProductEventsFilters } from "./web-analytics" // toUInt8(cond) — a condition as a projectable 0/1 column. export function flag(cond: CH.Condition): CH.Expr { - return CH.compileTypedFnCall("toUInt8", T.uint8.schema, cond) + return CH.compileTypedFnCall("toUInt8", T.uint8.schema, cond) } // toUInt64(toUnixTimestamp64Milli(ts)) — `windowFunnel` accepts Date, DateTime @@ -51,7 +56,7 @@ export function epochMs(ts: CH.Expr): CH.Expr { // argMinIf(value, orderBy, cond) — the `value` on the earliest row matching // `cond`. Returns one of its inputs unchanged, so it decodes as that input does. export function argMinIf(value: CH.Expr, orderBy: CH.Expr, cond: CH.Condition): CH.Expr { - return CH.compileTypedFnCall("argMinIf", CH.schemaOf(value), value, orderBy, cond) + return CH.compileTypedFnCall("argMinIf", CH.schemaOf(value), value, orderBy, cond) } // Every UNION ALL branch below is built from a different table, and the shape @@ -247,7 +252,7 @@ export function identityLinksByVisitor() { UserId: $.UserId, FirstSeen: CH.min_($.FirstSeen), })) - .where(($) => [$.OrgId.eq(param.string("orgId"))]) + .where(($) => [$.OrgId.eq(orgIdParam)]) .groupBy("VisitorId", "UserId") return fromQuery(firstSeenPerPair, "pair_links") @@ -402,7 +407,7 @@ function eventsBranch(plan: FunnelPlan): FunnelBranch { Value: CH.max_(sessionDimensionColumn($, sessionDimension)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.StartTime.gte(param.dateTimeString("startTime")), $.StartTime.lte(param.dateTimeString("endTime")), ]) @@ -450,7 +455,7 @@ function eventsBranch(plan: FunnelPlan): FunnelBranch { undefined, ) return [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), anyStep, @@ -499,7 +504,7 @@ function sessionEntryBranch(plan: FunnelPlan, step: Extract { const key = personKey(keyBy, $, keyBy === "person" ? $[LINK_ALIAS] : undefined) return [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.StartTime.gte(param.dateTimeString("startTime")), $.StartTime.lte(param.dateTimeString("endTime")), sessionDimensionColumn($, step.dimension).eq(step.value), @@ -529,7 +534,7 @@ function levelsQuery(plan: FunnelPlan) { const source = plan.sessionStep ? hasEventStep ? fromUnion( - unionAll(sessionEntryBranch(plan, plan.sessionStep), eventsBranch(plan)), + unionAll(sessionEntryBranch(plan, plan.sessionStep), eventsBranch(plan)), "funnel_events", ) : fromQuery(sessionEntryBranch(plan, plan.sessionStep), "funnel_events") @@ -677,7 +682,7 @@ export function productEventNamesQuery( persons: CH.uniq(CH.if_($.UserId.neq(""), $.UserId, $.VisitorId)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), CH.when(filters.host, (v: string) => $.Host.eq(v)), @@ -743,7 +748,7 @@ export function productEventsForTraceQuery( attributes: $.Attributes, })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), $.TraceId.eq(param.string("traceId")), @@ -784,7 +789,7 @@ export function productEventTraceSamplesQuery( visitorId: $.VisitorId, })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), $.EventName.eq(param.string("eventName")), @@ -825,7 +830,7 @@ function personEventsBranch(plan: FunnelPlan) { .where(($) => { const key = personKey(keyBy, $, keyBy === "person" ? $[LINK_ALIAS] : undefined) return [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), key.neq(""), diff --git a/packages/query-engine/src/ch/queries/query-helpers.ts b/packages/query-engine/src/ch/queries/query-helpers.ts index c043732995..d4a442dd54 100644 --- a/packages/query-engine/src/ch/queries/query-helpers.ts +++ b/packages/query-engine/src/ch/queries/query-helpers.ts @@ -5,15 +5,15 @@ import { finiteOrZero } from "./format" import type { AttributeFilter, MetricType } from "@maple/domain/query-engine" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import type { ColumnAccessor } from "@maple-dev/effect-clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import type { ColumnAccessor } from "@maple-dev/effect-orm/clickhouse" import type { ServiceOverviewSpans, Traces, TracesAggregatesHourly } from "../tables" -import { MetricsSum, MetricsGauge, MetricsHistogram, MetricsExpHistogram } from "../tables" +import { MetricsSum, MetricsGauge, MetricsHistogram, MetricsExpHistogram, orgIdParam } from "../tables" import { deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import { buildAttrFilterCondition, httpDisplaySpanName } from "../../traces-shared" import type { AttributeIndexMode } from "../../capabilities" -import * as T from "@maple-dev/effect-clickhouse/types" +import * as T from "@maple-dev/effect-orm/clickhouse" // APDEX expressions @@ -276,7 +276,7 @@ export function tracesBaseWhereConditions( const services = inclusionValues(opts.serviceName, opts.serviceNames) const spanNames = inclusionValues(opts.spanName, opts.spanNames) const conditions: Array = [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), CH.when(services, (v: readonly string[]) => @@ -427,7 +427,7 @@ export function serviceOverviewWhereConditions( const mm = opts.matchModes const services = inclusionValues(opts.serviceName, opts.serviceNames) const conditions: Array = [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), CH.when(services, (v: readonly string[]) => @@ -531,7 +531,7 @@ export function tracesAggregatesWhereConditions( const services = inclusionValues(opts.serviceName, opts.serviceNames) const spanNames = inclusionValues(opts.spanName, opts.spanNames) const conditions: Array = [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), hourBounds ? $.Hour.gte(CH.rawExpr(hourBounds.gte, T.dateTimeString)) : $.Hour.gte(param.dateTimeSeconds("startTime")), diff --git a/packages/query-engine/src/ch/queries/releases.test.ts b/packages/query-engine/src/ch/queries/releases.test.ts index fda9e65737..6e42a40036 100644 --- a/packages/query-engine/src/ch/queries/releases.test.ts +++ b/packages/query-engine/src/ch/queries/releases.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { releaseErrorFingerprintsQuery, releasesListQuery, releasesTimelineQuery } from "./releases" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", } diff --git a/packages/query-engine/src/ch/queries/releases.ts b/packages/query-engine/src/ch/queries/releases.ts index a2556b1b99..a0912198f1 100644 --- a/packages/query-engine/src/ch/queries/releases.ts +++ b/packages/query-engine/src/ch/queries/releases.ts @@ -6,17 +6,17 @@ // splice the services list already reads. Nothing scans the raw traces table. import { Schema } from "effect" -import * as T from "@maple-dev/effect-clickhouse/types" -import * as CH from "@maple-dev/effect-clickhouse/expr" +import * as T from "@maple-dev/effect-orm/clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" import { param, from, fromQuery, type CHQuery, type CompiledQueryRowSchema, -} from "@maple-dev/effect-clickhouse" -import type { ColumnDefs } from "@maple-dev/effect-clickhouse/types" -import { ErrorEventsByTime, ServiceOverviewSpans } from "../tables" +} from "@maple-dev/effect-orm/clickhouse" +import type { ColumnDefs } from "@maple-dev/effect-orm/clickhouse" +import { ErrorEventsByTime, ServiceOverviewSpans, orgIdParam } from "../tables" import { CHNumber } from "../schema" import { serviceOverviewWhereConditions } from "./query-helpers" import { serviceOverviewWindows, serviceWindowTiersForBucket } from "./services" @@ -251,7 +251,7 @@ export function releaseErrorFingerprintsQuery(opts: ReleaseErrorFingerprintsOpts firstSeen: CH.min_($.Timestamp), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(opts.serviceName), $.ServiceVersion.eq(param.string("serviceVersion")), $.Timestamp.gte(param.dateTimeSeconds("startTime")), diff --git a/packages/query-engine/src/ch/queries/rollup-splice.test.ts b/packages/query-engine/src/ch/queries/rollup-splice.test.ts index 0511d33235..c1b2646ebd 100644 --- a/packages/query-engine/src/ch/queries/rollup-splice.test.ts +++ b/packages/query-engine/src/ch/queries/rollup-splice.test.ts @@ -1,10 +1,10 @@ import { describe, expect, it } from "vitest" import { compileUnsafe } from "../index" -import { compile as compileFragment } from "@maple-dev/effect-clickhouse/sql" +import { compile as compileFragment } from "@maple-dev/effect-orm/sql" import * as CH from "../index" import { edgeCondition, hourGrain, interiorBounds, interiorConditions, minuteGrain } from "./rollup-splice" -import { paramPlaceholder } from "@maple-dev/effect-clickhouse" -import * as T from "@maple-dev/effect-clickhouse/types" +import { paramPlaceholder } from "@maple-dev/effect-orm/clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" // These pin the tiling invariant: the raw edge and the aggregate interior must // cover the window exactly once. Getting it wrong does not raise — it inflates @@ -21,7 +21,10 @@ const sqlOf = (cond: CH.Condition) => compileFragment(cond.toFragment()) /** Compile a trivial query carrying `conds` so the WHERE text can be inspected. */ const whereSql = (conds: ReadonlyArray) => { - const t = CH.table("t", { OrgId: CH.string, Hour: CH.dateTime, Minute: CH.dateTime }) + const t = CH.table("t", { + external: true, + columns: { OrgId: CH.string, Hour: CH.dateTime, Minute: CH.dateTime }, + }) return compileUnsafe( CH.from(t) .select(($) => ({ c: $.OrgId })) diff --git a/packages/query-engine/src/ch/queries/rollup-splice.ts b/packages/query-engine/src/ch/queries/rollup-splice.ts index 5557cf0302..6b80bc8294 100644 --- a/packages/query-engine/src/ch/queries/rollup-splice.ts +++ b/packages/query-engine/src/ch/queries/rollup-splice.ts @@ -22,9 +22,9 @@ // The expressions are SQL strings rather than DSL nodes because they embed // param placeholders that `compile()` substitutes later. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { paramPlaceholder } from "@maple-dev/effect-clickhouse" -import * as T from "@maple-dev/effect-clickhouse/types" +import * as CH from "@maple-dev/effect-orm/expr" +import { paramPlaceholder } from "@maple-dev/effect-orm/clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" /** * One tier boundary of a splice. `unit` names the bucket size; the rest are diff --git a/packages/query-engine/src/ch/queries/series-cap.test.ts b/packages/query-engine/src/ch/queries/series-cap.test.ts index 8538745a14..a043fdf212 100644 --- a/packages/query-engine/src/ch/queries/series-cap.test.ts +++ b/packages/query-engine/src/ch/queries/series-cap.test.ts @@ -1,10 +1,11 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { tracesTimeseriesQuery } from "./traces" import { logsTimeseriesQuery } from "./logs" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", bucketSeconds: 3600, diff --git a/packages/query-engine/src/ch/queries/series-cap.ts b/packages/query-engine/src/ch/queries/series-cap.ts index 3b1bed7bbe..d444301558 100644 --- a/packages/query-engine/src/ch/queries/series-cap.ts +++ b/packages/query-engine/src/ch/queries/series-cap.ts @@ -12,10 +12,10 @@ // group-by), the inner query is returned unchanged so existing SQL snapshots // stay byte-identical. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { fromQuery, type CHQuery } from "@maple-dev/effect-clickhouse" -import type { ColumnDefs } from "@maple-dev/effect-clickhouse/types" -import { uint64 } from "@maple-dev/effect-clickhouse/types" +import * as CH from "@maple-dev/effect-orm/expr" +import { fromQuery, type CHQuery } from "@maple-dev/effect-orm/clickhouse" +import type { ColumnDefs } from "@maple-dev/effect-orm/clickhouse" +import { uint64 } from "@maple-dev/effect-orm/clickhouse" function hasRealGroupBy(groupBy: readonly string[] | undefined): boolean { return !!groupBy && groupBy.some((key) => key !== "none") @@ -62,7 +62,10 @@ export function finalizeTimeseries>( // A CTE referenced by both a top-groups subquery and the result query repeats // the base scan in ClickHouse. Windows rank the already aggregated buckets // while reading the tenant-scoped inner query once. - const peaks = fromQuery(inner.orderBy(), "__series_base").select(() => ({ + // Read back by name through `outputColumns`, so the inner query's own output + // type is not needed past this point. + const base: CHQuery> = inner.orderBy() + const peaks = fromQuery(base, "__series_base").select(() => ({ ...passthrough, __series_peak: CH.over( CH.max_(CH.dynamicColumn(rankColumn, outputColumns[rankColumn])), diff --git a/packages/query-engine/src/ch/queries/service-endpoints.test.ts b/packages/query-engine/src/ch/queries/service-endpoints.test.ts index 1e845119ce..06c67ec5b3 100644 --- a/packages/query-engine/src/ch/queries/service-endpoints.test.ts +++ b/packages/query-engine/src/ch/queries/service-endpoints.test.ts @@ -1,14 +1,15 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { serviceOperationsSummaryQuery } from "./service-operations" import { serviceEndpointsSummaryQuery, serviceEndpointsSummaryRawQuery, splitEndpointName, } from "./service-endpoints" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", } diff --git a/packages/query-engine/src/ch/queries/service-infra.test.ts b/packages/query-engine/src/ch/queries/service-infra.test.ts index f987d9fffc..6539501f8c 100644 --- a/packages/query-engine/src/ch/queries/service-infra.test.ts +++ b/packages/query-engine/src/ch/queries/service-infra.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from "@effect/vitest" import { Effect, Exit } from "effect" import { serviceWorkloadsSQL } from "./service-infra" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", } diff --git a/packages/query-engine/src/ch/queries/service-infra.ts b/packages/query-engine/src/ch/queries/service-infra.ts index f41acfa6c4..f556f330ad 100644 --- a/packages/query-engine/src/ch/queries/service-infra.ts +++ b/packages/query-engine/src/ch/queries/service-infra.ts @@ -25,14 +25,15 @@ // over two grouped subqueries), then compiled to SQL — no hand-written SQL. import { Schema, Effect } from "effect" -import { compile, type CompiledQuery, type CompiledQueryRowSchema } from "@maple-dev/effect-clickhouse" +import { compile, type CompiledQuery, type CompiledQueryRowSchema } from "@maple-dev/effect-orm/clickhouse" import { rawCompiledQuery } from "../raw-sql" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import { from, fromQuery } from "@maple-dev/effect-clickhouse" -import { MetricsGauge, ServicePlatformsHourly } from "../tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { from, fromQuery } from "@maple-dev/effect-orm/clickhouse" +import { MetricsGauge, ServicePlatformsHourly, orgIdParam } from "../tables" import { CHNumber } from "../schema" -import type { QueryBuilderError } from "@maple-dev/effect-clickhouse" +import type { QueryBuilderError } from "@maple-dev/effect-orm/clickhouse" +import type { OrgId } from "@maple/domain" export interface ServiceWorkloadsOpts { services: ReadonlyArray @@ -72,7 +73,7 @@ FORMAT JSON` export function serviceWorkloadsSQL( opts: ServiceWorkloadsOpts, - params: { orgId: string; startTime: string; endTime: string }, + params: { orgId: OrgId; startTime: string; endTime: string }, ): Effect.Effect, QueryBuilderError> { if (opts.services.length === 0) { // Reads no table at all (`WHERE 0`), so it cannot cross tenants; it stands @@ -125,7 +126,7 @@ export function serviceWorkloadsSQL( } }) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(CH.toStartOfHour(CH.toDateTime(param.dateTimeString("startTime")))), $.Hour.lte(param.dateTimeSeconds("endTime")), CH.inList($.ServiceName, opts.services), @@ -172,7 +173,7 @@ export function serviceWorkloadsSQL( } }) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TimeUnix.gte(param.dateTimeString("startTime")), $.TimeUnix.lte(param.dateTimeString("endTime")), CH.inList($.MetricName, [ diff --git a/packages/query-engine/src/ch/queries/service-map-rollup.test.ts b/packages/query-engine/src/ch/queries/service-map-rollup.test.ts index 23e390cbec..fdf1ed9fb4 100644 --- a/packages/query-engine/src/ch/queries/service-map-rollup.test.ts +++ b/packages/query-engine/src/ch/queries/service-map-rollup.test.ts @@ -12,6 +12,7 @@ import { serviceMapRollupMissingHours, serviceMapRollupWindowParams, } from "./service-map-rollup" +import { OrgId } from "@maple/domain" describe("service-map rollup hour planning", () => { const hour = 3_600_000 @@ -30,13 +31,13 @@ describe("service-map rollup hour planning", () => { }) it("formats hour and window bounds as warehouse datetimes", () => { - expect(serviceMapRollupHourParams("org_1", base)).toEqual({ - orgId: "org_1", + expect(serviceMapRollupHourParams(OrgId.make("org_1"), base)).toEqual({ + orgId: OrgId.make("org_1"), hourStart: "2024-01-01 00:00:00", hourEnd: "2024-01-01 01:00:00", }) - expect(serviceMapRollupWindowParams("org_1", base, base + 3 * hour)).toEqual({ - orgId: "org_1", + expect(serviceMapRollupWindowParams(OrgId.make("org_1"), base, base + 3 * hour)).toEqual({ + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-01 03:00:00", }) @@ -44,7 +45,7 @@ describe("service-map rollup hour planning", () => { }) const hourParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), hourStart: "2024-01-01 00:00:00", hourEnd: "2024-01-01 01:00:00", } @@ -53,7 +54,7 @@ describe("service-map rollup compiled row schemas", () => { it.effect("decodes existing-hour rows with numeric strings", () => Effect.gen(function* () { const compiled = yield* serviceMapEdgesExistingHoursSQL({ - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", }) @@ -137,7 +138,7 @@ describe("service-map rollup compiled row schemas", () => { describe("service-map rollup routing", () => { const windowParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", } diff --git a/packages/query-engine/src/ch/queries/service-map-rollup.ts b/packages/query-engine/src/ch/queries/service-map-rollup.ts index 688c336a70..7d5c40ec3e 100644 --- a/packages/query-engine/src/ch/queries/service-map-rollup.ts +++ b/packages/query-engine/src/ch/queries/service-map-rollup.ts @@ -13,17 +13,17 @@ // into `ingest` with no reshaping. import { Schema, Effect } from "effect" -import type { CompiledQuery, CompiledQueryRowSchema } from "@maple-dev/effect-clickhouse" -import { compile } from "@maple-dev/effect-clickhouse" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import { from, fromQuery } from "@maple-dev/effect-clickhouse" +import type { CompiledQuery, CompiledQueryRowSchema } from "@maple-dev/effect-orm/clickhouse" +import { compile } from "@maple-dev/effect-orm/clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { from, fromQuery } from "@maple-dev/effect-orm/clickhouse" import { OrgId } from "@maple/domain" -import { ServiceAddressResolutionsHourly, ServiceMapEdgesHourly, Traces } from "../tables" +import { ServiceAddressResolutionsHourly, ServiceMapEdgesHourly, Traces, orgIdParam } from "../tables" import { deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import { serviceMapEdgeJoinQuery } from "./service-map" import { CHNumber } from "../schema" -import type { QueryBuilderError } from "@maple-dev/effect-clickhouse" +import type { QueryBuilderError } from "@maple-dev/effect-orm/clickhouse" import { formatWarehouseDateTime } from "../../datetime" // Hour planning shared by the cloud rollup service and the local server's @@ -65,14 +65,14 @@ export const serviceMapResolutionRepairHours = ( candidates.filter((hourMs) => hasHour(sealed, hourMs) && !hasHour(resolved, hourMs)) /** Params for the per-hour edge and resolution rollups of the hour starting at `hourMs`. */ -export const serviceMapRollupHourParams = (orgId: string, hourMs: number): ServiceMapEdgesRollupParams => ({ +export const serviceMapRollupHourParams = (orgId: OrgId, hourMs: number): ServiceMapEdgesRollupParams => ({ orgId, hourStart: formatWarehouseDateTime(hourMs), hourEnd: formatWarehouseDateTime(hourMs + SERVICE_MAP_ROLLUP_HOUR_MS), }) /** Params for the existing-hours probes over `[oldestHourMs, currentHourMs)`. */ -export const serviceMapRollupWindowParams = (orgId: string, oldestHourMs: number, currentHourMs: number) => ({ +export const serviceMapRollupWindowParams = (orgId: OrgId, oldestHourMs: number, currentHourMs: number) => ({ orgId, startTime: formatWarehouseDateTime(oldestHourMs), endTime: formatWarehouseDateTime(currentHourMs), @@ -113,7 +113,7 @@ const ServiceMapEdgesHourlyOutputSchema: CompiledQueryRowSchema, QueryBuilderError> { @@ -142,7 +142,7 @@ export function serviceMapEdgesExistingHoursSQL(params: { const query = from(ServiceMapEdgesHourly) .select(($) => ({ hourTs: CH.toUnixTimestamp($.Hour) })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(param.dateTimeSeconds("startTime")), $.Hour.lt(param.dateTimeSeconds("endTime")), ]) @@ -173,14 +173,14 @@ export function serviceMapEdgesExistingHoursSQL(params: { * read and skips nearly all of them. */ export function serviceMapResolutionsExistingHoursSQL(params: { - orgId: string + orgId: OrgId startTime: string endTime: string }): Effect.Effect, QueryBuilderError> { const query = from(ServiceAddressResolutionsHourly) .select(($) => ({ hourTs: CH.toUnixTimestamp($.Hour) })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(param.dateTimeSeconds("startTime")), $.Hour.lt(param.dateTimeSeconds("endTime")), ]) @@ -267,7 +267,7 @@ export function serviceMapResolutionsRollupSQL( CH.inList($.SpanKind, ["Client", "Producer"]), $.Timestamp.gte(param.dateTimeString("hourStart")), $.Timestamp.lt(param.dateTimeString("hourEnd")), - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.SpanAttributes.get("server.address").neq(""), ]) @@ -283,7 +283,7 @@ export function serviceMapResolutionsRollupSQL( CH.inList($.SpanKind, ["Server", "Consumer"]), $.Timestamp.gte(param.dateTimeString("hourStart")), $.Timestamp.lt(param.dateTimeString("hourEnd")), - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), ]) const query = fromQuery(parents, "p") diff --git a/packages/query-engine/src/ch/queries/service-map.test.ts b/packages/query-engine/src/ch/queries/service-map.test.ts index 20aeb65819..f4b4790f7c 100644 --- a/packages/query-engine/src/ch/queries/service-map.test.ts +++ b/packages/query-engine/src/ch/queries/service-map.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "@effect/vitest" import { Effect, Exit } from "effect" -import { compileUnsafe, param, toDateTime } from "@maple-dev/effect-clickhouse" +import { compileUnsafe, param, toDateTime } from "@maple-dev/effect-orm/clickhouse" import { serviceDbEdgesSQL, serviceDbEdgesForServiceQuery, @@ -16,6 +16,7 @@ import { import { serviceMapEdgesRollupSQL, serviceMapResolutionsRollupSQL } from "./service-map-rollup" import { ServiceMapEdgesHourly } from "../tables" import { DB_NAMESPACE_ATTR_SQL } from "@maple/domain/tinybird/db-query-shape-sql" +import { OrgId } from "@maple/domain" // The read side must derive `DbNamespace` byte-identically to the write side's // `DB_NAMESPACE_ATTR_SQL` (Hyperdrive-collapsing coalesce) or the sealed-hour and @@ -33,7 +34,7 @@ const SEALED_DB_QUERIES = sealedNamespaceCollapse("service_map_db_query_shapes_h const RAW_NAMESPACE = DB_NAMESPACE_ATTR_SQL.replace(/(? { it("escapes single quotes in serviceName / orgId to prevent SQL injection", () => { const { sql } = Effect.runSync( - serviceExternalEdgesSQL({ serviceName: "weird'service" }, { ...baseParams, orgId: "org'attack" }), + serviceExternalEdgesSQL( + { serviceName: "weird'service" }, + { ...baseParams, orgId: OrgId.make("org'attack") }, + ), ) expect(sql).toContain("ServiceName = 'weird\\'service'") expect(sql).toContain("OrgId = 'org\\'attack'") @@ -195,7 +199,7 @@ describe("serviceExternalEdgesSQL", () => { describe("serviceMapResolutionsRollupSQL", () => { const hourParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), hourStart: "2024-01-01 00:00:00", hourEnd: "2024-01-01 01:00:00", } @@ -261,7 +265,7 @@ describe("serviceDependenciesSQL", () => { serviceDependenciesSQL( {}, { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 10:15:00", endTime: "2024-01-01 12:30:00", }, @@ -350,7 +354,7 @@ describe("serviceMapEdgeJoinQuery", () => { const compiledRollup = () => Effect.runSync( serviceMapEdgesRollupSQL({ - orgId: "org_1", + orgId: OrgId.make("org_1"), hourStart: "2024-01-01 10:00:00", hourEnd: "2024-01-01 11:00:00", }), @@ -380,7 +384,7 @@ describe("serviceMapEdgeJoinQuery", () => { rangeEnd: toDateTime(param.dateTimeString("hourEnd")), parentServiceName: "web", }), - { orgId: "org_1", hourStart: "2024-01-01 10:00:00", hourEnd: "2024-01-01 11:00:00" }, + { orgId: OrgId.make("org_1"), hourStart: "2024-01-01 10:00:00", hourEnd: "2024-01-01 11:00:00" }, ) // Inside the parent subquery — before the JOIN — so ClickHouse can skip // the full Client/Producer scan rather than filtering after the join. @@ -391,7 +395,7 @@ describe("serviceMapEdgeJoinQuery", () => { describe("serviceDependenciesForServiceQuery", () => { it("keeps partial start and end hours outside the hourly rollup", () => { const { sql } = compileUnsafe(serviceDependenciesForServiceQuery({ serviceName: "artifacts-api" }), { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 10:15:00", endTime: "2024-01-01 12:30:00", }) diff --git a/packages/query-engine/src/ch/queries/service-map.ts b/packages/query-engine/src/ch/queries/service-map.ts index aa109a5aa2..e8d435d0a2 100644 --- a/packages/query-engine/src/ch/queries/service-map.ts +++ b/packages/query-engine/src/ch/queries/service-map.ts @@ -17,14 +17,14 @@ import { } from "@maple/domain/tinybird/db-query-shape-sql" import { deploymentEnvExpr, messagingDestinationExpr } from "@maple/domain/tinybird/semconv-renames" import { Schema, Effect } from "effect" -import { compile, type CompiledQuery, type CompiledQueryRowSchema } from "@maple-dev/effect-clickhouse" -import { defineCondFn, defineFn } from "@maple-dev/effect-clickhouse" -import * as CH from "@maple-dev/effect-clickhouse/expr" +import { compile, type CompiledQuery, type CompiledQueryRowSchema } from "@maple-dev/effect-orm/clickhouse" +import { defineCondFn, defineFn } from "@maple-dev/effect-orm/clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" // From the root, not `/expr`: this overload takes a `CHQuery`, so the subquery // keeps its params, table names and column types checked. -import { inSubquery } from "@maple-dev/effect-clickhouse" -import { param } from "@maple-dev/effect-clickhouse" -import { from, fromQuery, fromUnion } from "@maple-dev/effect-clickhouse" +import { inSubquery } from "@maple-dev/effect-orm/clickhouse" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { from, fromQuery, fromUnion } from "@maple-dev/effect-orm/clickhouse" import { ServiceAddressResolutionsHourly, ServiceExternalEdgesHourly, @@ -36,12 +36,14 @@ import { ServicePlatformsHourly, type StringMap, Traces, + orgIdParam, } from "../tables" -import { unionAll } from "@maple-dev/effect-clickhouse" +import { unionAll } from "@maple-dev/effect-orm/clickhouse" import { edgeCondition, interiorConditions } from "./rollup-splice" import { CHNumber, CHNumberOrZero } from "../schema" -import * as T from "@maple-dev/effect-clickhouse/types" -import type { QueryBuilderError } from "@maple-dev/effect-clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" +import type { QueryBuilderError } from "@maple-dev/effect-orm/clickhouse" +import type { OrgId } from "@maple/domain" // Local CH function declarations used by the live topology-join branch's // sample-weighting math. Kept here (not promoted to ch/functions/) because @@ -162,7 +164,7 @@ function serviceMapEdgeJoinSource(opts: { $.SpanKind.in_("Client", "Producer"), $.Timestamp.gte(opts.rangeStart), $.Timestamp.lt(opts.rangeEnd), - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), envFilter($.DeploymentEnv), edgeOnly, opts.parentServiceName ? $.ServiceName.eq(opts.parentServiceName) : undefined, @@ -180,7 +182,7 @@ function serviceMapEdgeJoinSource(opts: { .where(($) => [ $.Timestamp.gte(opts.rangeStart), $.Timestamp.lt(opts.rangeEnd), - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), envFilter($.DeploymentEnv), edgeOnly, ]) @@ -249,7 +251,7 @@ export function serviceMapEdgeJoinQuery(opts: { */ export function serviceDependenciesSQL( opts: ServiceDependenciesOpts, - params: { orgId: string; startTime: string; endTime: string }, + params: { orgId: OrgId; startTime: string; endTime: string }, ): Effect.Effect, QueryBuilderError> { return compile( serviceDependenciesQueryBase({ deploymentEnv: opts.deploymentEnv }), @@ -334,7 +336,7 @@ export function serviceDependenciesQueryBase(opts: { serviceName?: string; deplo ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), opts.serviceName ? $.SourceService.eq(opts.serviceName) : undefined, ...interiorConditions($.Hour), envFilterMv($.DeploymentEnv), @@ -456,7 +458,7 @@ const ServiceDbEdgesOutputSchema: CompiledQueryRowSchema = export function serviceDbEdgesSQL( opts: ServiceDbEdgesOpts, - params: { orgId: string; startTime: string; endTime: string }, + params: { orgId: OrgId; startTime: string; endTime: string }, ): Effect.Effect, QueryBuilderError> { return compile(serviceDbEdgesQueryBase(opts), params, { rowSchema: ServiceDbEdgesOutputSchema, @@ -548,7 +550,7 @@ function serviceDbEdgesQueryBase(opts: { serviceName?: string; deploymentEnv?: s bucketDurationQuantiles: CH.rawExpr(EDGE_TDIGEST_MERGE_STATE_EXPR, EDGE_DURATION_STATE), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), opts.serviceName ? $.ServiceName.eq(opts.serviceName) : undefined, ...interiorConditions($.Hour), $.DbSystem.neq(""), @@ -580,7 +582,7 @@ function serviceDbEdgesQueryBase(opts: { serviceName?: string; deploymentEnv?: s bucketDurationQuantiles: CH.rawExpr(EDGE_TDIGEST_RAW_STATE_EXPR, EDGE_DURATION_STATE), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), // Scoped to a service (implies non-empty) or, org-wide, require a named // service — the hourly MV already filters `ServiceName != ''` at write // time, so this keeps the raw in-progress-hour branch consistent and @@ -648,7 +650,7 @@ export function serviceDbEdgesForServiceQuery(opts: ServiceDbEdgesForServiceOpts // with the rollup MV so a shape's key is stable across the sealed/live boundary. export interface ServiceDbQuerySummaryParams { - readonly orgId: string + readonly orgId: OrgId readonly dbSystem: string /** * Scope to one database identity (`DbNamespace`). `undefined` = unscoped @@ -778,7 +780,7 @@ const signaturesHourlyFilters = ( }, params: ServiceDbQuerySummaryParams, ) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), ...interiorConditions($.Hour), $.DbSystem.eq(params.dbSystem), // `undefined` = unscoped; `''` is a real value (the legacy/unknown node). @@ -809,7 +811,7 @@ const serviceDbRawFilters = ( params: ServiceDbQuerySummaryParams, scope: "edge" | "fullWindow", ) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(CH.toDateTime(param.dateTimeString("startTime"))), $.Timestamp.lte(CH.toDateTime(param.dateTimeString("endTime"))), scope === "edge" ? edgeCondition("Timestamp") : undefined, @@ -1119,7 +1121,7 @@ const ServiceExternalEdgesOutputSchema: CompiledQueryRowSchema, QueryBuilderError> { const startHour = CH.toStartOfHour(CH.toDateTime(param.dateTimeString("startTime"))) const endHour = CH.toStartOfHour(CH.toDateTime(param.dateTimeString("endTime"))) @@ -1143,7 +1145,7 @@ export function serviceExternalEdgesSQL( bucketDurationQuantiles: CH.rawExpr(EDGE_TDIGEST_MERGE_STATE_EXPR, EDGE_DURATION_STATE), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(opts.serviceName), ...interiorConditions($.Hour), $.TargetName.neq(""), @@ -1199,7 +1201,7 @@ export function serviceExternalEdgesSQL( .where(($) => { const attr = (key: string) => $.SpanAttributes.get(key) return [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(opts.serviceName), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), @@ -1230,7 +1232,7 @@ export function serviceExternalEdgesSQL( const internalResolutions = from(ServiceAddressResolutionsHourly) .select(($) => ({ ParentServerAddress: $.ParentServerAddress })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.SourceService.eq(opts.serviceName), $.Hour.gte(startHour), $.Hour.lt(endHour), @@ -1304,7 +1306,7 @@ export interface ServicePlatformsOutput { export function servicePlatformsSQL( opts: ServicePlatformsOpts, - params: { orgId: string; startTime: string; endTime: string }, + params: { orgId: OrgId; startTime: string; endTime: string }, ): Effect.Effect, QueryBuilderError> { const query = from(ServicePlatformsHourly) .select(($) => ({ @@ -1323,7 +1325,7 @@ export function servicePlatformsSQL( processRuntimeName: CH.max_($.ProcessRuntimeName), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(CH.toStartOfHour(CH.toDateTime(param.dateTimeString("startTime")))), $.Hour.lte(param.dateTimeSeconds("endTime")), $.ServiceName.neq(""), @@ -1397,7 +1399,7 @@ export function dbQueryVolumeQuery(opts: DbQueryVolumeOpts) { bLastSeen: CH.max_($.Hour), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), ...interiorConditions($.Hour), opts.dbSystem ? $.DbSystem.eq(opts.dbSystem) : undefined, opts.serviceName ? $.ServiceName.eq(opts.serviceName) : undefined, @@ -1421,7 +1423,7 @@ export function dbQueryVolumeQuery(opts: DbQueryVolumeOpts) { bLastSeen: CH.max_(CH.toDateTime($.Timestamp)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(CH.toDateTime(param.dateTimeString("startTime"))), $.Timestamp.lte(CH.toDateTime(param.dateTimeString("endTime"))), edgeCondition("Timestamp"), diff --git a/packages/query-engine/src/ch/queries/service-operations.test.ts b/packages/query-engine/src/ch/queries/service-operations.test.ts index 4245b046a2..255c89d917 100644 --- a/packages/query-engine/src/ch/queries/service-operations.test.ts +++ b/packages/query-engine/src/ch/queries/service-operations.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { Schema } from "effect" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { NORMALIZED_SPAN_NAME_SQL } from "@maple/domain/tinybird/span-display-name" import { serviceOperationsSummaryQuery, @@ -10,9 +10,10 @@ import { serviceOperationsTimeseriesRawQuery, serviceOperationsTimeseriesRowSchema, } from "./service-operations" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", } @@ -105,7 +106,7 @@ describe("serviceOperationsSummaryQuery", () => { it("uses disjoint raw and rollup boundaries for partial edge minutes", () => { const q = serviceOperationsSummaryQuery({ serviceName: "api" }) const { sql } = compileUnsafe(q, { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:30", endTime: "2024-01-01 00:02:15", }) @@ -121,7 +122,7 @@ describe("serviceOperationsSummaryQuery", () => { ["2024-01-01 00:00:10", "2024-01-01 00:00:10"], ] as const) { const { sql } = compileUnsafe(serviceOperationsSummaryQuery({ serviceName: "api" }), { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime, endTime, }) diff --git a/packages/query-engine/src/ch/queries/service-operations.ts b/packages/query-engine/src/ch/queries/service-operations.ts index 0b74caa9ee..35f8e64066 100644 --- a/packages/query-engine/src/ch/queries/service-operations.ts +++ b/packages/query-engine/src/ch/queries/service-operations.ts @@ -14,15 +14,21 @@ // unweighted, matching every other raw-Traces query. import { Schema } from "effect" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import { defineCondFn, from, fromUnion, unionAll, type ColumnAccessor } from "@maple-dev/effect-clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { + defineCondFn, + from, + fromUnion, + unionAll, + type ColumnAccessor, +} from "@maple-dev/effect-orm/clickhouse" import { httpDisplaySpanName } from "../../traces-shared" import { CHNumber } from "../schema" -import { ServiceOperationsHourly, ServiceOperationsMinutely, Traces } from "../tables" +import { ServiceOperationsHourly, ServiceOperationsMinutely, Traces, orgIdParam } from "../tables" import { tracesBaseWhereConditions } from "./query-helpers" import { edgeCondition, hourGrain, interiorConditions, minuteGrain } from "./rollup-splice" -import * as T from "@maple-dev/effect-clickhouse/types" +import * as T from "@maple-dev/effect-orm/clickhouse" export interface ServiceOperationsSummaryOpts { serviceName: string @@ -206,7 +212,7 @@ export function serviceOperationsSummaryQuery(opts: ServiceOperationsSummaryOpts bDurationQuantiles: CH.rawExpr(ROLLUP_DURATION_STATE, DURATION_STATE), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(opts.serviceName), rollupEnvironmentCondition($, opts.environments), ...interiorConditions($.Minute, minuteGrain), @@ -226,7 +232,7 @@ export function serviceOperationsSummaryQuery(opts: ServiceOperationsSummaryOpts bDurationQuantiles: CH.rawExpr(ROLLUP_DURATION_STATE, DURATION_STATE), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(opts.serviceName), hourlyEnvironmentCondition($, opts.environments), ...interiorConditions($.Hour, hourGrain), @@ -336,7 +342,7 @@ export function serviceOperationsTimeseriesQuery(opts: ServiceOperationsTimeseri count: CH.sum($.EstimatedSpanCount), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(opts.serviceName), rollupEnvironmentCondition($, opts.environments), ...interiorConditions($.Minute, minuteGrain), @@ -354,7 +360,7 @@ export function serviceOperationsTimeseriesQuery(opts: ServiceOperationsTimeseri count: CH.sum($.EstimatedSpanCount), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(opts.serviceName), hourlyEnvironmentCondition($, opts.environments), ...interiorConditions($.Hour, hourGrain), @@ -452,7 +458,7 @@ export function routeUsageQuery(opts: RouteUsageOpts) { bLast: CH.max_($.Minute), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), CH.when(opts.serviceName, (value: string) => $.ServiceName.eq(value)), rollupEnvironmentCondition($, opts.environments), ...interiorConditions($.Minute, minuteGrain), @@ -473,7 +479,7 @@ export function routeUsageQuery(opts: RouteUsageOpts) { bLast: CH.max_($.Hour), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), CH.when(opts.serviceName, (value: string) => $.ServiceName.eq(value)), hourlyEnvironmentCondition($, opts.environments), ...interiorConditions($.Hour, hourGrain), diff --git a/packages/query-engine/src/ch/queries/services.test.ts b/packages/query-engine/src/ch/queries/services.test.ts index 250b2c60ea..4c4cfdfeb0 100644 --- a/packages/query-engine/src/ch/queries/services.test.ts +++ b/packages/query-engine/src/ch/queries/services.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" -import { compileUnsafe, compileUnionUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe, compileUnionUnsafe } from "@maple-dev/effect-orm/clickhouse" import { serviceOverviewQuery, serviceOverviewRowSchema, @@ -14,9 +14,10 @@ import { serviceUsageWithPreviousQuery, servicesFacetsQuery, } from "./services" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", bucketSeconds: 3600, @@ -431,7 +432,7 @@ describe("serviceUsageQuery", () => { describe("serviceUsageWithPreviousQuery", () => { const params = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-02 00:00:00", endTime: "2024-01-03 00:00:00", previousStartTime: "2024-01-01 00:00:00", diff --git a/packages/query-engine/src/ch/queries/services.ts b/packages/query-engine/src/ch/queries/services.ts index 3446933ef3..b541d6fff1 100644 --- a/packages/query-engine/src/ch/queries/services.ts +++ b/packages/query-engine/src/ch/queries/services.ts @@ -3,9 +3,9 @@ // DSL-based query definitions for service overview, releases, apdex, and usage. import { Schema } from "effect" -import * as T from "@maple-dev/effect-clickhouse/types" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" import { from, fromQuery, @@ -13,15 +13,16 @@ import { type CHQuery, type ColumnAccessor, type CompiledQueryRowSchema, -} from "@maple-dev/effect-clickhouse" -import { unionAll, type CHUnionQuery } from "@maple-dev/effect-clickhouse" -import type { ColumnDefs } from "@maple-dev/effect-clickhouse/types" +} from "@maple-dev/effect-orm/clickhouse" +import { unionAll, type CHUnionQuery } from "@maple-dev/effect-orm/clickhouse" +import type { ColumnDefs } from "@maple-dev/effect-orm/clickhouse" import { ServiceOverviewHourly, ServiceOverviewMinutely, ServiceOverviewSpans, ServiceUsage, TracesAggregatesHourly, + orgIdParam, } from "../tables" import { CHNumber } from "../schema" import { apdexExprs, serviceOverviewWhereConditions, hourFloor, type FacetOutput } from "./query-helpers" @@ -135,7 +136,7 @@ export function serviceOverviewWindows(filters: ServiceWindowFilters, tiers: Ser >( $: A, ) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), CH.when(filters.serviceName, (value: string) => $.ServiceName.eq(value)), filters.environments?.length ? CH.inList($.DeploymentEnv, filters.environments) : undefined, filters.namespaces?.length ? CH.inList($.ServiceNamespace, filters.namespaces) : undefined, @@ -501,7 +502,7 @@ export function serviceHealthSnapshotQuery(opts: ServiceHealthSnapshotOpts) { ), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.IsEntryPoint.eq(1), $.Hour.gte(hourFloor("startTime")), $.Hour.lte(hourFloor("endTime")), @@ -809,7 +810,7 @@ export function serviceUsageQuery(opts: ServiceUsageOpts) { .add(CH.sum($.ExpHistogramMetricSizeBytes)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), // `service_usage` is keyed on top-of-hour `Hour`. Comparing to the raw // `startTime` / `endTime` literals misses every sub-hour window — e.g. // "last 15 min" at 22:23–22:38 returns no rows because `Hour=22:00 < @@ -888,7 +889,7 @@ export function serviceUsageWithPreviousQuery(opts: ServiceUsageOpts) { .add(CH.sumIf($.ExpHistogramMetricSizeBytes, inPrevious($))), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), // Scan the union window [previousStartTime, endTime] once; sumIf splits // it into the two periods. Hour-floored bounds match serviceUsageQuery. $.Hour.gte(hourFloor("previousStartTime")), diff --git a/packages/query-engine/src/ch/queries/session-events.test.ts b/packages/query-engine/src/ch/queries/session-events.test.ts index 43d6ff87f9..e2c14a9c32 100644 --- a/packages/query-engine/src/ch/queries/session-events.test.ts +++ b/packages/query-engine/src/ch/queries/session-events.test.ts @@ -1,8 +1,9 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { sessionActivityQuery, IDLE_GAP_THRESHOLD_MS } from "./session-events" +import { OrgId } from "@maple/domain" -const sessionParams = { orgId: "org_1", sessionId: "sess_1" } +const sessionParams = { orgId: OrgId.make("org_1"), sessionId: "sess_1" } const WINDOW = { startTime: "2026-06-24 04:00:00", endTime: "2026-06-25 06:00:00" } // sessionActivityQuery diff --git a/packages/query-engine/src/ch/queries/session-events.ts b/packages/query-engine/src/ch/queries/session-events.ts index 7d8892a40c..1e40b14942 100644 --- a/packages/query-engine/src/ch/queries/session-events.ts +++ b/packages/query-engine/src/ch/queries/session-events.ts @@ -7,10 +7,16 @@ // // Plain MergeTree, immutable append; no ReplacingMergeTree dedup needed. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import { from, fromQuery, type ColumnAccessor } from "@maple-dev/effect-clickhouse" -import { SessionEvents } from "../tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { + from, + fromQuery, + type CHQuery, + type ColumnAccessor, + type InferOutput, +} from "@maple-dev/effect-orm/clickhouse" +import { SessionEvents, orgIdParam } from "../tables" // The builder's `count`, which knows the result is a `UInt64` — a local // `compileFnCall` copy shadowed it and decoded nothing. @@ -83,7 +89,7 @@ export function sessionTranscriptQuery(opts: SessionTranscriptOpts = {}) { attributes: CH.toJSONString($.Attributes), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.SessionId.eq(param.string("sessionId")), CH.when(opts.startTime, (v: string) => $.Timestamp.gte(v)), CH.when(opts.endTime, (v: string) => $.Timestamp.lte(v)), @@ -131,7 +137,7 @@ export function sessionEventMatchQuery(opts: SessionEventMatchOpts) { matchCount: count(), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), CH.when(opts.type, (v: string) => $.Type.eq(v)), @@ -202,9 +208,22 @@ function sessionGapSelect($: ColumnAccessor) { } } +function sessionGaps() { + return from(SessionEvents).select(sessionGapSelect) +} + +/** The per-event gap query, whatever params its WHERE binds. */ +type SessionGaps

= CHQuery< + typeof SessionEvents.columns, + InferOutput>, + {}, + undefined, + P +> + // Aggregate per-session gaps into active / idle totals. Generic over the inner // gap subquery so both variants share the threshold split. -function activeIdleAggregate(inner: ReturnType) { +function activeIdleAggregate

(inner: SessionGaps

) { return fromQuery(inner, "g") .select(($) => ({ sessionId: $.sessionId, @@ -215,31 +234,16 @@ function activeIdleAggregate(inner: ReturnType) { .groupBy("sessionId") } -function sessionActivityGaps(opts: { single: boolean } & SessionActivityOpts) { - return from(SessionEvents) - .select(sessionGapSelect) - .where(($) => - opts.single - ? [ - $.OrgId.eq(param.string("orgId")), - $.SessionId.eq(param.string("sessionId")), - CH.when(opts.startTime, (v: string) => $.Timestamp.gte(v)), - CH.when(opts.endTime, (v: string) => $.Timestamp.lte(v)), - ] - : [ - $.OrgId.eq(param.string("orgId")), - $.Timestamp.gte(param.dateTimeString("startTime")), - $.Timestamp.lte(param.dateTimeString("endTime")), - ], - ) -} - // Single session (detail page + MCP get_session_traces). Binds `sessionId` as a // param; optional startTime/endTime hints prune daily partitions. export function sessionActivityQuery(opts: SessionActivityOpts = {}) { - return activeIdleAggregate(sessionActivityGaps({ single: true, ...opts })) - .limit(1) - .format("JSON") + const gaps = sessionGaps().where(($) => [ + $.OrgId.eq(orgIdParam), + $.SessionId.eq(param.string("sessionId")), + CH.when(opts.startTime, (v: string) => $.Timestamp.gte(v)), + CH.when(opts.endTime, (v: string) => $.Timestamp.lte(v)), + ]) + return activeIdleAggregate(gaps).limit(1).format("JSON") } // Every session in the org + time window, keyed by sessionId. Returned as an @@ -247,5 +251,10 @@ export function sessionActivityQuery(opts: SessionActivityOpts = {}) { // active time. Binds the same `startTime`/`endTime` params as the list query, so // they resolve to the same window when compiled together. export function sessionActivityAggregateQuery() { - return activeIdleAggregate(sessionActivityGaps({ single: false })) + const gaps = sessionGaps().where(($) => [ + $.OrgId.eq(orgIdParam), + $.Timestamp.gte(param.dateTimeString("startTime")), + $.Timestamp.lte(param.dateTimeString("endTime")), + ]) + return activeIdleAggregate(gaps) } diff --git a/packages/query-engine/src/ch/queries/session-replays.test.ts b/packages/query-engine/src/ch/queries/session-replays.test.ts index ee8bafb730..ef46160d93 100644 --- a/packages/query-engine/src/ch/queries/session-replays.test.ts +++ b/packages/query-engine/src/ch/queries/session-replays.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "@effect/vitest" -import { compileUnsafe, compileUnionUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe, compileUnionUnsafe } from "@maple-dev/effect-orm/clickhouse" import { getSessionReplayQuery, sessionReplaysFacetsQuery, @@ -9,9 +9,10 @@ import { sessionsForTraceQuery, sessionTraceSummariesQuery, } from "./session-replays" +import { OrgId } from "@maple/domain" -const baseParams = { orgId: "org_1" } -const sessionParams = { orgId: "org_1", sessionId: "sess_1" } +const baseParams = { orgId: OrgId.make("org_1") } +const sessionParams = { orgId: OrgId.make("org_1"), sessionId: "sess_1" } const WINDOW = { startTime: "2026-06-24 04:00:00", endTime: "2026-06-25 06:00:00" } // sessionTraceSummariesQuery diff --git a/packages/query-engine/src/ch/queries/session-replays.ts b/packages/query-engine/src/ch/queries/session-replays.ts index dbde6c5378..b86f78ac09 100644 --- a/packages/query-engine/src/ch/queries/session-replays.ts +++ b/packages/query-engine/src/ch/queries/session-replays.ts @@ -16,14 +16,14 @@ // Stale-prone post-aggregation predicates (e.g. exact Status) are deliberately // not exposed as SQL filters since the DSL has no HAVING clause. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { compileFnCallCond } from "@maple-dev/effect-clickhouse" -import * as T from "@maple-dev/effect-clickhouse/types" -import { inSubquery, param } from "@maple-dev/effect-clickhouse" -import { from, fromQuery, type ColumnAccessor, type CHQuery } from "@maple-dev/effect-clickhouse" -import { unionAll, type CHUnionQuery } from "@maple-dev/effect-clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" +import { compileFnCallCond } from "@maple-dev/effect-orm/clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" +import { inSubquery, param } from "@maple-dev/effect-orm/clickhouse" +import { from, fromQuery, type ColumnAccessor, type CHQuery } from "@maple-dev/effect-orm/clickhouse" +import { unionAll, type CHUnionQuery } from "@maple-dev/effect-orm/clickhouse" import { SESSION_LIVE_WINDOW_SECONDS, type SessionTag } from "@maple/domain/query-engine" -import { ProductEvents, SessionReplays, SessionReplayEvents, TraceDetailSpans } from "../tables" +import { ProductEvents, SessionReplays, SessionReplayEvents, TraceDetailSpans, orgIdParam } from "../tables" import { sessionActivityAggregateQuery, sessionEventMatchQuery } from "./session-events" import { sessionQualityExpr, sessionTagFacet, taggedSessionIds } from "./session-tags" import type { FacetOutput } from "./query-helpers" @@ -43,27 +43,27 @@ function has(array: CH.Expr>, element: CH.Expr): CH.Condi // length(array) — element count. function arrayLength(array: CH.Expr>): CH.Expr { - return CH.compileTypedFnCall("length", T.uint64.schema, array) + return CH.compileTypedFnCall("length", T.uint64.schema, array) } // floor / log2 / pow — plain numeric functions the DSL doesn't export, needed // only by the duration histogram below. Declared here like argMax above. function floor_(value: CH.Expr): CH.Expr { - return CH.compileTypedFnCall("floor", T.float64.schema, value) + return CH.compileTypedFnCall("floor", T.float64.schema, value) } function log2(value: CH.Expr): CH.Expr { - return CH.compileTypedFnCall("log2", T.float64.schema, value) + return CH.compileTypedFnCall("log2", T.float64.schema, value) } function pow(base: number, exponent: CH.Expr): CH.Expr { - return CH.compileTypedFnCall("pow", T.float64.schema, CH.lit(base), exponent) + return CH.compileTypedFnCall("pow", T.float64.schema, CH.lit(base), exponent) } // greatest(value, floor) over a nullable numeric column — clamps and, since // callers pair it with a `> 0` predicate that already excludes NULLs, narrows. function greatestNonNull(value: CH.Expr, floor: number): CH.Expr { - return CH.compileTypedFnCall("greatest", T.float64.schema, value, CH.lit(floor)) + return CH.compileTypedFnCall("greatest", T.float64.schema, value, CH.lit(floor)) } // assumeNotNull(x) — drops the Nullable wrapper for callers whose WHERE has @@ -72,7 +72,7 @@ function assumeNotNull(value: CH.Expr): CH.Expr { // The `Nullable` wrapper goes away in SQL but the codec is left as-is: it // already accepts every non-null the column can produce, and narrowing it // would mean rebuilding a schema this function cannot see inside. - return CH.compileTypedFnCall("assumeNotNull", CH.schemaOf(value), value) + return CH.compileTypedFnCall("assumeNotNull", CH.schemaOf(value), value) } // ifNotFinite(x, fallback) — quantile() over an empty set yields nan, and @@ -96,7 +96,7 @@ function pageVisitSessions(pagePath: string) { return from(ProductEvents) .select(($) => ({ SessionId: $.SessionId })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), $.Kind.eq("navigation"), @@ -115,7 +115,7 @@ function tagFilter( return inSubquery( $.SessionId, taggedSessionIds(tags, ($$) => [ - $$.OrgId.eq(param.string("orgId")), + $$.OrgId.eq(orgIdParam), $$.StartTime.gte(param.dateTimeString("startTime")), $$.StartTime.lte(param.dateTimeString("endTime")), ]), @@ -312,7 +312,7 @@ export function sessionReplaysListQuery( visitorIsNew: argMax($.VisitorIsNew, $.Version), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.StartTime.gte(param.dateTimeString("startTime")), $.StartTime.lte(param.dateTimeString("endTime")), CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), @@ -588,7 +588,7 @@ export function sessionReplaysFacetsQuery( $: ColumnAccessor, exclude?: SessionFacetKey, ): Array => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.StartTime.gte(param.dateTimeString("startTime")), $.StartTime.lte(param.dateTimeString("endTime")), exclude === "service" ? undefined : CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), @@ -640,7 +640,7 @@ export function sessionReplaysFacetsQuery( facetType: CH.lit("page"), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), $.Kind.eq("navigation"), @@ -765,7 +765,7 @@ export function sessionReplaysFacetsQuery( facetType: CH.lit("error"), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.StartTime.gte(param.dateTimeString("startTime")), $.StartTime.lte(param.dateTimeString("endTime")), CH.when(opts.serviceName, (v: string) => $.ServiceName.eq(v)), @@ -868,7 +868,7 @@ export function getSessionReplayQuery(opts: SessionReplayDetailOpts = {}) { lastActivityAt: $.LastActivityAt, })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.SessionId.eq(param.string("sessionId")), CH.when(opts.startTime, (v: string) => $.StartTime.gte(v)), CH.when(opts.endTime, (v: string) => $.StartTime.lte(v)), @@ -939,7 +939,7 @@ export function sessionReplayChunkIndexQuery(opts: SessionReplayChunkIndexOpts = isCheckpoint: $.IsCheckpoint, })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.SessionId.eq(param.string("sessionId")), CH.when(opts.startTime, (v: string) => $.Timestamp.gte(v)), CH.when(opts.endTime, (v: string) => $.Timestamp.lte(v)), @@ -988,7 +988,7 @@ export function sessionReplayEventsQuery(opts: SessionReplayEventsOpts = {}) { isCheckpoint: $.IsCheckpoint, })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.SessionId.eq(param.string("sessionId")), CH.when(opts.startTime, (v: string) => $.Timestamp.gte(v)), CH.when(opts.endTime, (v: string) => $.Timestamp.lte(v)), @@ -1025,7 +1025,7 @@ export function sessionsForTraceQuery(opts: SessionsForTraceOpts) { durationMs: argMax($.DurationMs, $.Version), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.StartTime.gte(param.dateTimeString("startTime")), $.StartTime.lte(param.dateTimeString("endTime")), has($.TraceIds, CH.lit(opts.traceId)), @@ -1105,7 +1105,7 @@ export function sessionTraceSummariesQuery(opts: SessionTraceSummariesOpts) { } }) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TraceId.in_(...opts.traceIds), CH.when(opts.startTime, (v: string) => $.Timestamp.gte(v)), CH.when(opts.endTime, (v: string) => $.Timestamp.lte(v)), diff --git a/packages/query-engine/src/ch/queries/session-tags.ts b/packages/query-engine/src/ch/queries/session-tags.ts index 06e2b08872..158d04927d 100644 --- a/packages/query-engine/src/ch/queries/session-tags.ts +++ b/packages/query-engine/src/ch/queries/session-tags.ts @@ -5,9 +5,9 @@ // column each list row carries. Nothing is stored, so changing a threshold // re-tags history. Tags that need `session_events` belong in a rollup, not here. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import * as T from "@maple-dev/effect-clickhouse/types" -import { from, fromQuery, type ColumnAccessor } from "@maple-dev/effect-clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" +import * as T from "@maple-dev/effect-orm/clickhouse" +import { from, fromQuery, type ColumnAccessor } from "@maple-dev/effect-orm/clickhouse" import { SESSION_QUALITY_TAGS, SESSION_TAG_THRESHOLDS, type SessionTag } from "@maple/domain/query-engine" import { SessionReplays } from "../tables" import { isBotCond } from "../user-agent" @@ -17,7 +17,7 @@ type ReplaysWhere = ($: ReplaysAccessor) => Array // dateDiff(unit, start, end) as whole units; the DSL has no typed wrapper. function dateDiffMs(start: CH.Expr, end: CH.Expr): CH.Expr { - return CH.compileTypedFnCall("dateDiff", T.int64.schema, CH.lit("millisecond"), start, end) + return CH.compileTypedFnCall("dateDiff", T.int64.schema, CH.lit("millisecond"), start, end) } // Only the ended row carries DurationMs. A tab killed without its unload row has diff --git a/packages/query-engine/src/ch/queries/signal-presence.test.ts b/packages/query-engine/src/ch/queries/signal-presence.test.ts index e294aa105c..c6d3dc3850 100644 --- a/packages/query-engine/src/ch/queries/signal-presence.test.ts +++ b/packages/query-engine/src/ch/queries/signal-presence.test.ts @@ -1,10 +1,11 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" -import { compileUnionUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnionUnsafe } from "@maple-dev/effect-orm/clickhouse" import { signalPresenceQuery } from "./signal-presence" +import { OrgId } from "@maple/domain" const params = { - orgId: "org_1", + orgId: OrgId.make("org_1"), // Deliberately sub-hour: `service_usage` is keyed on top-of-hour `Hour`, so // the branches must floor both bounds or a partial hour vanishes. startTime: "2024-01-01 00:23:00", diff --git a/packages/query-engine/src/ch/queries/signal-presence.ts b/packages/query-engine/src/ch/queries/signal-presence.ts index 0dc7b984e2..bb12517a00 100644 --- a/packages/query-engine/src/ch/queries/signal-presence.ts +++ b/packages/query-engine/src/ch/queries/signal-presence.ts @@ -22,9 +22,9 @@ // and `DateTime64` (sessions, product events), and emitting the raw columns // would force a UNION supertype with inconsistent precision. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { from, param, unionAll, type CHUnionQuery } from "@maple-dev/effect-clickhouse" -import { ProductEvents, ServiceUsage, SessionReplays } from "../tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { from, param, unionAll, type CHUnionQuery } from "@maple-dev/effect-orm/clickhouse" +import { ProductEvents, ServiceUsage, SessionReplays, orgIdParam } from "../tables" import { hourFloor } from "./query-helpers" /** The signals an empty state can be missing. Stable — the UI keys copy on these. */ @@ -60,7 +60,7 @@ export function signalPresenceQuery(): CHUnionQuery { lastSeen: CH.toString_(CH.max_($.Hour)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(hourFloor("startTime")), $.Hour.lte(hourFloor("endTime")), $.TraceCount.gt(0), @@ -74,7 +74,7 @@ export function signalPresenceQuery(): CHUnionQuery { lastSeen: CH.toString_(CH.max_($.Hour)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(hourFloor("startTime")), $.Hour.lte(hourFloor("endTime")), $.LogCount.gt(0), @@ -94,7 +94,7 @@ export function signalPresenceQuery(): CHUnionQuery { lastSeen: CH.toString_(CH.max_($.Hour)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Hour.gte(hourFloor("startTime")), $.Hour.lte(hourFloor("endTime")), $.SumMetricCount.add($.GaugeMetricCount) @@ -111,7 +111,7 @@ export function signalPresenceQuery(): CHUnionQuery { lastSeen: CH.toString_(CH.max_($.StartTime)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.StartTime.gte(param.dateTimeString("startTime")), $.StartTime.lte(param.dateTimeString("endTime")), ]) @@ -124,7 +124,7 @@ export function signalPresenceQuery(): CHUnionQuery { lastSeen: CH.toString_(CH.max_($.Timestamp)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), ]) diff --git a/packages/query-engine/src/ch/queries/top-operations.ts b/packages/query-engine/src/ch/queries/top-operations.ts index a4a36db8bd..d79be02e00 100644 --- a/packages/query-engine/src/ch/queries/top-operations.ts +++ b/packages/query-engine/src/ch/queries/top-operations.ts @@ -8,15 +8,15 @@ import { finiteOrZero } from "./format" import { Match } from "effect" import type { TracesMetric } from "@maple/domain/query-engine" -import { compile } from "@maple-dev/effect-clickhouse/sql" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { avg, count, countIf, quantile } from "@maple-dev/effect-clickhouse" -import { if_ } from "@maple-dev/effect-clickhouse" -import { round } from "@maple-dev/effect-clickhouse" -import { param } from "@maple-dev/effect-clickhouse" -import { from } from "@maple-dev/effect-clickhouse" -import { Traces } from "../tables" -import * as T from "@maple-dev/effect-clickhouse/types" +import { compile } from "@maple-dev/effect-orm/sql" +import * as CH from "@maple-dev/effect-orm/expr" +import { avg, count, countIf, quantile } from "@maple-dev/effect-orm/clickhouse" +import { if_ } from "@maple-dev/effect-orm/clickhouse" +import { round } from "@maple-dev/effect-orm/clickhouse" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { from } from "@maple-dev/effect-orm/clickhouse" +import { Traces, orgIdParam } from "../tables" +import * as T from "@maple-dev/effect-orm/clickhouse" /** * Wrap an expression in parentheses. The DSL's arithmetic combinators @@ -83,7 +83,7 @@ export function topOperationsQuery(opts: TopOperationsOpts) { value: finiteOrZero(metricExpr(opts.metric, $)), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.ServiceName.eq(param.string("serviceName")), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), diff --git a/packages/query-engine/src/ch/queries/traces.test.ts b/packages/query-engine/src/ch/queries/traces.test.ts index cbf3812366..e3fa4e5818 100644 --- a/packages/query-engine/src/ch/queries/traces.test.ts +++ b/packages/query-engine/src/ch/queries/traces.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { slowTracesQuery, spanSearchQuery, @@ -13,9 +13,10 @@ import { tracesTimeseriesQuery, } from "./traces" import { canUseTracesAggregatesMv } from "./query-helpers" +import { OrgId } from "@maple/domain" const baseParams = { - orgId: "org_1", + orgId: OrgId.make("org_1"), startTime: "2024-01-01 00:00:00", endTime: "2024-01-02 00:00:00", bucketSeconds: 3600, diff --git a/packages/query-engine/src/ch/queries/traces.ts b/packages/query-engine/src/ch/queries/traces.ts index 6743cfae14..23da89bc74 100644 --- a/packages/query-engine/src/ch/queries/traces.ts +++ b/packages/query-engine/src/ch/queries/traces.ts @@ -3,11 +3,22 @@ // DSL-based query definitions for traces timeseries, breakdown, and list. import type { TracesMetric } from "@maple/domain/query-engine" -import { compileFnCall, subqueryCond, subqueryExpr, untypedSubqueryExpr } from "@maple-dev/effect-clickhouse" -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param } from "@maple-dev/effect-clickhouse" -import { from, fromUnion, unionAll, type CHQuery, type ColumnAccessor } from "@maple-dev/effect-clickhouse" -import type { Table } from "@maple-dev/effect-clickhouse" +import { + compileFnCall, + subqueryCond, + subqueryExpr, + untypedSubqueryExpr, +} from "@maple-dev/effect-orm/clickhouse" +import * as CH from "@maple-dev/effect-orm/expr" +import { param } from "@maple-dev/effect-orm/clickhouse" +import { + from, + fromUnion, + unionAll, + type CHQuery, + type ColumnAccessor, +} from "@maple-dev/effect-orm/clickhouse" +import type { Table } from "@maple-dev/effect-orm/clickhouse" import { ServiceMapSpans, ServiceOverviewSpans, @@ -17,6 +28,7 @@ import { TraceListMv, Traces, TracesAggregatesHourly, + orgIdParam, } from "../tables" import { deploymentEnvExpr, @@ -24,8 +36,8 @@ import { httpResponseStatusCodeExpr, } from "@maple/domain/tinybird/semconv-renames" import { METRIC_NEEDS } from "../../traces-shared" -import type { ColumnDefs } from "@maple-dev/effect-clickhouse/types" -import * as T from "@maple-dev/effect-clickhouse/types" +import type { ColumnDefs } from "@maple-dev/effect-orm/clickhouse" +import * as T from "@maple-dev/effect-orm/clickhouse" import { finalizeTimeseries } from "./series-cap" import { edgeCondition, hourGrain, interiorBounds, interiorConditions, minuteGrain } from "./rollup-splice" import { @@ -544,7 +556,7 @@ export function tracesTimeseriesQuery( >( $: A, ) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), opts.serviceName ? $.ServiceName.eq(opts.serviceName) : undefined, opts.environments?.length ? CH.inList($.DeploymentEnv, opts.environments) : undefined, opts.namespaces?.length ? CH.inList($.ServiceNamespace, opts.namespaces) : undefined, @@ -1139,7 +1151,7 @@ export function slowTracesQuery(opts: SlowTracesOpts) { timestamp: CH.toString_($.Timestamp), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), CH.when(opts.service, (v: string) => $.ServiceName.eq(v)), @@ -1392,7 +1404,7 @@ export function traceSummariesQuery(opts: TraceSummariesOpts) { httpStatusCode: argMin($.HttpStatusCode, $.Timestamp), })) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), matchingTraceIds ? subqueryCond(matchingTraceIds, (sql) => `TraceId IN (${sql})`) : undefined, @@ -1552,7 +1564,7 @@ export interface TraceListOutput { // `compileFnCall`) did not, which cost every query selecting one its row schema. const fromUnixTimestamp64Nano = (nanos: CH.Expr): CH.Expr => - CH.compileTypedFnCall("fromUnixTimestamp64Nano", T.dateTime64String.schema, nanos) + CH.compileTypedFnCall("fromUnixTimestamp64Nano", T.dateTime64String.schema, nanos) const subtractHours = (d: CH.Expr, hours: CH.Expr): CH.Expr => compileFnCall("subtractHours", d, hours) @@ -1610,7 +1622,7 @@ function traceListMvWhereConditions( const services = inclusionValues(opts.serviceName, opts.serviceNames) const spanNames = inclusionValues(opts.spanName, opts.spanNames) const conditions: Array = [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), CH.when(services, (v: readonly string[]) => @@ -1799,7 +1811,7 @@ export function traceListQuery(opts: TraceListOpts) { } }) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), // Padded, not exact: children can start slightly before their root // (clock skew) or outlive the window, but they cannot drift a full // hour — the same ±1h convention as the trace-detail partition hint @@ -1864,7 +1876,7 @@ export function traceServicesByTraceIdsQuery(opts: TraceServicesByTraceIdsOpts) } }) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TraceId.in_(...opts.traceIds), $.Timestamp.gte(param.dateTimeSeconds("startTime")), $.Timestamp.lte(param.dateTimeSeconds("endTime")), @@ -1908,7 +1920,7 @@ export function traceSpanStatsByTraceIdsQuery(opts: TraceSpanStatsByTraceIdsOpts } }) .where(($) => [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.TraceId.in_(...opts.traceIds), $.Timestamp.gte(subtractHours(CH.toDateTime(param.dateTimeString("startTime")), CH.lit(1))), $.Timestamp.lte(addHours(CH.toDateTime(param.dateTimeString("endTime")), CH.lit(1))), diff --git a/packages/query-engine/src/ch/queries/web-analytics-ai.test.ts b/packages/query-engine/src/ch/queries/web-analytics-ai.test.ts index 1618e5b5b8..d2bdb725d0 100644 --- a/packages/query-engine/src/ch/queries/web-analytics-ai.test.ts +++ b/packages/query-engine/src/ch/queries/web-analytics-ai.test.ts @@ -1,13 +1,18 @@ import { describe, expect, it } from "vitest" -import { compileUnsafe } from "@maple-dev/effect-clickhouse" +import { compileUnsafe } from "@maple-dev/effect-orm/clickhouse" import { webAnalyticsAiCrawledPagesQuery, webAnalyticsAiCrawlerFormatsQuery, webAnalyticsAiCrawlersQuery, webAnalyticsAiReferralsQuery, } from "./web-analytics-ai" +import { OrgId } from "@maple/domain" -const params = { orgId: "org_1", startTime: "2026-09-19 00:00:00", endTime: "2026-09-26 00:00:00" } +const params = { + orgId: OrgId.make("org_1"), + startTime: "2026-09-19 00:00:00", + endTime: "2026-09-26 00:00:00", +} const oneLine = (sql: string): string => sql.replace(/\s+/g, " ") diff --git a/packages/query-engine/src/ch/queries/web-analytics-ai.ts b/packages/query-engine/src/ch/queries/web-analytics-ai.ts index acc89f27de..9867b7fd88 100644 --- a/packages/query-engine/src/ch/queries/web-analytics-ai.ts +++ b/packages/query-engine/src/ch/queries/web-analytics-ai.ts @@ -4,19 +4,19 @@ // - crawls: `ai_crawler_requests`, Server spans from AI fetchers, which never // run the browser SDK. Only `host` / `pagePath` narrow them. -import * as CH from "@maple-dev/effect-clickhouse/expr" +import * as CH from "@maple-dev/effect-orm/expr" import { Schema } from "effect" -import { param, from, compileFnCall, compileTypedFnCall } from "@maple-dev/effect-clickhouse" -import type { CHQuery, ColumnAccessor } from "@maple-dev/effect-clickhouse" +import { param, from, compileFnCall, compileTypedFnCall } from "@maple-dev/effect-orm/clickhouse" +import type { CHQuery, ColumnAccessor } from "@maple-dev/effect-orm/clickhouse" import { AI_PRODUCTS } from "@maple/domain/ai-traffic" -import { AiCrawlerRequests, SessionReplays } from "../tables" +import { AiCrawlerRequests, SessionReplays, orgIdParam } from "../tables" import { replaysWhere, type WebAnalyticsFilters } from "./web-analytics" const stringArray = (values: ReadonlyArray) => CH.arrayOf(...values.map((value) => CH.lit(value))) /** `transform(value, from, to, '')`: an exact-match lookup table. */ function lookup(value: CH.Expr, pairs: ReadonlyArray): CH.Expr { - return compileTypedFnCall( + return compileTypedFnCall( "transform", Schema.String, value, @@ -101,7 +101,7 @@ export type WebAnalyticsAiCrawlerFilters = Pick $.Host.eq(v)), diff --git a/packages/query-engine/src/ch/queries/web-analytics.ts b/packages/query-engine/src/ch/queries/web-analytics.ts index c7b4548815..f14db33c7b 100644 --- a/packages/query-engine/src/ch/queries/web-analytics.ts +++ b/packages/query-engine/src/ch/queries/web-analytics.ts @@ -8,10 +8,10 @@ // use `uniq[If](SessionId)`. Page views come from append-only navigation events. // WHERE clauses use only columns written identically to both session versions. -import * as CH from "@maple-dev/effect-clickhouse/expr" -import { param, from, inSubquery, unionAll, compileFnCall } from "@maple-dev/effect-clickhouse" -import type { ColumnAccessor, CHQuery, CHUnionQuery } from "@maple-dev/effect-clickhouse" -import { SessionReplays, SessionEvents, ProductEvents } from "../tables" +import * as CH from "@maple-dev/effect-orm/expr" +import { param, from, inSubquery, unionAll, compileFnCall } from "@maple-dev/effect-orm/clickhouse" +import type { ColumnAccessor, CHQuery, CHUnionQuery } from "@maple-dev/effect-orm/clickhouse" +import { SessionReplays, SessionEvents, ProductEvents, orgIdParam } from "../tables" import { isBotCond } from "../user-agent" import type { FacetOutput } from "./query-helpers" import { SESSION_LIVE_WINDOW_SECONDS, WEB_ANALYTICS_UNSET } from "@maple/domain/query-engine" @@ -161,7 +161,7 @@ function eventConditionsRaw( only?: "host" | "pagePath", ): Array { return [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), $.Type.eq(kind), @@ -196,7 +196,7 @@ function eventConditionsRollup( only?: "host" | "pagePath", ): Array { return [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.Timestamp.gte(param.dateTimeString("startTime")), $.Timestamp.lte(param.dateTimeString("endTime")), $.Kind.eq(kind), @@ -336,7 +336,7 @@ export function replaysWhere( : undefined return [ - $.OrgId.eq(param.string("orgId")), + $.OrgId.eq(orgIdParam), $.StartTime.gte(param.dateTimeString("startTime")), $.StartTime.lte(param.dateTimeString("endTime")), navigationFilter, diff --git a/packages/query-engine/src/ch/raw-sql.ts b/packages/query-engine/src/ch/raw-sql.ts index 4daf08e465..63892a2bab 100644 --- a/packages/query-engine/src/ch/raw-sql.ts +++ b/packages/query-engine/src/ch/raw-sql.ts @@ -4,8 +4,11 @@ // is a policy of the codebase using it, not of a generic query builder. This is // where Maple pins that policy. -import { type CompiledQuery, rawCompiledQuery as rawCompiledQueryUntyped } from "@maple-dev/effect-clickhouse" -import type { CompiledQueryRowSchema, TenantScope } from "@maple-dev/effect-clickhouse" +import { + type CompiledQuery, + rawCompiledQuery as rawCompiledQueryUntyped, +} from "@maple-dev/effect-orm/clickhouse" +import type { CompiledQueryRowSchema, TenantScope } from "@maple-dev/effect-orm/clickhouse" /** * Why a query is handwritten SQL rather than a builder query. diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 4a8784204f..e0cadf4c92 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -3,8 +3,7 @@ // Derived from packages/domain/src/tinybird/datasources.ts // These define the ClickHouse table schemas used by the query DSL. -import { type ColumnDefs, type Table, table as chTable } from "@maple-dev/effect-clickhouse" -import * as T from "@maple-dev/effect-clickhouse/types" +import * as T from "@maple-dev/effect-orm/clickhouse" import { OrgId, SpanId, TraceId } from "@maple/domain" /** @@ -35,23 +34,26 @@ export type StringMap = T.CHMap * Identity columns carry their branded domain schemas, so a query that SELECTs * one derives a row schema whose decoded type is the brand — no declared * `rowSchema` needed just to keep `OrgId`/`TraceId`/`SpanId` in the output - * type. Comparisons still take plain strings/params: the builder widens a - * branded column's comparison type to its primitive. + * type. Comparisons take the brand too: compare `OrgId` with `orgIdParam`. */ -const orgId = T.custom("String", OrgId) -const traceId = T.custom("String", TraceId) -const spanId = T.custom("String", SpanId) +export const orgId = T.brand(T.string, OrgId) +const traceId = T.brand(T.string, TraceId) +const spanId = T.brand(T.string, SpanId) + +/** The tenant param every warehouse query filters `OrgId` on. */ +export const orgIdParam = T.param.of(orgId, "orgId") /** * Every Maple warehouse table is keyed by `OrgId`, so tenancy is declared once * here rather than at 37 call sites. Requiring the column in the signature is * the point: a new table without one is a type error, not a table that silently - * compiles every query as `cross-tenant`. + * compiles every query as `cross-tenant`. The DDL is owned by Tinybird's + * `datasources.ts`, so the tables are external to effect-orm's migrations. */ -const table = ( +const table = ( name: Name, columns: Columns, -): Table => chTable(name, columns, { tenantColumn: "OrgId" }) +) => T.table(name, { external: true, columns, tenantColumn: "OrgId" }) export const Traces = table("traces", { OrgId: orgId, diff --git a/packages/query-engine/src/ch/user-agent.ts b/packages/query-engine/src/ch/user-agent.ts index f7aa993f5a..0b32932534 100644 --- a/packages/query-engine/src/ch/user-agent.ts +++ b/packages/query-engine/src/ch/user-agent.ts @@ -18,7 +18,7 @@ // latter. Reaching the non-executing crawlers needs `http.user_agent` off // Server spans, which is a different source and a different feature. -import * as CH from "@maple-dev/effect-clickhouse/expr" +import * as CH from "@maple-dev/effect-orm/expr" /** * What a bot is *for*, which is the axis a site owner acts on: an AI training diff --git a/packages/query-engine/src/dashboard-variables/interpolate.ts b/packages/query-engine/src/dashboard-variables/interpolate.ts index e66909c1cb..57107fa0e8 100644 --- a/packages/query-engine/src/dashboard-variables/interpolate.ts +++ b/packages/query-engine/src/dashboard-variables/interpolate.ts @@ -16,7 +16,7 @@ // references an "All" selection is dropped entirely (All = don't filter). // - everything else gets plain text substitution. -import { escapeClickHouseString } from "@maple-dev/effect-clickhouse/sql" +import { escapeClickHouseString } from "@maple-dev/effect-orm/sql" import { splitWhereClause } from "@maple/domain/where-clause" /** Sentinel for the "All" selection, both in URLs and resolved values. */ diff --git a/packages/query-engine/src/execution/executor.test.ts b/packages/query-engine/src/execution/executor.test.ts index 217b79b004..6a26b6dfe7 100644 --- a/packages/query-engine/src/execution/executor.test.ts +++ b/packages/query-engine/src/execution/executor.test.ts @@ -60,7 +60,7 @@ const chdbConfig: ResolvedWarehouseConfig = { // listRuleChecksQuery declares .route("ingest") at its definition — // alert_checks only exists in the managed ingest pipeline. const compiled = compileUnsafe(listRuleChecksQuery({ limit: 1 }), { - orgId: "org_test", + orgId: OrgId.make("org_test"), ruleId: "rule_test", }) const _ingestRouteIsPartOfTheCompiledType: "ingest" = compiled.route diff --git a/packages/query-engine/src/execution/executor.ts b/packages/query-engine/src/execution/executor.ts index ef06eac0e7..f680982d0f 100644 --- a/packages/query-engine/src/execution/executor.ts +++ b/packages/query-engine/src/execution/executor.ts @@ -13,7 +13,7 @@ import { } from "@maple/domain/http" import type { WarehouseQueryName } from "@maple/domain/warehouse-queries" import { compilePipeQuery, type CompiledQuery, type TenantScope } from "../ch" -import { parseStatement, withFormat, withSettings } from "@maple-dev/effect-clickhouse/sql" +import { parseStatement, withFormat, withSettings } from "@maple-dev/effect-orm/sql" import type { WarehouseDriverError } from "./driver-error" import type { WarehouseExecutorApi } from "../observability" import { diff --git a/packages/query-engine/src/execution/fingerprint.ts b/packages/query-engine/src/execution/fingerprint.ts index 1614b03635..2f0b1a9538 100644 --- a/packages/query-engine/src/execution/fingerprint.ts +++ b/packages/query-engine/src/execution/fingerprint.ts @@ -1,4 +1,4 @@ -import { splitTerminalClauses } from "@maple-dev/effect-clickhouse/sql" +import { splitTerminalClauses } from "@maple-dev/effect-orm/sql" /** * Cap traced SQL at 16 KB. OTel's default attribute size limit is 32 KB, and diff --git a/packages/query-engine/src/execution/ports.ts b/packages/query-engine/src/execution/ports.ts index 28eed15746..b00163b59b 100644 --- a/packages/query-engine/src/execution/ports.ts +++ b/packages/query-engine/src/execution/ports.ts @@ -1,5 +1,5 @@ import type { Effect, Option } from "effect" -import type { ClickHouseStatement } from "@maple-dev/effect-clickhouse/sql" +import type { ClickHouseStatement } from "@maple-dev/effect-orm/sql" import type { OrgId, UserId } from "@maple/domain" import type { RawSqlValidationError, diff --git a/packages/query-engine/src/local.ts b/packages/query-engine/src/local.ts index d84accfabf..ff30c1cfdc 100644 --- a/packages/query-engine/src/local.ts +++ b/packages/query-engine/src/local.ts @@ -10,6 +10,13 @@ import { Effect, Schema } from "effect" import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http" +import { OrgId } from "@maple/domain/primitives" + +/** + * Local mode's single synthetic tenant. The ingest binary writes every span, + * log and metric under this `OrgId`, so every local query compiles with it. + */ +export const LOCAL_ORG_ID = OrgId.make("local") /** * The local server refused the query. Its own tag, and structured fields, so diff --git a/packages/query-engine/src/observability/WarehouseExecutor.ts b/packages/query-engine/src/observability/WarehouseExecutor.ts index 2bac9e8872..34d3cb13e0 100644 --- a/packages/query-engine/src/observability/WarehouseExecutor.ts +++ b/packages/query-engine/src/observability/WarehouseExecutor.ts @@ -1,4 +1,5 @@ import { Context, type Effect, type Option } from "effect" +import type { OrgId } from "@maple/domain" import type { WarehouseError } from "@maple/domain/http/warehouse-errors" import type { WarehouseQueryName } from "@maple/domain/warehouse-queries" import type { CompiledQueryInput } from "../ch" @@ -14,7 +15,7 @@ export type WarehouseExecutorError = WarehouseError export interface WarehouseExecutorApi { /** The org ID for the current tenant — needed for raw SQL queries. */ - readonly orgId: string + readonly orgId: OrgId readonly query: ( pipe: WarehouseQueryName, diff --git a/packages/query-engine/src/observability/error-detail.test.ts b/packages/query-engine/src/observability/error-detail.test.ts index 133bd8d2fb..9cae0a9233 100644 --- a/packages/query-engine/src/observability/error-detail.test.ts +++ b/packages/query-engine/src/observability/error-detail.test.ts @@ -4,6 +4,7 @@ import { errorDetail } from "./error-detail" import { WarehouseExecutor } from "./WarehouseExecutor" import type { WarehouseExecutorApi } from "./WarehouseExecutor" import { compiledQueryOf } from "../execution/compiled-input" +import { OrgId } from "@maple/domain" interface CapturedCalls { pipeCalls: Array<{ pipe: string; params: Record }> @@ -40,7 +41,7 @@ const makeMockExecutor = ( compiledRows: CompiledRows = {}, logsData: ReadonlyArray = [], ): WarehouseExecutorApi => ({ - orgId: "org_test", + orgId: OrgId.make("org_test"), compiledQuery: (compiled, options) => { const query = compiledQueryOf(compiled) captured.pipeCalls.push({ pipe: options?.context ?? "compiled", params: { sql: query.sql } }) diff --git a/packages/query-engine/src/observability/explore-attributes.test.ts b/packages/query-engine/src/observability/explore-attributes.test.ts index 65cb3fdc2b..36b2eb9883 100644 --- a/packages/query-engine/src/observability/explore-attributes.test.ts +++ b/packages/query-engine/src/observability/explore-attributes.test.ts @@ -4,6 +4,7 @@ import { exploreAttributeKeys, exploreAttributeValues } from "./explore-attribut import { WarehouseExecutor } from "./WarehouseExecutor" import type { WarehouseExecutorApi } from "./WarehouseExecutor" import { compiledQueryOf } from "../execution/compiled-input" +import { OrgId } from "@maple/domain" interface CapturedCalls { pipeCalls: Array<{ pipe: string; params: Record }> @@ -13,7 +14,7 @@ const makeMockExecutor = ( captured: CapturedCalls, rows: ReadonlyArray> = [], ): WarehouseExecutorApi => ({ - orgId: "org_test", + orgId: OrgId.make("org_test"), compiledQuery: (compiled) => compiledQueryOf(compiled).decodeRows([]).pipe(Effect.orDie), compiledQueryFirst: (compiled) => compiledQueryOf(compiled).decodeFirstRow([]).pipe(Effect.orDie), query: (pipe: string, params: Record) => { diff --git a/packages/query-engine/src/observability/find-slow-traces.test.ts b/packages/query-engine/src/observability/find-slow-traces.test.ts index d51f2c1532..7304d13766 100644 --- a/packages/query-engine/src/observability/find-slow-traces.test.ts +++ b/packages/query-engine/src/observability/find-slow-traces.test.ts @@ -5,6 +5,7 @@ import { findSlowTraces } from "./find-slow-traces" import { WarehouseExecutor } from "./WarehouseExecutor" import type { WarehouseExecutorApi } from "./WarehouseExecutor" import { compiledQueryOf } from "../execution/compiled-input" +import { OrgId } from "@maple/domain" interface CapturedCalls { pipeCalls: Array<{ pipe: string; params: Record }> @@ -15,7 +16,7 @@ const makeMockExecutor = ( slowRows: ReadonlyArray> = [], statsRows: ReadonlyArray> = [], ): WarehouseExecutorApi => ({ - orgId: "org_test", + orgId: OrgId.make("org_test"), compiledQuery: (compiled) => compiledQueryOf(compiled).decodeRows([]).pipe(Effect.orDie), compiledQueryFirst: (compiled) => compiledQueryOf(compiled).decodeFirstRow([]).pipe(Effect.orDie), query: (pipe: string, params: Record) => { @@ -127,7 +128,7 @@ describe("findSlowTraces", () => { it.effect("propagates warehouse errors from the executor", () => Effect.gen(function* () { const failingExecutor: WarehouseExecutorApi = { - orgId: "org_test", + orgId: OrgId.make("org_test"), compiledQuery: (compiled) => compiledQueryOf(compiled).decodeRows([]).pipe(Effect.orDie), compiledQueryFirst: (compiled) => compiledQueryOf(compiled).decodeFirstRow([]).pipe(Effect.orDie), diff --git a/packages/query-engine/src/observability/inspect-trace.test.ts b/packages/query-engine/src/observability/inspect-trace.test.ts index 8a0f5b071a..4eea171622 100644 --- a/packages/query-engine/src/observability/inspect-trace.test.ts +++ b/packages/query-engine/src/observability/inspect-trace.test.ts @@ -5,6 +5,7 @@ import { inspectTrace } from "./inspect-trace" import { WarehouseExecutor } from "./WarehouseExecutor" import type { WarehouseExecutorApi } from "./WarehouseExecutor" import { compiledQueryOf } from "../execution/compiled-input" +import { OrgId } from "@maple/domain" const TRACE_ID = "0af7651916cd43dd8448eb211c80319c" const NOW = Date.parse("2026-04-10T00:00:00Z") @@ -44,7 +45,7 @@ const makeExecutor = ( captured: Captured, opts: { window?: { start: string; end: string }; probeTimestamp?: string }, ): WarehouseExecutorApi => ({ - orgId: "org_test", + orgId: OrgId.make("org_test"), compiledQuery: (compiled) => compiledQueryOf(compiled).decodeRows([]).pipe(Effect.orDie), compiledQueryFirst: (compiled) => { const query = compiledQueryOf(compiled) diff --git a/packages/query-engine/src/observability/inspect-trace.ts b/packages/query-engine/src/observability/inspect-trace.ts index 4e6675f624..6f0c294dad 100644 --- a/packages/query-engine/src/observability/inspect-trace.ts +++ b/packages/query-engine/src/observability/inspect-trace.ts @@ -161,7 +161,7 @@ export const inspectTrace = Effect.fn("Observability.inspectTrace")(function* ( ? ([firstRead, { ...queryRangeToScan(range), widened: false }] as const) : yield* Effect.gen(function* () { const probe = yield* executor.compiledQueryFirst( - CH.compile(CH.traceTimeProbeQuery({ traceId, narrowByTime: true }), { + CH.compile(CH.recentTraceTimeProbeQuery({ traceId }), { orgId: executor.orgId, startTime: formatWarehouseDateTime(probeStartMs), }), diff --git a/packages/query-engine/src/observability/search-logs.test.ts b/packages/query-engine/src/observability/search-logs.test.ts index effed3af11..e43f643460 100644 --- a/packages/query-engine/src/observability/search-logs.test.ts +++ b/packages/query-engine/src/observability/search-logs.test.ts @@ -24,7 +24,7 @@ interface CapturedCalls { } const makeMockExecutor = (captured: CapturedCalls): WarehouseExecutorApi => ({ - orgId: "org_test", + orgId: OrgId.make("org_test"), compiledQuery: (compiled) => compiledQueryOf(compiled).decodeRows([]).pipe(Effect.orDie), compiledQueryFirst: (compiled) => compiledQueryOf(compiled).decodeFirstRow([]).pipe(Effect.orDie), query: ( diff --git a/packages/query-engine/src/observability/session-replays.test.ts b/packages/query-engine/src/observability/session-replays.test.ts index 43a415d059..edcdb89623 100644 --- a/packages/query-engine/src/observability/session-replays.test.ts +++ b/packages/query-engine/src/observability/session-replays.test.ts @@ -5,6 +5,7 @@ import { getSessionTraces } from "./session-replays" import { WarehouseExecutor } from "./WarehouseExecutor" import type { WarehouseExecutorApi } from "./WarehouseExecutor" import { compiledQueryOf } from "../execution/compiled-input" +import { OrgId } from "@maple/domain" interface Captured { sqls: string[] @@ -29,7 +30,7 @@ const makeExecutor = (captured: Captured, responses: MockResponses): WarehouseEx return responses.detail ?? [] } return { - orgId: "org_test", + orgId: OrgId.make("org_test"), query: () => Effect.succeed({ data: [] as ReadonlyArray }), compiledQuery: ((compiled) => { captured.sqls.push(compiledQueryOf(compiled).sql) @@ -202,7 +203,7 @@ describe("getSessionTraces", () => { it.effect("propagates warehouse errors from the executor", () => Effect.gen(function* () { const failing: WarehouseExecutorApi = { - orgId: "org_test", + orgId: OrgId.make("org_test"), query: () => Effect.succeed({ data: [] }), compiledQuery: () => Effect.fail( diff --git a/packages/query-engine/src/observability/span-detail.test.ts b/packages/query-engine/src/observability/span-detail.test.ts index 8c2a665491..e8cdf0208c 100644 --- a/packages/query-engine/src/observability/span-detail.test.ts +++ b/packages/query-engine/src/observability/span-detail.test.ts @@ -3,6 +3,7 @@ import { Effect, Layer } from "effect" import { spanDetail } from "./span-detail" import { WarehouseExecutor, type WarehouseExecutorApi } from "./WarehouseExecutor" import { compiledQueryOf } from "../execution/compiled-input" +import { OrgId } from "@maple/domain" const TRACE_ID = "0af7651916cd43dd8448eb211c80319c" const SPAN_ID = "b7ad6b7169203331" @@ -24,7 +25,7 @@ const detailRow = { /** Answers only unbounded lookups, as if the hinted window missed the span. */ const makeExecutor = (sqls: string[]): WarehouseExecutorApi => ({ - orgId: "org_test", + orgId: OrgId.make("org_test"), compiledQuery: (compiled) => compiledQueryOf(compiled).decodeRows([]).pipe(Effect.orDie), compiledQueryFirst: (compiled) => { const query = compiledQueryOf(compiled) diff --git a/packages/query-engine/src/profiles/query-profile.ts b/packages/query-engine/src/profiles/query-profile.ts index f07e91b735..9b9d8d5883 100644 --- a/packages/query-engine/src/profiles/query-profile.ts +++ b/packages/query-engine/src/profiles/query-profile.ts @@ -1,4 +1,4 @@ -import { parseStatement, renderStatement, withSettings } from "@maple-dev/effect-clickhouse/sql" +import { parseStatement, renderStatement, withSettings } from "@maple-dev/effect-orm/sql" /** * ClickHouse query settings forwarded via inline `SETTINGS` clause. diff --git a/packages/query-engine/src/registry/logs.ts b/packages/query-engine/src/registry/logs.ts index df10f93a49..954623f77f 100644 --- a/packages/query-engine/src/registry/logs.ts +++ b/packages/query-engine/src/registry/logs.ts @@ -4,6 +4,7 @@ import * as CH from "../ch" import { LOGS_BODY_SEARCH_SETTINGS } from "../profiles" import { timeRangeCache } from "../runtime/cache-policy" import { defineQuery, makeTimeBucketQueryCachePolicy } from "./query-definition" +import type { OrgId } from "@maple/domain" export const logsQueryOptions = (filters: LogsFilters | undefined) => { const deploymentEnv = filters?.deploymentEnvMatchMode @@ -77,7 +78,7 @@ export const logsCount = defineQuery({ cache: timeRangeCache, capabilityAware: true, settings: (input: LogsCountInput) => (input.filters?.search ? LOGS_BODY_SEARCH_SETTINGS : undefined), - compile: (input: LogsCountInput, orgId: string, capabilities) => + compile: (input: LogsCountInput, orgId: OrgId, capabilities) => CH.compile( CH.logsCountQuery({ ...logsQueryOptions(input.filters), @@ -97,7 +98,7 @@ export const logsTimeseries = defineQuery({ fallback: 15, }), capabilityAware: true, - compile: (input: LogsTimeseriesInput, orgId: string, capabilities) => + compile: (input: LogsTimeseriesInput, orgId: OrgId, capabilities) => CH.compile( CH.logsTimeseriesQuery({ ...logsQueryOptions(input.filters), diff --git a/packages/query-engine/src/registry/product-events.ts b/packages/query-engine/src/registry/product-events.ts index c0def734b0..e08c3660ce 100644 --- a/packages/query-engine/src/registry/product-events.ts +++ b/packages/query-engine/src/registry/product-events.ts @@ -17,6 +17,7 @@ import type { import * as CH from "../ch" import { timeRangeCache } from "../runtime/query-engine" import { defineQuery, makeTimeBucketQueryCachePolicy } from "./query-definition" +import type { OrgId } from "@maple/domain" // Product-event funnels read `product_events` only — server and mobile rows have // no raw `session_events` counterpart, so unlike the web-analytics pairs there is @@ -60,7 +61,7 @@ export const productEventsFunnel = defineQuery({ id: "productEventsFunnel", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ProductEventsFunnelRequest, orgId: string) => + compile: (payload: ProductEventsFunnelRequest, orgId: OrgId) => CH.compile(CH.productEventsFunnelQuery(productEventsFunnelOpts(payload)), { orgId, startTime: payload.startTime, @@ -72,7 +73,7 @@ export const productEventsFunnelBreakdown = defineQuery({ id: "productEventsFunnelBreakdown", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ProductEventsFunnelBreakdownRequest, orgId: string) => + compile: (payload: ProductEventsFunnelBreakdownRequest, orgId: OrgId) => CH.compile( CH.productEventsFunnelBreakdownQuery({ ...productEventsFunnelOpts(payload), @@ -87,7 +88,7 @@ export const productEventsFunnelTiming = defineQuery({ id: "productEventsFunnelTiming", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ProductEventsFunnelTimingRequest, orgId: string) => + compile: (payload: ProductEventsFunnelTimingRequest, orgId: OrgId) => CH.compile(CH.productEventsFunnelTimingQuery(productEventsFunnelOpts(payload)), { orgId, startTime: payload.startTime, @@ -99,7 +100,7 @@ export const productEventsFunnelLeavers = defineQuery({ id: "productEventsFunnelLeavers", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ProductEventsFunnelLeaversRequest, orgId: string) => + compile: (payload: ProductEventsFunnelLeaversRequest, orgId: OrgId) => CH.compile(CH.productEventsFunnelLeaversQuery(productEventsFunnelOpts(payload)), { orgId, startTime: payload.startTime, @@ -124,7 +125,7 @@ export const productEventsPaths = defineQuery({ id: "productEventsPaths", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ProductEventsPathsRequest, orgId: string) => + compile: (payload: ProductEventsPathsRequest, orgId: OrgId) => CH.compile(CH.productEventsPathsQuery(productEventsPathsOpts(payload)), { orgId, startTime: payload.startTime, @@ -136,7 +137,7 @@ export const productEventNames = defineQuery({ id: "productEventNames", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ProductEventNamesRequest, orgId: string) => + compile: (payload: ProductEventNamesRequest, orgId: OrgId) => CH.compile( CH.productEventNamesQuery({ filters: productEventsFilters(payload), @@ -154,7 +155,7 @@ export const productEventsForTrace = defineQuery({ id: "productEventsForTrace", profile: "list", cache: 60, - compile: (payload: ProductEventsForTraceRequest, orgId: string) => + compile: (payload: ProductEventsForTraceRequest, orgId: OrgId) => CH.compile(CH.productEventsForTraceQuery({ limit: payload.limit ?? 50 }), { orgId, startTime: payload.startTime, @@ -167,7 +168,7 @@ export const productEventTraceSamples = defineQuery({ id: "productEventTraceSamples", profile: "list", cache: 60, - compile: (payload: ProductEventTraceSamplesRequest, orgId: string) => + compile: (payload: ProductEventTraceSamplesRequest, orgId: OrgId) => CH.compile(CH.productEventTraceSamplesQuery({ limit: payload.limit ?? 20 }), { orgId, startTime: payload.startTime, @@ -242,7 +243,7 @@ export const productEventsTimeseries = defineQuery({ identity: ({ metric, filters, groupBy, seriesLimit }) => ({ metric, filters, groupBy, seriesLimit }), fallback: 15, }), - compile: (input: ProductEventsTimeseriesInput, orgId: string) => + compile: (input: ProductEventsTimeseriesInput, orgId: OrgId) => CH.compile( CH.productEventsTimeseriesQuery({ ...productEventsQueryOptions(input.filters), @@ -265,7 +266,7 @@ export const productEventsBreakdown = defineQuery({ id: "productEventsBreakdown", profile: "aggregation", cache: timeRangeCache, - compile: (input: ProductEventsBreakdownInput, orgId: string) => + compile: (input: ProductEventsBreakdownInput, orgId: OrgId) => CH.compile( CH.productEventsBreakdownQuery({ ...productEventsQueryOptions(input.query.filters), @@ -287,7 +288,7 @@ export const productEventsList = defineQuery({ id: "productEventsList", profile: "list", cache: timeRangeCache, - compile: (input: ProductEventsListInput, orgId: string) => + compile: (input: ProductEventsListInput, orgId: OrgId) => CH.compile( CH.productEventsListQuery({ ...productEventsQueryOptions(input.query.filters), diff --git a/packages/query-engine/src/registry/queries.ts b/packages/query-engine/src/registry/queries.ts index c2ee8c2116..373031f8e1 100644 --- a/packages/query-engine/src/registry/queries.ts +++ b/packages/query-engine/src/registry/queries.ts @@ -73,6 +73,7 @@ export { productEventsBreakdown, productEventsList, } from "./product-events" +import type { OrgId } from "@maple/domain" /** * Declarative compile, execution, and cache policy. Handlers retain response @@ -90,7 +91,7 @@ export const errorsByType = defineQuery({ id: "errorsByType", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ErrorsByTypeRequest, orgId: string) => + compile: (payload: ErrorsByTypeRequest, orgId: OrgId) => CH.compile( CH.errorsByTypeQuery({ rootOnly: payload.rootOnly, @@ -113,7 +114,7 @@ export const errorsTimeseries = defineQuery({ id: "errorsTimeseries", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ErrorsTimeseriesRequest, orgId: string) => + compile: (payload: ErrorsTimeseriesRequest, orgId: OrgId) => CH.compile( CH.errorsTimeseriesQuery({ fingerprintHash: payload.fingerprintHash, @@ -133,7 +134,7 @@ export const errorsSpark = defineQuery({ id: "errorsSpark", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ErrorsSparkRequest, orgId: string) => + compile: (payload: ErrorsSparkRequest, orgId: OrgId) => CH.compile( CH.errorsSparkQuery({ fingerprintHashes: payload.fingerprintHashes, @@ -160,7 +161,7 @@ export const errorsSummary = defineQuery({ id: "errorsSummary", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ErrorsSummaryRequest, orgId: string) => + compile: (payload: ErrorsSummaryRequest, orgId: OrgId) => CH.compile( CH.errorsSummaryQuery({ rootOnly: payload.rootOnly, @@ -178,7 +179,7 @@ export const errorRateByService = defineQuery({ id: "errorRateByService", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ErrorRateByServiceRequest, orgId: string) => + compile: (payload: ErrorRateByServiceRequest, orgId: OrgId) => CH.compile(CH.errorRateByServiceQuery(), { orgId, startTime: payload.startTime, @@ -195,7 +196,7 @@ export const serviceOverview = defineQuery({ // renders a services list with no commits and no latency. The bump is the only // thing standing between a deploy and that. cache: makeTimeRangeCachePolicy({ version: 3 }), - compile: (payload: ServiceOverviewRequest, orgId: string) => + compile: (payload: ServiceOverviewRequest, orgId: OrgId) => CH.compile( CH.serviceOverviewQuery({ environments: payload.environments, @@ -213,7 +214,7 @@ export const errorDetailTraces = defineQuery({ id: "errorDetailTraces", profile: "list", cache: timeRangeCache, - compile: (payload: ErrorDetailTracesRequest, orgId: string) => + compile: (payload: ErrorDetailTracesRequest, orgId: OrgId) => CH.compile( CH.errorDetailTracesQuery({ fingerprintHash: payload.fingerprintHash, @@ -229,7 +230,7 @@ export const serviceHealthSnapshot = defineQuery({ id: "serviceHealthSnapshot", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ServiceHealthSnapshotRequest, orgId: string) => + compile: (payload: ServiceHealthSnapshotRequest, orgId: OrgId) => CH.compile(CH.serviceHealthSnapshotQuery({ environments: payload.environments }), { orgId, startTime: payload.startTime, @@ -241,7 +242,7 @@ export const serviceHealthBaseline = defineQuery({ id: "serviceHealthBaseline", profile: "aggregation", cache: 3600, - compile: (payload: ServiceHealthBaselineRequest, orgId: string) => + compile: (payload: ServiceHealthBaselineRequest, orgId: OrgId) => CH.compile( CH.serviceHealthBaselineQuery({ environments: payload.environments, @@ -255,7 +256,7 @@ export const serviceApdex = defineQuery({ id: "serviceApdex", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ServiceApdexRequest, orgId: string) => { + compile: (payload: ServiceApdexRequest, orgId: OrgId) => { const bucketSeconds = payload.bucketSeconds ?? 60 return CH.compile( CH.serviceApdexTimeseriesQuery({ @@ -277,7 +278,7 @@ export const serviceDependenciesForService = defineQuery({ id: "serviceDependenciesForService", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ServiceDependenciesForServiceRequest, orgId: string) => + compile: (payload: ServiceDependenciesForServiceRequest, orgId: OrgId) => CH.compile( CH.serviceDependenciesForServiceQuery({ serviceName: payload.serviceName, @@ -295,7 +296,7 @@ export const serviceDbEdgesForService = defineQuery({ id: "serviceDbEdgesForService", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ServiceDbEdgesForServiceRequest, orgId: string) => + compile: (payload: ServiceDbEdgesForServiceRequest, orgId: OrgId) => CH.compile( CH.serviceDbEdgesForServiceQuery({ serviceName: payload.serviceName, @@ -317,7 +318,7 @@ export const listLogs = defineQuery({ settings: (payload: ListLogsRequest) => (payload.search ? LOGS_BODY_SEARCH_SETTINGS : undefined), cache: timeRangeCache, capabilityAware: true, - compile: (payload: ListLogsRequest, orgId: string, capabilities) => + compile: (payload: ListLogsRequest, orgId: OrgId, capabilities) => CH.compile( CH.logsListQuery({ attributeIndexMode: attributeIndexMode(capabilities, "logs"), @@ -367,7 +368,7 @@ export const listMetrics = defineQuery({ id: "listMetrics", profile: "discovery", cache: 60, - compile: (payload: ListMetricsRequest, orgId: string) => + compile: (payload: ListMetricsRequest, orgId: OrgId) => CH.compile( CH.listMetricsQuery({ serviceName: payload.service, @@ -384,7 +385,7 @@ export const metricsSummary = defineQuery({ id: "metricsSummary", profile: "discovery", cache: 60, - compile: (payload: MetricsSummaryRequest, orgId: string) => + compile: (payload: MetricsSummaryRequest, orgId: OrgId) => CH.compile(CH.metricsSummaryQuery({ serviceName: payload.service }), { orgId, startTime: payload.startTime, @@ -402,7 +403,7 @@ export const infraPresence = defineQuery({ id: "infraPresence", profile: "discovery", cache: 300, - compile: (payload: InfraPresenceRequest, orgId: string) => + compile: (payload: InfraPresenceRequest, orgId: OrgId) => CH.compileUnion(CH.infraPresenceQuery(), { orgId, startTime: payload.startTime, @@ -414,7 +415,7 @@ export const listHosts = defineQuery({ id: "listHosts", profile: "list", cache: timeRangeCache, - compile: (payload: ListHostsRequest, orgId: string) => + compile: (payload: ListHostsRequest, orgId: OrgId) => CH.compile( CH.listHostsQuery({ search: payload.search, @@ -429,7 +430,7 @@ export const hostDetailSummary = defineQuery({ id: "hostDetailSummary", profile: "aggregation", cache: timeRangeCache, - compile: (payload: HostDetailSummaryRequest, orgId: string) => + compile: (payload: HostDetailSummaryRequest, orgId: OrgId) => CH.compile(CH.hostDetailSummaryQuery({ hostName: payload.hostName }), { orgId, startTime: payload.startTime, @@ -441,7 +442,7 @@ export const podsSummary = defineQuery({ id: "podsSummary", profile: "aggregation", cache: timeRangeCache, - compile: (payload: PodsSummaryRequest, orgId: string) => + compile: (payload: PodsSummaryRequest, orgId: OrgId) => CH.compile( CH.listPodsSummaryQuery({ namespaces: payload.namespaces, @@ -456,7 +457,7 @@ export const podDetailSummary = defineQuery({ id: "podDetailSummary", profile: "aggregation", cache: timeRangeCache, - compile: (payload: PodDetailSummaryRequest, orgId: string) => + compile: (payload: PodDetailSummaryRequest, orgId: OrgId) => CH.compile(CH.podDetailSummaryQuery({ podName: payload.podName, namespace: payload.namespace }), { orgId, startTime: payload.startTime, @@ -468,7 +469,7 @@ export const listNodes = defineQuery({ id: "listNodes", profile: "list", cache: timeRangeCache, - compile: (payload: ListNodesRequest, orgId: string) => + compile: (payload: ListNodesRequest, orgId: OrgId) => CH.compile( CH.listNodesQuery({ search: payload.search, @@ -486,7 +487,7 @@ export const nodeDetailSummary = defineQuery({ id: "nodeDetailSummary", profile: "aggregation", cache: timeRangeCache, - compile: (payload: NodeDetailSummaryRequest, orgId: string) => + compile: (payload: NodeDetailSummaryRequest, orgId: OrgId) => CH.compile(CH.nodeDetailSummaryQuery({ nodeName: payload.nodeName }), { orgId, startTime: payload.startTime, @@ -498,7 +499,7 @@ export const listWorkloads = defineQuery({ id: "listWorkloads", profile: "list", cache: timeRangeCache, - compile: (payload: ListWorkloadsRequest, orgId: string) => + compile: (payload: ListWorkloadsRequest, orgId: OrgId) => CH.compile( CH.listWorkloadsQuery({ kind: payload.kind, @@ -519,7 +520,7 @@ export const workloadDetailSummary = defineQuery({ id: "workloadDetailSummary", profile: "aggregation", cache: timeRangeCache, - compile: (payload: WorkloadDetailSummaryRequest, orgId: string) => + compile: (payload: WorkloadDetailSummaryRequest, orgId: OrgId) => CH.compile( CH.workloadDetailSummaryQuery({ kind: payload.kind, @@ -537,7 +538,7 @@ export const releasesList = defineQuery({ id: "releasesList", profile: "list", cache: timeRangeCache, - compile: (payload: ReleasesListRequest, orgId: string) => + compile: (payload: ReleasesListRequest, orgId: OrgId) => CH.compile( CH.releasesListQuery({ environments: payload.environments, @@ -554,7 +555,7 @@ export const releasesTimeline = defineQuery({ id: "releasesTimeline", profile: "list", cache: timeRangeCache, - compile: (payload: ReleasesListRequest, orgId: string) => + compile: (payload: ReleasesListRequest, orgId: OrgId) => CH.compile( CH.releasesTimelineQuery({ environments: payload.environments, @@ -576,7 +577,7 @@ export const releaseVersions = defineQuery({ id: "releaseVersions", profile: "list", cache: timeRangeCache, - compile: (payload: ReleaseDetailRequest, orgId: string) => + compile: (payload: ReleaseDetailRequest, orgId: OrgId) => CH.compile( CH.releasesListQuery({ serviceName: payload.serviceName, @@ -592,7 +593,7 @@ export const releaseTimeline = defineQuery({ id: "releaseTimeline", profile: "list", cache: timeRangeCache, - compile: (payload: ReleaseDetailRequest, orgId: string) => + compile: (payload: ReleaseDetailRequest, orgId: OrgId) => CH.compile( CH.releasesTimelineQuery({ serviceName: payload.serviceName, @@ -612,7 +613,7 @@ export const releaseErrorFingerprints = defineQuery({ id: "releaseErrorFingerprints", profile: "list", cache: timeRangeCache, - compile: (payload: ReleaseDetailRequest, orgId: string) => + compile: (payload: ReleaseDetailRequest, orgId: OrgId) => CH.compile( CH.releaseErrorFingerprintsQuery({ serviceName: payload.serviceName, @@ -640,7 +641,7 @@ export const serviceReleases = defineQuery({ readonly endTime: string readonly releasesBucketSeconds?: number | undefined }, - orgId: string, + orgId: OrgId, ) => { const bucketSeconds = payload.releasesBucketSeconds ?? 300 return CH.compile( @@ -661,7 +662,7 @@ export const serviceEnvironments = defineQuery({ cache: timeRangeCache, compile: ( payload: { readonly serviceName: string; readonly startTime: string; readonly endTime: string }, - orgId: string, + orgId: OrgId, ) => CH.compile(CH.serviceEnvironmentsQuery({ serviceName: payload.serviceName }), { orgId, @@ -681,7 +682,7 @@ export const serviceExternalEdges = defineQuery({ readonly startTime: string readonly endTime: string }, - orgId: string, + orgId: OrgId, ) => CH.serviceExternalEdgesSQL( { deploymentEnv: payload.deploymentEnv, serviceName: payload.serviceName }, @@ -694,7 +695,7 @@ export const serviceUsage = defineQuery({ profile: "aggregation", // Usage totals tolerate a minute of staleness. cache: 60, - compile: (payload: ServiceUsageRequest, orgId: string) => { + compile: (payload: ServiceUsageRequest, orgId: OrgId) => { const prevStart = payload.previousStartTime const prevEnd = payload.previousEndTime return prevStart != null && prevEnd != null @@ -717,7 +718,7 @@ export const serviceDependencies = defineQuery({ id: "serviceDependencies", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ServiceDependenciesRequest, orgId: string) => + compile: (payload: ServiceDependenciesRequest, orgId: OrgId) => CH.serviceDependenciesSQL( { deploymentEnv: payload.deploymentEnv }, { orgId, startTime: payload.startTime, endTime: payload.endTime }, @@ -728,7 +729,7 @@ export const serviceDbEdges = defineQuery({ id: "serviceDbEdges", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ServiceDbEdgesRequest, orgId: string) => + compile: (payload: ServiceDbEdgesRequest, orgId: OrgId) => CH.serviceDbEdgesSQL( { deploymentEnv: payload.deploymentEnv }, { orgId, startTime: payload.startTime, endTime: payload.endTime }, @@ -739,7 +740,7 @@ export const serviceWorkloads = defineQuery({ id: "serviceWorkloads", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ServiceWorkloadsRequest, orgId: string) => + compile: (payload: ServiceWorkloadsRequest, orgId: OrgId) => CH.serviceWorkloadsSQL( { services: payload.services }, { orgId, startTime: payload.startTime, endTime: payload.endTime }, @@ -750,14 +751,14 @@ export const servicePlatforms = defineQuery({ id: "servicePlatforms", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ServicePlatformsRequest, orgId: string) => + compile: (payload: ServicePlatformsRequest, orgId: OrgId) => CH.servicePlatformsSQL( { deploymentEnv: payload.deploymentEnv }, { orgId, startTime: payload.startTime, endTime: payload.endTime }, ), }) -const dbQueryParams = (payload: ServiceDbQuerySummaryRequest, orgId: string) => ({ +const dbQueryParams = (payload: ServiceDbQuerySummaryRequest, orgId: OrgId) => ({ orgId, dbSystem: payload.dbSystem, dbNamespace: payload.dbNamespace, @@ -773,7 +774,7 @@ export const serviceDbQuerySummary = defineQuery({ id: "serviceDbQuerySummary", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ServiceDbQuerySummaryRequest, orgId: string) => + compile: (payload: ServiceDbQuerySummaryRequest, orgId: OrgId) => CH.serviceDbQuerySummarySQL(dbQueryParams(payload, orgId)), }) @@ -781,7 +782,7 @@ export const serviceDbQueryTimeseries = defineQuery({ id: "serviceDbQueryTimeseries", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ServiceDbQuerySummaryRequest, orgId: string) => + compile: (payload: ServiceDbQuerySummaryRequest, orgId: OrgId) => CH.serviceDbQueryTimeseriesSQL(dbQueryParams(payload, orgId)), }) @@ -789,7 +790,7 @@ export const serviceDbTopQueries = defineQuery({ id: "serviceDbTopQueries", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ServiceDbQuerySummaryRequest, orgId: string) => + compile: (payload: ServiceDbQuerySummaryRequest, orgId: OrgId) => CH.serviceDbTopQueriesSQL(dbQueryParams(payload, orgId)), }) @@ -835,7 +836,7 @@ const webAnalyticsSummaryDef = (useProductEvents: boolean) => ({ id: "webAnalyticsSummary" as const, profile: "aggregation" as const, cache: timeRangeCache, - compile: (payload: WebAnalyticsSummaryRequest, orgId: string) => + compile: (payload: WebAnalyticsSummaryRequest, orgId: OrgId) => CH.compile(CH.webAnalyticsSummaryQuery(webAnalyticsFilters(payload, useProductEvents)), { orgId, startTime: payload.startTime, @@ -868,7 +869,7 @@ const webAnalyticsLiveDef = (useProductEvents: boolean) => ({ id: "webAnalyticsLive" as const, profile: "aggregation" as const, cache: 15, - compile: (payload: WebAnalyticsLiveRequest, orgId: string) => { + compile: (payload: WebAnalyticsLiveRequest, orgId: OrgId) => { const now = Date.now() return CH.compile( CH.webAnalyticsLiveQuery({ @@ -891,7 +892,7 @@ const webAnalyticsTimeseriesDef = (useProductEvents: boolean) => ({ id: "webAnalyticsTimeseries" as const, profile: "aggregation" as const, cache: timeRangeCache, - compile: (payload: WebAnalyticsTimeseriesRequest, orgId: string) => + compile: (payload: WebAnalyticsTimeseriesRequest, orgId: OrgId) => CH.compile( CH.webAnalyticsTimeseriesQuery({ ...webAnalyticsFilters(payload, useProductEvents), @@ -908,7 +909,7 @@ const webAnalyticsPageviewsDef = (useProductEvents: boolean) => ({ id: "webAnalyticsPageviews" as const, profile: "aggregation" as const, cache: timeRangeCache, - compile: (payload: WebAnalyticsPageviewsRequest, orgId: string) => + compile: (payload: WebAnalyticsPageviewsRequest, orgId: OrgId) => CH.compile( CH.webAnalyticsPageviewsTimeseriesQuery({ ...webAnalyticsFilters(payload, useProductEvents), @@ -925,7 +926,7 @@ const webAnalyticsPagesDef = (useProductEvents: boolean) => ({ id: "webAnalyticsPages" as const, profile: "aggregation" as const, cache: timeRangeCache, - compile: (payload: WebAnalyticsPagesRequest, orgId: string) => + compile: (payload: WebAnalyticsPagesRequest, orgId: OrgId) => CH.compile( CH.webAnalyticsPagesQuery({ ...webAnalyticsFilters(payload, useProductEvents), @@ -942,7 +943,7 @@ const webAnalyticsEventsDef = (useProductEvents: boolean) => ({ id: "webAnalyticsEvents" as const, profile: "aggregation" as const, cache: timeRangeCache, - compile: (payload: WebAnalyticsEventsRequest, orgId: string) => + compile: (payload: WebAnalyticsEventsRequest, orgId: OrgId) => CH.compile( CH.webAnalyticsEventsQuery({ ...webAnalyticsFilters(payload, useProductEvents), @@ -961,7 +962,7 @@ const webAnalyticsBreakdownsDef = (useProductEvents: boolean) => ({ // Bound memory across the UNION fan-out. settings: { maxThreads: 4 }, cache: timeRangeCache, - compile: (payload: WebAnalyticsBreakdownsRequest, orgId: string) => + compile: (payload: WebAnalyticsBreakdownsRequest, orgId: OrgId) => CH.compileUnion( CH.webAnalyticsBreakdownsQuery({ ...webAnalyticsFilters(payload, useProductEvents), @@ -978,7 +979,7 @@ const webAnalyticsAiReferralsDef = (useProductEvents: boolean) => ({ id: "webAnalyticsAiReferrals" as const, profile: "aggregation" as const, cache: timeRangeCache, - compile: (payload: WebAnalyticsAiReferralsRequest, orgId: string) => + compile: (payload: WebAnalyticsAiReferralsRequest, orgId: OrgId) => CH.compile( CH.webAnalyticsAiReferralsQuery({ ...webAnalyticsFilters(payload, useProductEvents), @@ -992,7 +993,7 @@ export const webAnalyticsAiReferrals = defineQuery(webAnalyticsAiReferralsDef(tr export const webAnalyticsAiReferralsRaw = defineQuery(webAnalyticsAiReferralsDef(false)) // The three crawler reads share one request; each is a small scan of `ai_crawler_requests`. -const aiCrawlerWindow = (payload: WebAnalyticsAiCrawlersRequest, orgId: string) => ({ +const aiCrawlerWindow = (payload: WebAnalyticsAiCrawlersRequest, orgId: OrgId) => ({ orgId, startTime: payload.startTime, endTime: payload.endTime, @@ -1002,7 +1003,7 @@ export const webAnalyticsAiCrawlers = defineQuery({ id: "webAnalyticsAiCrawlers", profile: "aggregation", cache: timeRangeCache, - compile: (payload: WebAnalyticsAiCrawlersRequest, orgId: string) => + compile: (payload: WebAnalyticsAiCrawlersRequest, orgId: OrgId) => CH.compile( CH.webAnalyticsAiCrawlersQuery({ host: payload.host, pagePath: payload.pagePath }), aiCrawlerWindow(payload, orgId), @@ -1013,7 +1014,7 @@ export const webAnalyticsAiCrawlerFormats = defineQuery({ id: "webAnalyticsAiCrawlerFormats", profile: "aggregation", cache: timeRangeCache, - compile: (payload: WebAnalyticsAiCrawlersRequest, orgId: string) => + compile: (payload: WebAnalyticsAiCrawlersRequest, orgId: OrgId) => CH.compile( CH.webAnalyticsAiCrawlerFormatsQuery({ host: payload.host, pagePath: payload.pagePath }), aiCrawlerWindow(payload, orgId), @@ -1024,7 +1025,7 @@ export const webAnalyticsAiCrawledPages = defineQuery({ id: "webAnalyticsAiCrawledPages", profile: "aggregation", cache: timeRangeCache, - compile: (payload: WebAnalyticsAiCrawlersRequest, orgId: string) => + compile: (payload: WebAnalyticsAiCrawlersRequest, orgId: OrgId) => CH.compile( CH.webAnalyticsAiCrawledPagesQuery({ host: payload.host, @@ -1041,7 +1042,7 @@ export const podFacets = defineQuery({ // Bound Map-column decompression memory across the UNION fan-out. settings: { maxThreads: 4 }, cache: 60, - compile: (payload: PodFacetsRequest, orgId: string) => + compile: (payload: PodFacetsRequest, orgId: OrgId) => CH.compileUnion( CH.podFacetsQuery({ search: payload.search, @@ -1076,7 +1077,7 @@ export const nodeFacets = defineQuery({ // Bound Map-column decompression memory across the UNION fan-out. settings: { maxThreads: 4 }, cache: 60, - compile: (payload: NodeFacetsRequest, orgId: string) => + compile: (payload: NodeFacetsRequest, orgId: OrgId) => CH.compileUnion( CH.nodeFacetsQuery({ search: payload.search, @@ -1094,7 +1095,7 @@ export const workloadFacets = defineQuery({ // Bound Map-column decompression memory across the UNION fan-out. settings: { maxThreads: 4 }, cache: 60, - compile: (payload: WorkloadFacetsRequest, orgId: string) => + compile: (payload: WorkloadFacetsRequest, orgId: OrgId) => CH.compileUnion( CH.workloadFacetsQuery({ kind: payload.kind, @@ -1143,7 +1144,7 @@ export const listPods = defineQuery({ id: "listPods", profile: "list", cache: timeRangeCache, - compile: (payload: ListPodsRequest, orgId: string) => + compile: (payload: ListPodsRequest, orgId: OrgId) => CH.compile( CH.listPodsQuery({ ...listPodsFilters(payload), @@ -1161,7 +1162,7 @@ export const listPodsCount = defineQuery({ id: "listPodsCount", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ListPodsRequest, orgId: string) => + compile: (payload: ListPodsRequest, orgId: OrgId) => CH.compile(CH.listPodsSummaryQuery(listPodsFilters(payload)), { orgId, startTime: payload.startTime, @@ -1190,7 +1191,7 @@ export const listContainers = defineQuery({ id: "listContainers", profile: "list", cache: timeRangeCache, - compile: (payload: ListContainersRequest, orgId: string) => + compile: (payload: ListContainersRequest, orgId: OrgId) => CH.compile( CH.listContainersQuery({ ...listContainersFilters(payload), @@ -1208,7 +1209,7 @@ export const listContainersCount = defineQuery({ id: "listContainersCount", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ListContainersRequest, orgId: string) => + compile: (payload: ListContainersRequest, orgId: OrgId) => CH.compile(CH.listContainersSummaryQuery(listContainersFilters(payload)), { orgId, startTime: payload.startTime, @@ -1220,7 +1221,7 @@ export const containersSummary = defineQuery({ id: "containersSummary", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ContainersSummaryRequest, orgId: string) => + compile: (payload: ContainersSummaryRequest, orgId: OrgId) => CH.compile( CH.listContainersSummaryQuery({ hostNames: payload.hostNames, @@ -1234,7 +1235,7 @@ export const containerDetailSummary = defineQuery({ id: "containerDetailSummary", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ContainerDetailSummaryRequest, orgId: string) => + compile: (payload: ContainerDetailSummaryRequest, orgId: OrgId) => CH.compile( CH.containerDetailSummaryQuery({ containerName: payload.containerName, @@ -1248,7 +1249,7 @@ export const containerCountersSummary = defineQuery({ id: "containerCountersSummary", profile: "aggregation", cache: timeRangeCache, - compile: (payload: ContainerDetailSummaryRequest, orgId: string) => + compile: (payload: ContainerDetailSummaryRequest, orgId: OrgId) => CH.compile( CH.containerCountersSummaryQuery({ containerName: payload.containerName, @@ -1264,7 +1265,7 @@ export const containerFacets = defineQuery({ // Bound Map-column decompression memory across the UNION fan-out. settings: { maxThreads: 4 }, cache: 60, - compile: (payload: ContainerFacetsRequest, orgId: string) => + compile: (payload: ContainerFacetsRequest, orgId: OrgId) => CH.compileUnion(CH.containerFacetsQuery(listContainersFilters(payload)), { orgId, startTime: payload.startTime, @@ -1278,8 +1279,8 @@ export const spanHierarchyProbeRecent = defineQuery({ id: "spanHierarchyProbeRecent", profile: "discovery", cache: undefined, - compile: (payload: { readonly traceId: string; readonly startTime: string }, orgId: string) => - CH.compile(CH.traceTimeProbeQuery({ traceId: payload.traceId, narrowByTime: true }), { + compile: (payload: { readonly traceId: string; readonly startTime: string }, orgId: OrgId) => + CH.compile(CH.recentTraceTimeProbeQuery({ traceId: payload.traceId }), { orgId, startTime: payload.startTime, }), @@ -1289,8 +1290,8 @@ export const spanHierarchyProbe = defineQuery({ id: "spanHierarchyProbe", profile: "discovery", cache: undefined, - compile: (payload: { readonly traceId: string }, orgId: string) => - CH.compile(CH.traceTimeProbeQuery({ traceId: payload.traceId, narrowByTime: false }), { + compile: (payload: { readonly traceId: string }, orgId: OrgId) => + CH.compile(CH.traceTimeProbeQuery({ traceId: payload.traceId }), { orgId, }), }) @@ -1306,7 +1307,7 @@ export const spanHierarchy = defineQuery({ readonly startTime?: string | undefined readonly endTime?: string | undefined }, - orgId: string, + orgId: OrgId, ) => { const narrowByTime = payload.startTime != null && payload.endTime != null return CH.compile( @@ -1328,7 +1329,7 @@ const serviceOperationsSummaryOptions = (payload: ServiceOperationsRequest) => ( limit: payload.limit, }) -const serviceOperationsParams = (payload: ServiceOperationsRequest, orgId: string) => ({ +const serviceOperationsParams = (payload: ServiceOperationsRequest, orgId: OrgId) => ({ orgId, startTime: payload.startTime, endTime: payload.endTime, @@ -1338,7 +1339,7 @@ export const serviceOperationsSummary = defineQuery({ id: "serviceOperations", profile: "aggregation", cache: undefined, - compile: (payload: ServiceOperationsRequest, orgId: string) => + compile: (payload: ServiceOperationsRequest, orgId: OrgId) => CH.compile( CH.serviceOperationsSummaryQuery(serviceOperationsSummaryOptions(payload)), serviceOperationsParams(payload, orgId), @@ -1357,7 +1358,7 @@ export const serviceOperationsSummaryRaw = defineQuery({ profile: "aggregation", settings: SERVICE_OPERATIONS_RAW_SETTINGS, cache: undefined, - compile: (payload: ServiceOperationsRequest, orgId: string) => + compile: (payload: ServiceOperationsRequest, orgId: OrgId) => CH.compile( CH.serviceOperationsSummaryRawQuery(serviceOperationsSummaryOptions(payload)), serviceOperationsParams(payload, orgId), @@ -1375,7 +1376,7 @@ export const serviceEndpointsSummary = defineQuery({ id: "serviceEndpoints", profile: "aggregation", cache: undefined, - compile: (payload: ServiceOperationsRequest, orgId: string) => + compile: (payload: ServiceOperationsRequest, orgId: OrgId) => CH.compile( CH.serviceEndpointsSummaryQuery(serviceOperationsSummaryOptions(payload)), serviceOperationsParams(payload, orgId), @@ -1388,7 +1389,7 @@ export const serviceEndpointsSummaryRaw = defineQuery({ profile: "aggregation", settings: SERVICE_OPERATIONS_RAW_SETTINGS, cache: undefined, - compile: (payload: ServiceOperationsRequest, orgId: string) => + compile: (payload: ServiceOperationsRequest, orgId: OrgId) => CH.compile( CH.serviceEndpointsSummaryRawQuery(serviceOperationsSummaryOptions(payload)), serviceOperationsParams(payload, orgId), @@ -1413,7 +1414,7 @@ export const serviceOperationsTimeseries = defineQuery({ id: "serviceOperationsTimeseries", profile: "aggregation", cache: undefined, - compile: (payload: ServiceOperationsTimeseriesInput, orgId: string) => + compile: (payload: ServiceOperationsTimeseriesInput, orgId: OrgId) => CH.compile(CH.serviceOperationsTimeseriesQuery(serviceOperationsTimeseriesOptions(payload)), { ...serviceOperationsParams(payload, orgId), bucketSeconds: payload.bucketSeconds, @@ -1425,7 +1426,7 @@ export const serviceOperationsTimeseriesRaw = defineQuery({ profile: "aggregation", settings: SERVICE_OPERATIONS_RAW_SETTINGS, cache: undefined, - compile: (payload: ServiceOperationsTimeseriesInput, orgId: string) => + compile: (payload: ServiceOperationsTimeseriesInput, orgId: OrgId) => CH.compile(CH.serviceOperationsTimeseriesRawQuery(serviceOperationsTimeseriesOptions(payload)), { ...serviceOperationsParams(payload, orgId), bucketSeconds: payload.bucketSeconds, diff --git a/packages/query-engine/src/registry/query-definition.ts b/packages/query-engine/src/registry/query-definition.ts index 88f32e8d13..c366460621 100644 --- a/packages/query-engine/src/registry/query-definition.ts +++ b/packages/query-engine/src/registry/query-definition.ts @@ -1,6 +1,6 @@ // BOUNDARY: This module intentionally carries opaque values; callers decode them before domain use. import type { Effect } from "effect" -import type { CompiledQuery, QueryBuilderError } from "@maple-dev/effect-clickhouse" +import type { CompiledQuery, QueryBuilderError } from "@maple-dev/effect-orm/clickhouse" import type { WarehouseCapabilities } from "../capabilities" import type { QueryProfileName, WarehouseQuerySettings } from "../profiles/query-profile" import { @@ -8,6 +8,7 @@ import { type DirectRouteCachePolicy, type DirectRouteCachePolicyInput, } from "../runtime/cache-policy" +import type { OrgId } from "@maple/domain" /** Range-independent identity lets overlapping windows reuse stored buckets. */ export interface TimeBucketQueryCachePolicy { @@ -42,7 +43,7 @@ export interface QueryDefinition { */ readonly compile: ( payload: Payload, - orgId: string, + orgId: OrgId, capabilities: WarehouseCapabilities, ) => Effect.Effect, QueryBuilderError> } diff --git a/packages/query-engine/src/runtime/query-definition-runner.ts b/packages/query-engine/src/runtime/query-definition-runner.ts index d8a2ba6a59..ee249b3e77 100644 --- a/packages/query-engine/src/runtime/query-definition-runner.ts +++ b/packages/query-engine/src/runtime/query-definition-runner.ts @@ -1,11 +1,12 @@ -import type { CompiledQuery, CompiledQueryInput, QueryBuilderError } from "@maple-dev/effect-clickhouse" +import type { CompiledQuery, CompiledQueryInput, QueryBuilderError } from "@maple-dev/effect-orm/clickhouse" import type { Effect, Option } from "effect" import { baselineWarehouseCapabilities, type WarehouseCapabilities } from "../capabilities" import type { SqlQueryOptions } from "../profiles" import type { QueryDefinition } from "../registry/query-definition" +import type { OrgId } from "@maple/domain" export interface QueryDefinitionTenant { - readonly orgId: string + readonly orgId: OrgId } export interface QueryDefinitionWarehouse { diff --git a/packages/query-engine/src/runtime/query-engine.ts b/packages/query-engine/src/runtime/query-engine.ts index 523bad45ce..7374918f4d 100644 --- a/packages/query-engine/src/runtime/query-engine.ts +++ b/packages/query-engine/src/runtime/query-engine.ts @@ -915,6 +915,9 @@ const annotateWarehouseError = > = CH.CHQuery & CH.NeedsSelect + /** * Compile a CHQuery, execute it via the warehouse SQL executor, and return typed rows. * The inner WarehouseQueryService.executeSql span carries the full SQL, fingerprint, @@ -928,7 +931,7 @@ const executeCHQuery = Effect.fnUntraced(function* < >( warehouse: QueryEngineWarehouse, tenant: T, - query: CH.CHQuery | ((capabilities: WarehouseCapabilities) => CH.CHQuery), + query: SelectQuery | ((capabilities: WarehouseCapabilities) => SelectQuery), params: Params, context: string, profile: QueryProfileName = "aggregation", @@ -1396,7 +1399,7 @@ const isMissingTraceFacetsRollup = (error: unknown): boolean => { * Only a read that actually included the rollup can be missing it. */ const withTraceFacetsFallback = ( - orgId: string, + orgId: OrgId, usesRollup: boolean, run: (rawOnly: boolean) => Effect.Effect, ): Effect.Effect => @@ -1752,7 +1755,9 @@ export const makeQueryEngineExecute = (warehouse: QueryEn groupByAttributeKey: tracesQuery.groupBy === "attribute" ? opts.groupByAttributeKeys?.[0] : undefined, groupByResourceAttributeKey: - tracesQuery.groupBy === "attribute" ? opts.groupByResourceAttributeKey : undefined, + tracesQuery.groupBy === "attribute" + ? opts.groupByResourceAttributeKey + : undefined, limit: tracesQuery.limit, apdexThresholdMs: tracesQuery.metric === "apdex" ? tracesQuery.apdexThresholdMs : undefined, diff --git a/packages/query-engine/src/runtime/raw-sql.test.ts b/packages/query-engine/src/runtime/raw-sql.test.ts index 5261779d9b..56dc3feca1 100644 --- a/packages/query-engine/src/runtime/raw-sql.test.ts +++ b/packages/query-engine/src/runtime/raw-sql.test.ts @@ -9,9 +9,10 @@ import { RawSqlValidationError, } from "@maple/domain/http" import { makeExecuteRawSql, prepareRawSql } from "./raw-sql" +import { OrgId } from "@maple/domain" const baseInput = { - orgId: "org_abc", + orgId: OrgId.make("org_abc"), startTime: "2026-05-14 00:00:00", endTime: "2026-05-14 06:00:00", granularitySeconds: 60, @@ -184,7 +185,7 @@ describe("prepareRawSql", () => { Effect.gen(function* () { const result = yield* prepareRawSql({ ...baseInput, - orgId: "org'); DROP TABLE Logs --", + orgId: OrgId.make("org'); DROP TABLE Logs --"), sql: "SELECT 1 FROM Logs WHERE $__orgFilter", }) assert.include(result.sql, "OrgId = 'org\\')\\x3B DROP TABLE Logs --'") @@ -255,7 +256,7 @@ it.effect("does not expand replacement patterns in interpolated values", () => Effect.gen(function* () { const prepared = yield* prepareRawSql({ ...baseInput, - orgId: "org_$'_$&", + orgId: OrgId.make("org_$'_$&"), sql: "SELECT count() FROM Logs WHERE $__orgFilter AND Body = 'tail'", }) assert.include(prepared.sql, "OrgId = 'org_$\\'_$&'") diff --git a/packages/query-engine/src/runtime/raw-sql.ts b/packages/query-engine/src/runtime/raw-sql.ts index a6afae93ad..5066d26235 100644 --- a/packages/query-engine/src/runtime/raw-sql.ts +++ b/packages/query-engine/src/runtime/raw-sql.ts @@ -7,7 +7,8 @@ import { } from "@maple/domain/http" import { ORG_FILTER_MACRO_RE, rawSqlIssue, type RawSqlWorkload } from "@maple/domain/raw-sql" import type { QueryProfileName } from "../profiles" -import { escapeClickHouseString, splitTerminalClauses } from "@maple-dev/effect-clickhouse/sql" +import { escapeClickHouseString, splitTerminalClauses } from "@maple-dev/effect-orm/sql" +import type { OrgId } from "@maple/domain" // User-authored ClickHouse SQL: validation, macro expansion, and execution. // @@ -24,7 +25,7 @@ export type { RawSqlWorkload } export interface PrepareRawSqlInput { readonly sql: string - readonly orgId: string + readonly orgId: OrgId readonly startTime: string readonly endTime: string readonly granularitySeconds: number diff --git a/packages/query-engine/src/traces-shared.ts b/packages/query-engine/src/traces-shared.ts index f564176039..7378689151 100644 --- a/packages/query-engine/src/traces-shared.ts +++ b/packages/query-engine/src/traces-shared.ts @@ -36,9 +36,9 @@ export const TRACE_LIST_MV_RESOURCE_MAP: Record = { // Attribute filter → typed Condition -import * as CH from "@maple-dev/effect-clickhouse/expr" +import * as CH from "@maple-dev/effect-orm/expr" import { normalizedSpanNameExpr } from "@maple/domain/tinybird/span-display-name" -import * as T from "@maple-dev/effect-clickhouse/types" +import * as T from "@maple-dev/effect-orm/clickhouse" // Semconv rename coalescing // From e0e620f8acc21936847c51af51e0304921e39e53 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 5 Oct 2026 22:55:52 +0200 Subject: [PATCH 2/3] Pin @maple-dev/effect-orm 0.2.0 --- apps/api/package.json | 2 +- apps/clickhouse-builder-docs/package.json | 2 +- packages/backend/package.json | 2 +- packages/domain/package.json | 2 +- packages/query-engine-integrations/package.json | 2 +- packages/query-engine/package.json | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/api/package.json b/apps/api/package.json index a57f5655ee..96b6cedabe 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -21,7 +21,7 @@ }, "dependencies": { "@effect/platform-bun": "catalog:effect", - "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", + "@maple-dev/effect-orm": "0.2.0", "@maple-dev/effect-sdk": "workspace:*", "@maple/ai-model-catalog": "workspace:*", "@maple/auth": "workspace:*", diff --git a/apps/clickhouse-builder-docs/package.json b/apps/clickhouse-builder-docs/package.json index 3de670cd5f..734bd07844 100644 --- a/apps/clickhouse-builder-docs/package.json +++ b/apps/clickhouse-builder-docs/package.json @@ -12,7 +12,7 @@ }, "dependencies": { "@base-ui/react": "^1.8.0", - "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", + "@maple-dev/effect-orm": "0.2.0", "effect": "catalog:effect", "fumadocs-core": "^16.15.15", "fumadocs-mdx": "^15.4.5", diff --git a/packages/backend/package.json b/packages/backend/package.json index f22f5a24df..f6209951d5 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -23,7 +23,7 @@ "@distilled.cloud/cloudflare": "1.0.0-rc.13", "@distilled.cloud/core": "1.0.0-rc.13", "@distilled.cloud/railway": "1.0.0-rc.13", - "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", + "@maple-dev/effect-orm": "0.2.0", "@maple-dev/effect-clickhouse-http": "workspace:*", "@maple/alerting-core": "workspace:*", "@maple/auth": "workspace:*", diff --git a/packages/domain/package.json b/packages/domain/package.json index 288e9e507a..d5fb3818f7 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -59,7 +59,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", + "@maple-dev/effect-orm": "0.2.0", "@maple/primitives": "workspace:*", "@maple/query-model": "workspace:*", "@maple/widgets": "workspace:*", diff --git a/packages/query-engine-integrations/package.json b/packages/query-engine-integrations/package.json index 92856e20c0..e05be57aae 100644 --- a/packages/query-engine-integrations/package.json +++ b/packages/query-engine-integrations/package.json @@ -15,7 +15,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", + "@maple-dev/effect-orm": "0.2.0", "@maple/domain": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect" diff --git a/packages/query-engine/package.json b/packages/query-engine/package.json index cd1ca8c8af..6774ce1e8f 100644 --- a/packages/query-engine/package.json +++ b/packages/query-engine/package.json @@ -28,7 +28,7 @@ "typecheck": "tsc --noEmit" }, "dependencies": { - "@maple-dev/effect-orm": "link:@maple-dev/effect-orm", + "@maple-dev/effect-orm": "0.2.0", "@maple/cache": "workspace:*", "@maple/domain": "workspace:*", "@maple/query-model": "workspace:*", From 898c193a5399646b92b33e40cd878b44b2ba81b0 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 5 Oct 2026 22:58:50 +0200 Subject: [PATCH 3/3] Lock @maple-dev/effect-orm 0.2.0 --- bun.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bun.lock b/bun.lock index b5226deb46..54ce7e81ef 100644 --- a/bun.lock +++ b/bun.lock @@ -79,7 +79,7 @@ "name": "@maple/api", "dependencies": { "@effect/platform-bun": "catalog:effect", - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "0.2.0", "@maple-dev/effect-sdk": "workspace:*", "@maple/ai-model-catalog": "workspace:*", "@maple/auth": "workspace:*", @@ -596,8 +596,8 @@ "@distilled.cloud/cloudflare": "1.0.0-rc.13", "@distilled.cloud/core": "1.0.0-rc.13", "@distilled.cloud/railway": "1.0.0-rc.13", - "@maple-dev/effect-clickhouse": "0.2.0", "@maple-dev/effect-clickhouse-http": "workspace:*", + "@maple-dev/effect-orm": "0.2.0", "@maple/alerting-core": "workspace:*", "@maple/auth": "workspace:*", "@maple/cache": "workspace:*", @@ -738,7 +738,7 @@ "name": "@maple/domain", "version": "0.0.0", "dependencies": { - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "0.2.0", "@maple/primitives": "workspace:*", "@maple/query-model": "workspace:*", "@maple/widgets": "workspace:*", @@ -833,7 +833,7 @@ "packages/query-engine": { "name": "@maple/query-engine", "dependencies": { - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "0.2.0", "@maple/cache": "workspace:*", "@maple/domain": "workspace:*", "@maple/query-model": "workspace:*", @@ -850,7 +850,7 @@ "packages/query-engine-integrations": { "name": "@maple/query-engine-integrations", "dependencies": { - "@maple-dev/effect-clickhouse": "0.2.0", + "@maple-dev/effect-orm": "0.2.0", "@maple/domain": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect", @@ -1528,10 +1528,10 @@ "@maple-dev/browser": ["@maple-dev/browser@workspace:packages/browser"], - "@maple-dev/effect-clickhouse": ["@maple-dev/effect-clickhouse@0.2.0", "", { "peerDependencies": { "effect": "^4.0.0" }, "bin": { "ch-bench": "dist/benchmark/bin.mjs" } }, "sha512-pFTekfGfuzJyiD0pQp8HSTNs1K10Z8K1i5Ozqexih9utHyWWan+tfAEsqWfCOCLiDdCP5Phun6r/bVZ683otvQ=="], - "@maple-dev/effect-clickhouse-http": ["@maple-dev/effect-clickhouse-http@workspace:lib/effect-clickhouse-http"], + "@maple-dev/effect-orm": ["@maple-dev/effect-orm@0.2.0", "", { "peerDependencies": { "effect": "^4.0.0" }, "bin": { "ch-bench": "dist/benchmark/bin.mjs", "effect-orm": "dist/kit/bin.mjs" } }, "sha512-vXzFm2mOmyKcixehXxSzkgmD5gZtjkVnZNLXTMG8uFHifSoIW1LjxV5oP9tbQQjSFy+sPRnBpW/tXp3EWu2ljg=="], + "@maple-dev/effect-sdk": ["@maple-dev/effect-sdk@workspace:packages/effect-sdk"], "@maple-examples/alchemy": ["@maple-examples/alchemy@workspace:examples/alchemy-maple"],