refactor: move the onboarding request classifier to project-onboarding (CM-1841)

services/apps/automatic_projects_discovery_worker/package.json

@@ -13,15 +13,12 @@
     "@crowd/archetype-standard": "workspace:*",
     "@crowd/archetype-worker": "workspace:*",
     "@crowd/common": "workspace:*",
-    "@crowd/common_services": "workspace:*",
     "@crowd/data-access-layer": "workspace:*",
     "@crowd/logging": "workspace:*",
     "@crowd/project-onboarding": "workspace:*",
     "@crowd/redis": "workspace:*",
     "@crowd/slack": "workspace:*",
-    "@crowd/snowflake": "workspace:*",
     "@crowd/temporal": "workspace:*",
-    "@crowd/types": "workspace:*",
     "@temporalio/activity": "catalog:",
     "@temporalio/client": "catalog:",
     "@temporalio/workflow": "catalog:",

services/apps/automatic_projects_discovery_worker/src/activities/activities.ts

@@ -23,6 +23,8 @@ import {
 } from '@crowd/data-access-layer/src/project-catalog/types'
 import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor'
 import { getServiceLogger } from '@crowd/logging'
+import { countNodes } from '@crowd/project-onboarding'
+import { withRequestClassifierDeps } from '@crowd/project-onboarding/src/requestClassifierDeps'
 import { SlackChannel, SlackPersona, sendSlackNotificationAsync } from '@crowd/slack'
 
 import { svc } from '../main'
@@ -37,8 +39,6 @@ import {
   buildRequestClassificationAlert,
   buildRequestClassificationAlertTitle,
 } from './requestClassificationAlert'
-import { withRequestClassificationDeps } from './requestClassificationDeps'
-import { countNodes } from './requestClassificationTrace'
 
 const log = getServiceLogger()
 
@@ -146,7 +146,7 @@ async function classifyAcceptedRows(
     return { rows: accepted, alerts: [], nodes: [] }
   }
 
-  return withRequestClassificationDeps(qx, (deps) =>
+  return withRequestClassifierDeps(qx, (deps) =>
     classifyDiscussions(accepted, requestTextBySourceUrl, deps, heartbeatClassification, dryRun),
   )
 }

services/apps/automatic_projects_discovery_worker/src/activities/requestClassification.test.ts

@@ -1,11 +1,10 @@
 import { describe, expect, it, vi } from 'vitest'
 
 import { IDbProjectCatalogCreate } from '@crowd/data-access-layer/src/project-catalog/types'
-import { IPccCandidate } from '@crowd/project-onboarding'
+import { IPccCandidate, IRequestClassificationDeps } from '@crowd/project-onboarding'
 
 import {
   AMBIGUOUS_SKIP_REASON,
-  IRequestClassificationDeps,
   LF_NOT_IN_CDP_SKIP_REASON,
   classifyDiscussionRows,
   classifyDiscussions,

services/apps/automatic_projects_discovery_worker/src/activities/requestClassification.ts

@@ -1,31 +1,16 @@
-import { getErrorMessage } from '@crowd/common'
 import { IDbProjectCatalogCreate } from '@crowd/data-access-layer/src/project-catalog/types'
 import { getServiceLogger } from '@crowd/logging'
 import {
   CdpIntegrationAction,
-  IOnboardingRequestLookups,
-  OnboardingResolution,
-  OnboardingRequestLlm,
-  parseOnboardingRequest,
-  resolveOnboardingRequest,
-} from '@crowd/project-onboarding'
-
-import {
   ClassificationNode,
-  IClassificationTrace,
+  IRequestClassificationDeps,
+  OnboardingResolution,
   buildClassificationLogEntry,
-  createClassificationTrace,
-  toClassificationNode,
-  traceLookups,
-} from './requestClassificationTrace'
+  classifyOnboardingRequest,
+} from '@crowd/project-onboarding'
 
 const log = getServiceLogger()
 
-export interface IRequestClassificationDeps {
-  queryLlm: OnboardingRequestLlm
-  lookups: IOnboardingRequestLookups
-}
-
 export interface IRequestClassificationAlert {
   sourceUrl: string
   repoUrls: string[]
@@ -65,40 +50,13 @@ function toSkipReason(resolution: Exclude<OnboardingResolution, { kind: 'non_lf_
   }
 }
 
-function unresolved(reason: string): OnboardingResolution {
-  return { kind: 'ambiguous', reason, candidates: [] }
-}
-
-async function resolveRequestText(
-  requestText: string,
-  deps: IRequestClassificationDeps,
-  trace: IClassificationTrace,
-): Promise<OnboardingResolution> {
-  try {
-    const parsed = await parseOnboardingRequest(requestText, deps.queryLlm)
-    if (parsed.ok === false) {
-      trace.failure = { stage: 'parse', reason: parsed.reason }
-      return unresolved(`Request could not be parsed: ${parsed.reason}`)
-    }
-
-    trace.parsed = parsed.request
-    return await resolveOnboardingRequest(parsed.request, traceLookups(deps.lookups, trace))
-  } catch (err) {
-    const reason = getErrorMessage(err)
-    trace.failure = { stage: 'resolve', reason }
-    return unresolved(`Classification failed: ${reason}`)
-  }
-}
-
 export async function classifyDiscussionRows(
   rows: IDbProjectCatalogCreate[],
   requestText: string,
   deps: IRequestClassificationDeps,
   dryRun = false,
 ): Promise<IClassifiedRows> {
-  const trace = createClassificationTrace()
-  const resolution = await resolveRequestText(requestText, deps, trace)
-  const node = toClassificationNode(resolution)
+  const { resolution, node, trace } = await classifyOnboardingRequest(requestText, deps)
 
   log.info(
     buildClassificationLogEntry(rows[0].sourceUrl ?? '', resolution, trace),

services/apps/automatic_projects_discovery_worker/src/bin/classificationCases.ts

@@ -1,4 +1,4 @@
-import { ClassificationNode } from '../activities/requestClassificationTrace'
+import { ClassificationNode } from '@crowd/project-onboarding'
 
 export interface IClassificationCase {
   id: string

services/apps/automatic_projects_discovery_worker/src/bin/classify-onboarding-cases.ts

@@ -3,9 +3,9 @@ import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/d
 import { IDbProjectCatalogCreate } from '@crowd/data-access-layer/src/project-catalog/types'
 import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor'
 import { getServiceLogger } from '@crowd/logging'
+import { withRequestClassifierDeps } from '@crowd/project-onboarding/src/requestClassifierDeps'
 
 import { classifyDiscussionRows } from '../activities/requestClassification'
-import { withRequestClassificationDeps } from '../activities/requestClassificationDeps'
 import { CLASSIFICATION_CASES, IClassificationCase } from './classificationCases'
 
 const log = getServiceLogger()
@@ -90,7 +90,7 @@ async function main(): Promise<void> {
   const cases = selectCases(readCaseFilter(argv))
   const qx = pgpQx(await getDbConnection(WRITE_DB_CONFIG()))
 
-  const outcomes = await withRequestClassificationDeps(qx, async (deps) => {
+  const outcomes = await withRequestClassifierDeps(qx, async (deps) => {
     const results: ICaseOutcome[] = []
     for (const testCase of cases) {
       const { nodes } = await classifyDiscussionRows(toRow(testCase), testCase.text, deps, true)

services/apps/automatic_projects_discovery_worker/tsconfig.json

@@ -11,9 +11,6 @@
     {
       "path": "../../libs/common"
     },
-    {
-      "path": "../../libs/common_services"
-    },
     {
       "path": "../../libs/data-access-layer"
     },
@@ -25,12 +22,6 @@
     },
     {
       "path": "../../libs/slack"
-    },
-    {
-      "path": "../../libs/snowflake"
-    },
-    {
-      "path": "../../libs/types"
     }
   ]
 }

services/libs/project-onboarding/package.json

@@ -3,7 +3,12 @@
   "private": true,
   "main": "src/index.ts",
   "dependencies": {
-    "@crowd/common": "workspace:*"
+    "@crowd/common": "workspace:*",
+    "@crowd/common_services": "workspace:*",
+    "@crowd/data-access-layer": "workspace:*",
+    "@crowd/logging": "workspace:*",
+    "@crowd/snowflake": "workspace:*",
+    "@crowd/types": "workspace:*"
   },
   "devDependencies": {
     "@types/node": "catalog:"

services/apps/automatic_projects_discovery_worker/src/activities/cdpSegmentLookup.ts → services/libs/project-onboarding/src/cdpSegmentLookup.ts

@@ -1,10 +1,7 @@
 import { fetchIntegrationsForSegment, findSubprojectsBySourceId } from '@crowd/data-access-layer'
 import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor'
-import {
-  CdpSegmentLookupResult,
-  ICdpSegmentMatch,
-  toCdpIntegrationState,
-} from '@crowd/project-onboarding'
+
+import { CdpSegmentLookupResult, ICdpSegmentMatch, toCdpIntegrationState } from './requestResolver'
 
 async function toSegmentMatch(
   qx: QueryExecutor,

services/apps/automatic_projects_discovery_worker/src/activities/requestClassificationTrace.test.ts → services/libs/project-onboarding/src/classificationTrace.test.ts

@@ -1,21 +1,20 @@
 import { describe, expect, it, vi } from 'vitest'
 
-import {
-  IOnboardingRequestLookups,
-  IParsedOnboardingRequest,
-  IPccCandidate,
-  OnboardingResolution,
-  PCC_MATCH_THRESHOLDS,
-} from '@crowd/project-onboarding'
-
 import {
   ClassificationNode,
   buildClassificationLogEntry,
   countNodes,
   createClassificationTrace,
   toClassificationNode,
   traceLookups,
-} from './requestClassificationTrace'
+} from './classificationTrace'
+import { IParsedOnboardingRequest } from './requestParser'
+import {
+  IOnboardingRequestLookups,
+  IPccCandidate,
+  OnboardingResolution,
+  PCC_MATCH_THRESHOLDS,
+} from './requestResolver'
 
 const pccProject: IPccCandidate = {
   projectId: 'pcc-1',

services/apps/automatic_projects_discovery_worker/src/activities/requestClassificationTrace.ts → services/libs/project-onboarding/src/classificationTrace.ts

@@ -1,13 +1,12 @@
+import { IParsedOnboardingRequest } from './requestParser'
 import {
   CdpIntegrationAction,
   CdpSegmentLookupResult,
   IOnboardingRequestLookups,
-  IParsedOnboardingRequest,
   IPccCandidate,
   OnboardingResolution,
   assessPccCandidates,
-} from '@crowd/project-onboarding'
-
+} from './requestResolver'
 export type ClassificationFailureStage = 'parse' | 'resolve'
 
 export interface IClassificationFailure {