diff --git a/app/api/cron/otel-retention/route.ts b/app/api/cron/otel-retention/route.ts new file mode 100644 index 00000000..32f0d7ea --- /dev/null +++ b/app/api/cron/otel-retention/route.ts @@ -0,0 +1,124 @@ +import { gzipSync } from "node:zlib"; +import { NextRequest, NextResponse } from "next/server"; +import { sql } from "drizzle-orm"; +import { put } from "@vercel/blob"; +import { dz } from "@/database/engine"; + +export const runtime = "nodejs"; +export const maxDuration = 300; + +// otel.proofer.tech 수신 데이터 중 보관 기간(기본 90일)이 지난 행을 지우지 않고 +// Vercel Blob(private 스토어 proofer-otel-archive)으로 옮긴다. 올라간 것을 확인한 행만 DB 에서 뺀다. +// otel/{tenant}/{yyyy}/{mm}/{dd}/{table}-{첫 id}-{끝 id}.ndjson.gz +// Vercel Cron 이 매일 부른다(vercel.json). CRON_SECRET 이 Authorization 헤더로 온다. +const DEFAULT_DAYS = 90; +const CHUNK = 5000; +const MAX_ROUNDS = 20; + +type Table = "otel_log_record" | "otel_raw_batch"; +type Row = { + id: number; + tenant_id: number; + slug: string; + day: string; +} & Record; + +function retentionDays(): number { + const n = Number(process.env.OTEL_RETENTION_DAYS); + return Number.isInteger(n) && n >= 7 ? n : DEFAULT_DAYS; +} + +async function archiveChunk(table: Table, days: number, token: string) { + const res = await dz.execute(sql` + select r.*, t.slug, to_char(r.received_at, 'YYYY/MM/DD') as day + from ${sql.identifier(table)} r + join otel_tenant t on t.id = r.tenant_id + where r.received_at < now() - make_interval(days => ${days}) + order by r.id + limit ${CHUNK} + `); + const rows = res.rows as Row[]; + if (rows.length === 0) return { rows: 0, files: 0 }; + + const groups = new Map(); + for (const r of rows) { + const key = `${r.slug}/${r.day}`; + const g = groups.get(key); + if (g) g.push(r); + else groups.set(key, [r]); + } + + let files = 0; + for (const [key, group] of groups) { + const first = group[0].id; + const last = group[group.length - 1].id; + const body = gzipSync( + group + .map(({ slug: _slug, day: _day, ...row }) => JSON.stringify(row)) + .join("\n") + "\n", + ); + // 같은 행 묶음을 다시 올리면 같은 이름이라 덮어쓴다. 재시도해도 중복 파일이 생기지 않는다. + await put(`otel/${key}/${table}-${first}-${last}.ndjson.gz`, body, { + access: "private", + token, + contentType: "application/gzip", + addRandomSuffix: false, + allowOverwrite: true, + }); + const ids = group.map((r) => r.id); + await dz.execute(sql` + delete from ${sql.identifier(table)} + where id in (${sql.join( + ids.map((id) => sql`${id}`), + sql`, `, + )}) + `); + files++; + } + return { rows: rows.length, files }; +} + +async function archive(table: Table, days: number, token: string) { + let rows = 0; + let files = 0; + for (let i = 0; i < MAX_ROUNDS; i++) { + const r = await archiveChunk(table, days, token); + rows += r.rows; + files += r.files; + if (r.rows < CHUNK) break; + } + return { rows, files }; +} + +export async function GET(req: NextRequest) { + const secret = process.env.CRON_SECRET; + if (!secret || req.headers.get("authorization") !== `Bearer ${secret}`) { + return new NextResponse(null, { status: 401 }); + } + const token = process.env.OTEL_ARCHIVE_READ_WRITE_TOKEN; + if (!token) { + console.error("otel archive: OTEL_ARCHIVE_READ_WRITE_TOKEN is not set"); + return NextResponse.json( + { error: "archive store not configured" }, + { status: 500 }, + ); + } + + const days = retentionDays(); + try { + const logs = await archive("otel_log_record", days, token); + const raw = await archive("otel_raw_batch", days, token); + console.log( + `otel archive: ${days}d, log=${logs.rows} rows/${logs.files} files, raw=${raw.rows} rows/${raw.files} files`, + ); + return NextResponse.json({ + days, + otel_log_record: logs, + otel_raw_batch: raw, + }); + } catch (error) { + // 업로드가 실패한 묶음은 지우지 않았으므로 다음 실행에서 다시 옮긴다. + console.error("otel archive failed", error); + return NextResponse.json({ error: "archive failed" }, { status: 500 }); + } +} diff --git a/package.json b/package.json index 7e9eb026..9b49a35b 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,7 @@ "@upstash/lock": "^0.1.0", "@upstash/redis": "^1.25.1", "@vercel/analytics": "^1.2.2", - "@vercel/blob": "^0.22.3", + "@vercel/blob": "^2.9.0", "@vercel/edge-config": "^1.1.0", "@vercel/kv": "^1.0.1", "@vercel/postgres": "^0.9.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5349446c..b209792a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -129,8 +129,8 @@ importers: specifier: ^1.2.2 version: 1.6.1(next@16.0.11(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.101.3))(react@18.3.1) '@vercel/blob': - specifier: ^0.22.3 - version: 0.22.3 + specifier: ^2.9.0 + version: 2.9.0 '@vercel/edge-config': specifier: ^1.1.0 version: 1.4.3(@opentelemetry/api@1.9.1)(next@16.0.11(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.101.3)) @@ -1687,10 +1687,6 @@ packages: resolution: {integrity: sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} - '@fastify/busboy@2.1.1': - resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} - engines: {node: '>=14'} - '@floating-ui/core@1.8.0': resolution: {integrity: sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==} @@ -4174,9 +4170,16 @@ packages: vue-router: optional: true - '@vercel/blob@0.22.3': - resolution: {integrity: sha512-l0t2KhbOO/I8ZNOl9zypYf1NE0837aO4/CPQNGR/RAxtj8FpdYKjhyUADUXj2gERLQmnhun+teaVs/G7vZJ/TQ==} - engines: {node: '>=16.14'} + '@vercel/blob@2.9.0': + resolution: {integrity: sha512-o+1CMRlT8/RvGo1aGkFv40F7v3MLdSOaVaIccepBXWrXgonKId3hHKSKGQ+iBDPoSe03crmHfhpHtTs0CCXiig==} + engines: {node: '>=20.0.0'} + + '@vercel/cli-config@0.3.1': + resolution: {integrity: sha512-sISbygqlFXYb8qfPu/+2Cf/A8IjrlAuZePdag3EDQr2PRA455s7UZtl9Ro674CWsB/saBRmz3jEkw0YsQs8OoQ==} + + '@vercel/cli-exec@1.0.1': + resolution: {integrity: sha512-g9XerViJ/paZujufXYcu5XYI2vU2rtB4sgdpjUHde5RnOkdmpu0ngH46LCFGHoPXO/C+qDPSczIHIRN+8Q2YKQ==} + engines: {node: '>= 18'} '@vercel/edge-config-fs@0.1.0': resolution: {integrity: sha512-NRIBwfcS0bUoUbRWlNGetqjvLSwgYH/BqKqDN7vK1g32p7dN96k0712COgaz6VFizAm9b0g6IG6hR6+hc0KCPg==} @@ -4200,6 +4203,10 @@ packages: engines: {node: '>=14.6'} deprecated: 'Vercel KV is deprecated. If you had an existing KV store, it should have moved to Upstash Redis which you will see under Vercel Integrations. For new projects, install a Redis integration from Vercel Marketplace: https://vercel.com/marketplace?category=storage&search=redis' + '@vercel/oidc@3.8.10': + resolution: {integrity: sha512-sPw1kIfQEA6/2dB+zlAJHnhhNBRJ8t/t4HfFvKO+g0EPBA7NbCKyGovuhgAV6Mnl865ka7HqCEguIy596czdAw==} + engines: {node: '>= 20'} + '@vercel/postgres@0.9.0': resolution: {integrity: sha512-WiI2g3+ce2g1u1gP41MoDj2DsMuQQ+us7vHobysRixKECGaLHpfTI7DuVZmHU087ozRAGr3GocSyqmWLLo+fig==} engines: {node: '>=14.6'} @@ -4777,10 +4784,6 @@ packages: resolution: {integrity: sha512-jheRLVMeUKrDBjVw2O5+k4EvR4t9wtxHL+bo/LxfkxsVeuGMy3a5SEGgXdAFA4FSzTrU8rQXQIrsZ3oBq5a0pQ==} engines: {node: '>=20'} - bytes@3.1.2: - resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} - engines: {node: '>= 0.8'} - cac@6.7.14: resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} engines: {node: '>=8'} @@ -6656,6 +6659,9 @@ packages: resolution: {integrity: sha512-5KoIu2Ngpyek75jXodFvnafB6DJgr3u8uuK0LEZJjrU19DrMD3EVERaR8sjz8CCGgpZvxPl9SuE1GMVPFHx1mw==} engines: {node: '>= 0.4'} + is-node-process@1.2.0: + resolution: {integrity: sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==} + is-number-object@1.1.1: resolution: {integrity: sha512-lZhclumE1G6VYD8VHe35wFaIif+CTy5SJIi5+3y4psDgWu4wPDoBhF8NxUOinEc7pHgiTsT6MaBb92rKhhD+Xw==} engines: {node: '>= 0.4'} @@ -6968,6 +6974,9 @@ packages: resolution: {integrity: sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==} hasBin: true + jose@5.10.0: + resolution: {integrity: sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==} + js-md5@0.8.3: resolution: {integrity: sha512-qR0HB5uP6wCuRMrWPTrkMaev7MJZwJuuw4fnwAzRgP4J4/F8RwtodOKpGp4XpqsLBFzzgqIO42efFAyz2Et6KQ==} @@ -7904,6 +7913,10 @@ packages: orderedmap@2.1.1: resolution: {integrity: sha512-TvAWxi0nDe1j/rtMcWcIj94+Ffe6n7zhow33h40SKxmsmozs6dz/e+EajymfoFcHd7sxNn8yHM8839uixMOV6g==} + os-paths@4.4.0: + resolution: {integrity: sha512-wrAwOeXp1RRMFfQY8Sy7VaGVmPocaLwSFOYCGKSyo8qmJ+/yaafCl5BCA1IQZWqFSRBrKDYFeR9d/VyQzfH/jg==} + engines: {node: '>= 6.0'} + own-keys@1.0.2: resolution: {integrity: sha512-19YVAg7T+WTrxggPukVq7DjTv6+PJ867TmhCvBsYwmbFCsZd344rq2Ld1p0wo8f8Qrrhgp82c6FJRqdXWtSEhg==} engines: {node: '>= 0.4'} @@ -9420,6 +9433,10 @@ packages: three@0.184.0: resolution: {integrity: sha512-wtTRjG92pM5eUg/KuUnHsqSAlPM296brTOcLgMRqEeylYTh/CdtvKUvCyyCQTzFuStieWxvZb8mVTMvdPyUpxg==} + throttleit@2.1.0: + resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==} + engines: {node: '>=18'} + through2@2.0.5: resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==} @@ -9672,10 +9689,6 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} - undici@5.29.0: - resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} - engines: {node: '>=14.0'} - undici@6.27.0: resolution: {integrity: sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg==} engines: {node: '>=18.17'} @@ -10066,6 +10079,14 @@ packages: utf-8-validate: optional: true + xdg-app-paths@5.5.1: + resolution: {integrity: sha512-hI3flOB4PLZIy5prbtTpirobtPE2ZtZ52szO+2mM9Efp6ErM398La+C1lIpNWDfNoQk+6Lsi6nMcCwVB7pxeMQ==} + engines: {node: '>= 6.0'} + + xdg-portable@7.3.0: + resolution: {integrity: sha512-sqMMuL1rc0FmMBOzCpd0yuy9trqF2yTTVe+E9ogwCSWQCdDEtQUwrZPT6AxqtsFGRNxycgncbP/xmOOSPw5ZUw==} + engines: {node: '>= 6.0'} + xml-name-validator@4.0.0: resolution: {integrity: sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==} engines: {node: '>=12'} @@ -10149,6 +10170,9 @@ packages: peerDependencies: zod: ^3.25.0 || ^4.0.0 + zod@4.1.11: + resolution: {integrity: sha512-WPsqwxITS2tzx1bzhIKsEs19ABD5vmCVa4xBo2tq/SrV4RNZtfws1EnCWQXM6yh8bD08a1idvkB5MZSBiZsjwg==} + zod@4.4.3: resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} @@ -11354,8 +11378,6 @@ snapshots: '@eslint/core': 0.17.0 levn: 0.4.1 - '@fastify/busboy@2.1.1': {} - '@floating-ui/core@1.8.0': dependencies: '@floating-ui/utils': 0.2.12 @@ -14017,12 +14039,23 @@ snapshots: next: 16.0.11(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.101.3) react: 18.3.1 - '@vercel/blob@0.22.3': + '@vercel/blob@2.9.0': dependencies: + '@vercel/oidc': 3.8.10 async-retry: 1.3.3 - bytes: 3.1.2 is-buffer: 2.0.5 - undici: 5.29.0 + is-node-process: 1.2.0 + throttleit: 2.1.0 + undici: 6.27.0 + + '@vercel/cli-config@0.3.1': + dependencies: + xdg-app-paths: 5.5.1 + zod: 4.1.11 + + '@vercel/cli-exec@1.0.1': + dependencies: + execa: 5.1.1 '@vercel/edge-config-fs@0.1.0': {} @@ -14037,6 +14070,12 @@ snapshots: dependencies: '@upstash/redis': 1.25.1 + '@vercel/oidc@3.8.10': + dependencies: + '@vercel/cli-config': 0.3.1 + '@vercel/cli-exec': 1.0.1 + jose: 5.10.0 + '@vercel/postgres@0.9.0': dependencies: '@neondatabase/serverless': 0.9.5 @@ -14667,8 +14706,6 @@ snapshots: byte-counter@0.1.0: {} - bytes@3.1.2: {} - cac@6.7.14: {} cacheable-lookup@5.0.4: {} @@ -16734,6 +16771,8 @@ snapshots: is-negative-zero@2.0.3: {} + is-node-process@1.2.0: {} + is-number-object@1.1.1: dependencies: call-bound: 1.0.4 @@ -17241,6 +17280,8 @@ snapshots: jiti@1.21.7: {} + jose@5.10.0: {} + js-md5@0.8.3: {} js-tokens@4.0.0: {} @@ -18266,6 +18307,8 @@ snapshots: orderedmap@2.1.1: {} + os-paths@4.4.0: {} + own-keys@1.0.2: dependencies: call-bound: 1.0.4 @@ -20081,6 +20124,8 @@ snapshots: three@0.184.0: {} + throttleit@2.1.0: {} + through2@2.0.5: dependencies: readable-stream: 2.3.8 @@ -20327,10 +20372,6 @@ snapshots: undici-types@6.21.0: {} - undici@5.29.0: - dependencies: - '@fastify/busboy': 2.1.1 - undici@6.27.0: {} undici@7.28.0: {} @@ -20878,6 +20919,15 @@ snapshots: bufferutil: 4.1.0 utf-8-validate: 6.0.6 + xdg-app-paths@5.5.1: + dependencies: + os-paths: 4.4.0 + xdg-portable: 7.3.0 + + xdg-portable@7.3.0: + dependencies: + os-paths: 4.4.0 + xml-name-validator@4.0.0: {} xml2js@0.5.0: @@ -20948,6 +20998,8 @@ snapshots: dependencies: zod: 4.4.3 + zod@4.1.11: {} + zod@4.4.3: {} zustand@4.5.7(@types/react@18.2.58)(react@18.3.1): diff --git a/vercel.json b/vercel.json index 930f4857..959e5624 100644 --- a/vercel.json +++ b/vercel.json @@ -3,6 +3,10 @@ { "path": "/cron/fetch-medium", "schedule": "0 0 * * *" + }, + { + "path": "/api/cron/otel-retention", + "schedule": "0 19 * * *" } ] -} \ No newline at end of file +}