From be5de3f0ac332ece288dd0c073ba6715c81b4f83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B8=B0=EB=AF=BC?= Date: Sun, 18 Oct 2026 22:20:59 +0900 Subject: [PATCH] =?UTF-8?q?test:=20=EA=B2=A9=EB=A6=AC=20=EB=B3=B5=EC=9B=90?= =?UTF-8?q?=EC=9D=98=20=EC=97=85=EB=A1=9C=EB=93=9C=20=EC=86=90=EC=8B=A4?= =?UTF-8?q?=EA=B3=BC=20=EB=AC=B8=EC=84=9C=20=EA=B8=B0=EB=8A=A5=20=EC=9E=AC?= =?UTF-8?q?=EA=B0=9C=20=EA=B3=84=EC=B8=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dr-functional-recovery-20261018/README.md | 172 +++ .../checks/contract-r5/events.log | 45 + .../checks/contract-r5/result.json | 14 + .../checks/contract-r5/spec.json | 9 + .../checks/status-compat-r1/result.json | 17 + .../checks/status-compat-r1/spec.json | 10 + .../checks/status-compat-r2/result.json | 17 + .../checks/status-compat-r2/spec.json | 10 + .../checks/task-cleanup-r1/result.json | 42 + .../checks/task-cleanup-r1/spec.json | 6 + .../checks/wal-boundary-r1/analysis.json | 12 + .../checks/wal-boundary-r1/result.json | 25 + .../checks/wal-boundary-r1/spec.json | 8 + .../export-index.json | 185 ++++ .../metrics.json | 118 +++ .../dr-functional-recovery-20261018/plan.md | 62 ++ .../preparation-findings.md | 38 + .../procedure.md | 58 ++ .../DR-PG-START-DIAG-20261018-R1/result.json | 19 + .../source-capture.json | 13 + .../DR-PG-START-DIAG-20261018-R1/spec.json | 10 + .../DR-PG-START-DIAG-20261018-R2/result.json | 22 + .../DR-PG-START-DIAG-20261018-R2/spec.json | 11 + .../BACKUP-result.json | 16 + .../INTERRUPT-result.json | 18 + .../RECEIVER-result.json | 113 ++ .../DR-RESTORE-20261018-FORMAL1/events.jsonl | 334 ++++++ .../final-audit.json | 465 +++++++++ .../final-snapshot.json | 803 +++++++++++++++ .../formal-spec.json | 18 + .../functional-observations.json | 32 + .../initial-audit.json | 465 +++++++++ .../initial-snapshot.json | 797 ++++++++++++++ .../DR-RESTORE-20261018-FORMAL1/inputs.json | 432 ++++++++ .../DR-RESTORE-20261018-FORMAL1/manifest.json | 6 + .../preparation-result.json | 6 + .../runtime-cleanup-observations.json | 34 + .../DR-RESTORE-20261018-FORMAL1/summary.json | 975 ++++++++++++++++++ .../operator-restore-error.json | 6 + .../ssh-cleanup-reconciliation-r2.json | 10 + .../summary.json | 10 + scripts/opensql/dr_recovery.py | 11 +- scripts/opensql/test_dr_recovery.py | 35 +- 43 files changed, 5506 insertions(+), 3 deletions(-) create mode 100644 docs/test-results/dr-functional-recovery-20261018/README.md create mode 100644 docs/test-results/dr-functional-recovery-20261018/checks/contract-r5/events.log create mode 100644 docs/test-results/dr-functional-recovery-20261018/checks/contract-r5/result.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/checks/contract-r5/spec.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r1/result.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r1/spec.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r2/result.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r2/spec.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/checks/task-cleanup-r1/result.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/checks/task-cleanup-r1/spec.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/checks/wal-boundary-r1/analysis.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/checks/wal-boundary-r1/result.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/checks/wal-boundary-r1/spec.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/export-index.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/metrics.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/plan.md create mode 100644 docs/test-results/dr-functional-recovery-20261018/preparation-findings.md create mode 100644 docs/test-results/dr-functional-recovery-20261018/procedure.md create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R1/result.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R1/source-capture.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R1/spec.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R2/result.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R2/spec.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/BACKUP-result.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/INTERRUPT-result.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/RECEIVER-result.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/events.jsonl create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/final-audit.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/final-snapshot.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/formal-spec.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/functional-observations.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/initial-audit.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/initial-snapshot.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/inputs.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/manifest.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/preparation-result.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/runtime-cleanup-observations.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/summary.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-REHEARSAL1/operator-restore-error.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-REHEARSAL1/ssh-cleanup-reconciliation-r2.json create mode 100644 docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-REHEARSAL1/summary.json diff --git a/docs/test-results/dr-functional-recovery-20261018/README.md b/docs/test-results/dr-functional-recovery-20261018/README.md new file mode 100644 index 00000000..faf704e8 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/README.md @@ -0,0 +1,172 @@ +# 업로드 성공 응답의 손실·문서 기능 재개 계측 + +관련 이슈: #560 · 판정 계약: [#561](https://github.com/DocGrid/docgrid/pull/561) · [실행 절차](procedure.md) · [준비·재시도](preparation-findings.md) · [수치 원장](metrics.json) + +## 목적 + +**격리 복원 절차는 PASS지만, 성공 응답 65건 중 복원된 업로드는 33건이고 DB 미복원은 32건이다.** 기능 코호트 5건은 모두 복원됐고 문서 기능 6종·최종 정합성·원복 관문은 통과했다. 절차의 PASS를 데이터 무손실이나 운영 백업 완료로 해석하지 않는다. + +HTTP 업로드 성공을 받은 문서가 복원 DB의 관계와 GCS 원본 바이트까지 함께 보존됐는지 확인한다. 최초 상태를 Worker·Flyway·수선 전에 감사하고, 후속 인덱싱으로 최초 손실을 덮어쓰지 않는다. + +| 측정 대상 | 실제 결과 | +|---|---:| +| 장애 전 성공 응답 | 65건 = 기능 5건 + 손실 관측 60건 | +| 최초 `RESTORED` | 33건 = 기능 5건 + 손실 관측 28건 | +| 최초 `DB_MISSING` | 32건, 전부 손실 관측 코호트 | +| 결과 불명 / HTTP 오류 | 각각 0건 | +| DB 미복원 32건 중 원본만 보존됨 | 27건. 원본이 있어도 DB 관계가 없어 업로드 복원으로 세지 않음 | +| 부분 DB / 원본 누락 / 해시 불일치 | 각각 0건. DB가 미복원된 요청의 원본은 이 분류와 별도로 위에서 집계 | +| 연속 복구 성공 응답 경계와 T0 차이 | **412.061초**. 중간 누락 뒤 다시 복원된 성공 응답 0건 | +| 문서 기능 확인 | 6/6 통과 | +| 최종 감사 대상 | 복원 문서 33건 + 새 문서 1건 = 34건 | +| 미완료 Job / 중복 Chunk / 중복 Embedding / 누락 Embedding | 각각 0건 | + +이는 **한 회차의 관측 손실 구간**이며 운영 RPO 보장이 아니다. 결과 불명 요청은 이번 실제 회차에서 발생하지 않았고, 계약 시험에서만 다뤘다. + +## 테스트 환경 + +| 구성요소 | 실제 사용·역할 | 경계 | +|---|---|---| +| 제품 코드 | `3bff887a5f6abdf4155957a658599f8f0366a141`, Java 17 JAR. SHA-256 `08c5c458110bc96fc4c92cc5b02bd160f6c08b59e21d9ef536f2fa50aa76b27d` | Java 제품 코드 변경 없음 | +| OpenSQL | 실제 설치된 PostgreSQL 17.8 기반 OpenSQL, vector 0.8.1, 기존 3노드의 전용 논리 DB | 물리 기준 백업은 클러스터 단위. 감사·HTTP는 시험 DB만 사용 | +| 복원 대상 | 기존 비동기 Standby 호스트를 직렬 재사용, 별도 PGDATA·소켓·포트 | 원래 DB와 동시 실행하지 않음. 리더·동기 Standby 유지. 1,800초 자동 원복 장치 | +| 앱·캐시 | 로컬 전용 JVM·Valkey. 복원 뒤 Worker 동시 처리 상한 2 | 공개 앱 설정·개인 환경 파일 변경 없음 | +| 임베딩 | 실제 BGE-M3, 1,024차원. 기존 모델 컨테이너 재사용 | 제어 프록시는 실제 응답의 대기만 제어. 가짜 벡터 없음 | +| GCS | source·preserved·database의 시험 소유 버킷 3개 | 공개 원본 버킷과 분리, generation·SHA-256·크기 고정 | +| 백업 도구 | Barman 3.19.1, PostgreSQL 클라이언트 17.8, `file`·`rsync` | 별도 PostgreSQL 서버로 대체하지 않음. 영구 replication slot 추가 없음 | +| 복원 도구 | 네트워크 없는 새 도구 컨테이너·빈 카탈로그 | 동결 입력만 마운트. 원본 카탈로그·ADC 마운트 없음 | + +계측은 `DR-RESTORE-20261018-FORMAL1` 한 회차다. 준비 단계는 2026-10-18 12:32 UTC 이후, 요청 구간은 12:49–13:01 UTC다. 한국 시각은 각각 21:32 이후·21:49–22:01이다. 모든 원장에는 시간대가 포함된다. 이 환경은 전용 성능 벤치마크 장비가 아니다. + +## 테스트 방법 + +```text +그림 1 — 성공 응답부터 외부 입력까지 [실행] + +시험 클라이언트 + ├─ 요청 전: request_id·Idempotency-Key·원본 SHA·크기 기록 + └─ POST /api/documents + → DocumentUploadFacade: 원본 GCS 저장·후보 검증 + → DocumentUploadService: DB Transaction + Document + Version + FileObject 관계 + Outbox Event + EmbeddingJob + 요청 키의 완료 응답 + ← HTTP 201 + 문서·버전·파일·Job ID 4종 + └─ 응답 직후: DB 밖 해시 연결 원장에 ACK 기록 + +시험 DB ── 실제 스트리밍 수신 ── Barman 기준 백업·완성 WAL·identity +원본 GCS ── 실제 바이트·generation 확인 ── 원본 보존 버킷 + │ + 300초 예약 전송 두 번 + 준비 전송 한 번 + ▼ + T0 뒤 추가 취득하지 않는 고정 입력 +``` + +업로드는 [원본 저장](https://github.com/DocGrid/docgrid/blob/3bff887a5f6abdf4155957a658599f8f0366a141/backend/src/main/java/com/opensource/docgrid/domain/document/service/DocumentUploadFacade.java#L90-L127)과 [DB 원장·Job·응답 기록](https://github.com/DocGrid/docgrid/blob/3bff887a5f6abdf4155957a658599f8f0366a141/backend/src/main/java/com/opensource/docgrid/domain/document/service/command/DocumentUploadService.java#L101-L160)을 거친다. DB 백업과 객체 사본의 진도가 같다고 가정하지 않고 성공 응답별로 대조한다. + +완료 문서 4건과 실제 모델 응답 대기 중 JVM을 종료한 Job 1건을 먼저 준비했다. 중단 직전에는 `PROCESSING`, 문서 `INDEXING`, 버전 `EMBEDDING`, Attempt 1개·Chunk 3개·Embedding 0개였다. 이후 Worker를 끄고 기준 백업을 만들었다. 손실 관측 60건은 목표 11초 간격이며 전송 동안 업로드가 정지하므로 실제 간격은 원장을 기준으로 한다. + +| 외부 전송 | 완료 UTC | 기준 백업 크기 | 완성 WAL + history 누계 | 보존 원본 누계 | 실제 전송 시간 | +|---|---|---:|---:|---:|---:| +| 준비 | 12:44:55.930 | 142,940,160바이트 | 4개 | 5건 | 59.888초 | +| 첫 예약 구간 | 12:55:35.364 | 같은 백업 | 7개 | 33건 | 73.514초 | +| 둘째 예약 구간 | 13:00:22.660 | 같은 백업 | 8개 = WAL 7개 + history 1개 | 60건 | 63.774초 | + +예약 시작 지연은 각각 5.005초·2.033초였다. 전송 경계에서는 `pg_switch_wal()`을 실행하고 완성된 파일만 보존했다. 전송 주기 300초만으로 RPO 300초를 보장하는 구성은 아니다. + +```text +그림 2 — 최초 감사가 활성화를 통제하는 내부 순서 [실행] + +T0 13:01:13.892692 UTC: 시험 API 중지 시작 + → API·수집기·원본 DB 터널 종료, 리스너 2개 닫힘 + → 원본 GCS 드라이버 접근 거부 확인 + → cutoff 13:01:24.539 UTC: 입력 고정 (T0 +10.646초) + │ + ▼ +외부 generation 다운로드 → SHA·크기 대조 → 빈 카탈로그 import·verify + → 새 PGDATA → 같은 라이선스 호스트에서 실제 OpenSQL 기동 + │ + ▼ +snapshot: 복원 소켓·data_directory 확인 + 읽기 전용 SQL + → initial_upload_requests를 user_id + request_key로 조회 + → 응답 ID 4종·관계 + 동결 보존 객체의 바이트 대조 + │ + ▼ +13:03:28.438 UTC: 최초 감사 단독 저장·fsync + ├─ 기능 코호트 5/5 RESTORED, DB_PARTIAL/객체/해시 오류 0 + └─ DB_MISSING 32건을 그대로 보존 → 활성화 허용 + │ + ▼ +복원 DB만 보존 버킷으로 재지정 → Flyway·Worker 기동 → 복원 계정 로그인 + → 실제 HTTP 기능 검사 → 최종 감사 별도 저장 → 무조건 원복·정리 +``` + +원본 버킷 IAM 차단이나 원본 사이트 전체 소실을 재현하지 않았다. 원본 데이터 보충은 드라이버와 복원 컨테이너 경계로 막았고, 기존 호스트의 역할 확인·원복 제어는 유지했다. 복원 서버는 원본 연결 정보를 비우고 동결한 마지막 WAL까지 재생했다. + +## 관측 결과 + +| 기능·절차 | 표본·결과 | cutoff 이후 | T0 이후 | +|---|---|---:|---:| +| 기존 다운로드 | 4건 HTTP 200·크기·SHA 일치 | 176.992초 | **187.638초** | +| 기존 검색 | 사전 고정 합성 Chunk 질의·기대 문서 Top 20 포함 1/1 | 192.841초 | **203.487초** | +| 중단 Job 재개 | 같은 Job `INDEXED`, Job 1개·Attempt 2개 | 238.483초 | **249.130초** | +| 새 업로드 | HTTP 201, 1/1 | 246.122초 | **256.769초** | +| 새 인덱싱 | `INDEXED`, 1/1 | 571.408초 | **582.054초** | +| 새 다운로드 | HTTP 200·원본 SHA 일치, 1/1 | 573.186초 | **583.832초** | + +복원 연산자 자체는 100.672초다(13:01:24.555–13:03:05.227 UTC). 원격 호스트 준비·기동을 포함하며 전체 기능 복구 시간과 다르다. 각 기능은 순차적으로 관측했으므로 **확인 시각**이며 실제로 처음 가능해진 최소 시각을 추정하지 않는다. 새 인덱싱 시간에는 복원된 대기 Job 처리 시간이 포함된다. + +```text +그림 3 — DB·GCS 시점 차이와 사용자 기능 [관측] + +65건의 ACK 순서 + ┌─────────────────────┬──────────────────────┐ + │ 앞 33건: RESTORED │ 뒤 32건: DB_MISSING │ + │ 기능 5 + 손실 28 │ 모두 손실 코호트 │ + └─────────────────────┴──────────────────────┘ + 마지막 복원 ACK 12:54:21.832 UTC + └─ T0까지 412.061초, 중간 누락 뒤 복원된 ACK 0건 + +원본 보존은 60건 + ├─ DB도 복원: 33건 → 바이트·관계 대조 통과 + └─ 원본만 존재: 27건 → 문서 복원으로 계산하지 않음 + 나머지 5건은 이번 동결 원본에도 포함되지 않음 + +복원 뒤 + 기존 다운로드 T0+187.638초 → 검색 +203.487초 → Job 재개 +249.130초 + → 새 업로드 +256.769초 → INDEXED +582.054초 → 원본 대조 +583.832초 + → 최종 34문서: 미완료/중복/누락 각 0 → 원래 클러스터 복귀 +``` + +### 결과 아티팩트 + +| 실행·목적 | 위치·방법 | 판정·근거 | +|---|---|---| +| 본시험 준비 | 전용 DB·실제 GCS/BGE/Barman, 9단계 | [준비 결과](runs/DR-RESTORE-20261018-FORMAL1/preparation-result.json), Barman 24/24·백업 DONE·verify 0 | +| 본시험 | 격리 OpenSQL·인증 HTTP, 성공 응답 65건 | [요약](runs/DR-RESTORE-20261018-FORMAL1/summary.json), [이벤트](runs/DR-RESTORE-20261018-FORMAL1/events.jsonl), [동결 입력](runs/DR-RESTORE-20261018-FORMAL1/inputs.json) | +| 최초 감사 | Worker·Flyway·수선 전, 요청 키·ID·원본 대조 | [최초 스냅샷](runs/DR-RESTORE-20261018-FORMAL1/initial-snapshot.json), [최초 감사](runs/DR-RESTORE-20261018-FORMAL1/initial-audit.json) | +| 기능·최종 상태 | 실제 BGE·다운로드·검색·신규 작업 | [기능 시각](runs/DR-RESTORE-20261018-FORMAL1/functional-observations.json), [최종 스냅샷](runs/DR-RESTORE-20261018-FORMAL1/final-snapshot.json), [최종 감사](runs/DR-RESTORE-20261018-FORMAL1/final-audit.json) | +| WAL 경계 추가 조회 | 복원 DB 읽기 전용 SQL·파일명 기반 끝 LSN 계산 | [관측](checks/wal-boundary-r1/result.json), [분석](checks/wal-boundary-r1/analysis.json) | +| 원복·정리 | 원래 프로세스 재개·역할·설정 해시·소유 자원 확인 | [정리 기록](runs/DR-RESTORE-20261018-FORMAL1/runtime-cleanup-observations.json) | +| 계약 회귀 | 로컬 Python, 실제 Java enum + 합성 감사/제어 프로세스 | [40/40 통과 로그](checks/contract-r5/events.log). 실제 클라우드 실패 입력 40종을 재현했다는 뜻은 아님 | + +개별 준비 단계와 전송 회차의 spec·result는 비공개 작업 보관소에서 실행별로 분리했다. 저장소에는 최초·최종 감사, 필수 준비 결과, 불완전 회차, 정리 등 필요한 증거 36개만 공개했다. 이전 리허설은 기동 단계에서 `INCOMPLETE`, 후속 기동 진단은 각각 `INCOMPLETE`·진단 `PASS`이며 본시험에 합산하지 않는다. [준비·재시도 상세](preparation-findings.md)를 함께 읽어야 한다. + +## 검증 과정에서 확인된 사항 + +1. **외부 전송 성공과 데이터 최신성은 다르다.** 마지막 입력에는 WAL 7개·history 1개와 원본 60건이 있었지만 실제 DB 복구는 ACK 33건까지였다. 복원 DB의 마지막 재생 LSN `0/25000000`은 동결한 마지막 WAL 끝과 일치한다. 마지막 재생 트랜잭션은 12:54:21.655964 UTC다. 고정 입력을 복원 중 빠뜨렸다는 근거는 발견하지 못했다. 수신·archiver·전송 사이 최신성 차이의 세부 원인은 이 회차에서 분리하지 않았다. 후속 운영화에는 최신 완료 쓰기와 외부 WAL 도달 경계를 연결하는 점검이 필요하다. +2. **판정기 상태 계약을 보완했다.** 기존 공통 목록은 실제 문서 `INDEXING`·버전 `EMBEDDING` 등을 거부했다. 엔티티별 상태 목록으로 나누고 실제 Java enum 18종·잘못된 엔티티 상태 3종을 독립 입력으로 시험했다. 수정 전 4개 메서드 실패 → 수정 후 40/40 통과다. 제품 Java 코드 변경은 없다. +3. **라이선스 경로는 복원 런처의 실행 조건이었다.** 같은 호스트·같은 백업 바이트에서 기존 `OPENSQL_LICENSE_PATH`를 전달한 뒤 실제 기동이 됐다. 설치 라이선스 수정·재발급이나 제품 결함·만료 판정은 하지 않았다. 첫 리허설은 원시 stderr를 보존하지 않아 후속 진단 오류를 소급 적용하지 않았다. +4. **시간·보장의 범위가 제한된다.** 외부 전송마다 WAL 전환을 제어했고 스트리밍/수신기 한 회차만 시험했다. 장기 정기 백업·수신 중단 경보·Primary 교체 재연결, OpenProxy 경로, 공개 사용자 전환, Ollama 생성 답변, 전체 사이트 소실은 미검증이다. 이 값은 공개 서비스 RTO나 RPO 0 보장이 아니다. +5. **증거와 비공개 실행 정보는 분리했다.** 앱·SSH 원시 출력은 사전 허용 이벤트 또는 사전 마스킹으로 수집했다. 접속 argv·인증정보·내부 자원 식별자·원본 백업·라이선스는 공개하지 않는다. 공개 사본의 해시 목록은 [export-index.json](export-index.json)에 있다. 접속 연산자의 임시 구현 전체를 배포하는 원클릭 클라우드 복구 패키지는 아니다. + +### 변경 구성요소 + +| 파일·구성요소 | 변경 목적 | +|---|---| +| `scripts/opensql/dr_recovery.py` | 문서·버전·Job의 상태 허용 목록을 실제 Java 계약별로 구분 | +| `scripts/opensql/test_dr_recovery.py` | Java enum을 독립 입력으로 읽고 엔티티별 수용·오입력 거부를 회귀 검증 | +| 이 결과 디렉터리 | 사전 계획·실제 절차·불완전 회차·최초/최종 감사·시간·정리를 함께 공개 | + +시험 버킷 3개 삭제·임시 ADC 복사본 제거·개인 환경 파일 두 개의 동일성·사용자용 원본 버킷 보존은 [별도 작업 정리 결과](checks/task-cleanup-r1/result.json)에서도 확인했다. 원래 ADC는 철회하지 않았다. + +공개 설정과 개인 환경 파일을 배포용으로 변경하지 않았다. 시험 GCS 객체 삭제는 활성 세대 정리이며, soft delete 보존이 남을 수 있으므로 즉시 물리 소거로 표현하지 않는다. diff --git a/docs/test-results/dr-functional-recovery-20261018/checks/contract-r5/events.log b/docs/test-results/dr-functional-recovery-20261018/checks/contract-r5/events.log new file mode 100644 index 00000000..7830ed00 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/checks/contract-r5/events.log @@ -0,0 +1,45 @@ +test_audit_failure_never_activates_and_cleans_up (test_dr_recovery.RecoveryContractTest) ... ok +test_bad_later_command_blocks_restore (test_dr_recovery.RecoveryContractTest) ... ok +test_changed_inputs_block_operator (test_dr_recovery.RecoveryContractTest) ... ok +test_cleanup_failure_is_not_pass (test_dr_recovery.RecoveryContractTest) ... ok +test_corrupt_ledger_is_not_summarized (test_dr_recovery.RecoveryContractTest) ... ok +test_document_status_wire_contract (test_dr_recovery.RecoveryContractTest) ... ok +test_duplicate_response_rejected (test_dr_recovery.RecoveryContractTest) ... ok +test_duplicate_wal_rejected (test_dr_recovery.RecoveryContractTest) ... ok +test_final_snapshot_cannot_use_original_cluster (test_dr_recovery.RecoveryContractTest) ... ok +test_finished_run_cannot_be_executed_again (test_dr_recovery.RecoveryContractTest) ... ok +test_frozen_inputs_cannot_be_replaced (test_dr_recovery.RecoveryContractTest) ... ok +test_http_error_may_be_present_in_restored_db (test_dr_recovery.RecoveryContractTest) ... ok +test_initial_loss_preserved_when_final_snapshot_is_repaired (test_dr_recovery.RecoveryContractTest) ... ok +test_initial_state_rejects_started_worker_or_flyway (test_dr_recovery.RecoveryContractTest) ... ok +test_inputs_missing_check_is_no_go (test_dr_recovery.RecoveryContractTest) ... ok +test_invalid_functional_receipt_is_not_persisted (test_dr_recovery.RecoveryContractTest) ... ok +test_job_status_wire_contract (test_dr_recovery.RecoveryContractTest) ... ok +test_missing_cleanup_blocks_all_actions (test_dr_recovery.RecoveryContractTest) ... ok +test_missing_observation_is_not_db_loss (test_dr_recovery.RecoveryContractTest) ... ok +test_missing_pinned_originals_is_no_go (test_dr_recovery.RecoveryContractTest) ... ok +test_missing_response_is_unknown_not_failure (test_dr_recovery.RecoveryContractTest) ... ok +test_object_missing_blocks_worker (test_dr_recovery.RecoveryContractTest) ... ok +test_old_and_new_success_do_not_hide_middle_loss (test_dr_recovery.RecoveryContractTest) ... ok +test_original_identity_must_be_preserved (test_dr_recovery.RecoveryContractTest) ... ok +test_partial_wal_rejected (test_dr_recovery.RecoveryContractTest) ... ok +test_private_consistency_field_is_rejected (test_dr_recovery.RecoveryContractTest) ... ok +test_private_extra_row_fields_are_rejected_before_recording (test_dr_recovery.RecoveryContractTest) ... ok +test_private_extra_snapshot_fields_are_rejected_before_recording (test_dr_recovery.RecoveryContractTest) ... ok +test_private_value_in_missing_row_is_rejected (test_dr_recovery.RecoveryContractTest) ... ok +test_racing_response_after_audit_is_rejected_when_replayed (test_dr_recovery.RecoveryContractTest) ... ok +test_real_hash_mismatch_is_not_restored (test_dr_recovery.RecoveryContractTest) ... ok +test_response_ledger_is_durable_and_private (test_dr_recovery.RecoveryContractTest) ... ok +test_restore_failure_cleans_up (test_dr_recovery.RecoveryContractTest) ... ok +test_restore_receipt_must_bind_frozen_manifest (test_dr_recovery.RecoveryContractTest) ... ok +test_status_from_another_entity_is_rejected (test_dr_recovery.RecoveryContractTest) ... ok +test_success_requires_all_four_ids (test_dr_recovery.RecoveryContractTest) ... ok +test_success_response_ids_must_match_restored_relations (test_dr_recovery.RecoveryContractTest) ... ok +test_success_stages_audit_before_activation_then_final_audit (test_dr_recovery.RecoveryContractTest) ... ok +test_version_status_wire_contract (test_dr_recovery.RecoveryContractTest) ... ok +test_wrong_preserved_generation_blocks_activation (test_dr_recovery.RecoveryContractTest) ... ok + +---------------------------------------------------------------------- +Ran 40 tests in 0.964s + +OK diff --git a/docs/test-results/dr-functional-recovery-20261018/checks/contract-r5/result.json b/docs/test-results/dr-functional-recovery-20261018/checks/contract-r5/result.json new file mode 100644 index 00000000..e8b49b01 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/checks/contract-r5/result.json @@ -0,0 +1,14 @@ +{ + "command": "python3 -m unittest discover -s scripts/opensql -p test_dr_recovery.py -v", + "elapsed_seconds": 1.014, + "exit_code": 0, + "finished_at_utc": "2026-10-18T12:32:21.944469+00:00", + "location": "local Python, no cloud changes", + "purpose": "실제 Java 상태 enum 정합성과 복원 판정·최초 감사 관문 회귀 확인", + "revision": "3bff887 plus uncommitted enum-contract fix", + "run_id": "DR-CONTRACT-20261018-R5", + "started_at_utc": "2026-10-18T12:32:20.930547+00:00", + "success_criteria": "40개 테스트 메서드 통과, 실패·오류 0", + "test_methods": 40, + "verdict": "PASS" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/checks/contract-r5/spec.json b/docs/test-results/dr-functional-recovery-20261018/checks/contract-r5/spec.json new file mode 100644 index 00000000..be410abd --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/checks/contract-r5/spec.json @@ -0,0 +1,9 @@ +{ + "command": "python3 -m unittest discover -s scripts/opensql -p test_dr_recovery.py -v", + "location": "local Python, no cloud changes", + "purpose": "실제 Java 상태 enum 정합성과 복원 판정·최초 감사 관문 회귀 확인", + "revision": "3bff887 plus uncommitted enum-contract fix", + "run_id": "DR-CONTRACT-20261018-R5", + "started_at_utc": "2026-10-18T12:32:20.930547+00:00", + "success_criteria": "40개 테스트 메서드 통과, 실패·오류 0" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r1/result.json b/docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r1/result.json new file mode 100644 index 00000000..112fe871 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r1/result.json @@ -0,0 +1,17 @@ +{ + "command": "python3 -m unittest discover -s scripts/opensql -p test_dr_recovery.py -v", + "controller_sha256": "f14e3732386f568ad62b206897a49a668ffd1a11b5acf3cd3d204ab05bcee03f", + "elapsed_seconds": 0.786, + "errors": 7, + "failed_cases": 3, + "failed_methods": 4, + "finished_at_utc": "2026-10-18T09:22:58.096989+00:00", + "location": "local Python, synthetic snapshot and actual enum source", + "purpose": "Reproduce actual Java enum values rejected by the recovery snapshot contract", + "revision": "3bff887 + added protocol tests; controller unchanged", + "run_id": "DR-STATUS-COMPAT-20261018-R1", + "started_at_utc": "2026-10-18T09:22:57.294170+00:00", + "success_criteria": "All 40 test methods pass, including actual Java enum acceptance and wrong-entity rejection", + "test_methods": 40, + "verdict": "FAIL" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r1/spec.json b/docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r1/spec.json new file mode 100644 index 00000000..9e1098e4 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r1/spec.json @@ -0,0 +1,10 @@ +{ + "command": "python3 -m unittest discover -s scripts/opensql -p test_dr_recovery.py -v", + "controller_sha256": "f14e3732386f568ad62b206897a49a668ffd1a11b5acf3cd3d204ab05bcee03f", + "location": "local Python, synthetic snapshot and actual enum source", + "purpose": "Reproduce actual Java enum values rejected by the recovery snapshot contract", + "revision": "3bff887 + added protocol tests; controller unchanged", + "run_id": "DR-STATUS-COMPAT-20261018-R1", + "started_at_utc": "2026-10-18T09:22:57.294170+00:00", + "success_criteria": "All 40 test methods pass, including actual Java enum acceptance and wrong-entity rejection" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r2/result.json b/docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r2/result.json new file mode 100644 index 00000000..5c993036 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r2/result.json @@ -0,0 +1,17 @@ +{ + "command": "python3 -m unittest discover -s scripts/opensql -p test_dr_recovery.py -v", + "controller_sha256": "be15ba19ecab6659682b04bb6fc83040ebed8f96cb1a081fef9046b7e13c9a7c", + "elapsed_seconds": 1.066, + "errors": 0, + "failed_cases": 0, + "failed_methods": 0, + "finished_at_utc": "2026-10-18T09:23:25.261391+00:00", + "location": "local Python, synthetic snapshot and actual enum source", + "purpose": "Verify enum-specific snapshot status validation after the reproduced mismatch fix", + "revision": "3bff887 + snapshot enum fix and tests", + "run_id": "DR-STATUS-COMPAT-20261018-R2", + "started_at_utc": "2026-10-18T09:23:24.172949+00:00", + "success_criteria": "All 40 test methods pass, including 18 actual Java states and three wrong-entity rejections", + "test_methods": 40, + "verdict": "PASS" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r2/spec.json b/docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r2/spec.json new file mode 100644 index 00000000..84bb1a2b --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/checks/status-compat-r2/spec.json @@ -0,0 +1,10 @@ +{ + "command": "python3 -m unittest discover -s scripts/opensql -p test_dr_recovery.py -v", + "controller_sha256": "be15ba19ecab6659682b04bb6fc83040ebed8f96cb1a081fef9046b7e13c9a7c", + "location": "local Python, synthetic snapshot and actual enum source", + "purpose": "Verify enum-specific snapshot status validation after the reproduced mismatch fix", + "revision": "3bff887 + snapshot enum fix and tests", + "run_id": "DR-STATUS-COMPAT-20261018-R2", + "started_at_utc": "2026-10-18T09:23:24.172949+00:00", + "success_criteria": "All 40 test methods pass, including 18 actual Java states and three wrong-entity rejections" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/checks/task-cleanup-r1/result.json b/docs/test-results/dr-functional-recovery-20261018/checks/task-cleanup-r1/result.json new file mode 100644 index 00000000..aa1f0314 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/checks/task-cleanup-r1/result.json @@ -0,0 +1,42 @@ +{ + "finished_at_utc": "2026-10-18T13:13:37.469354+00:00", + "intended_original_bucket_preserved": true, + "key_files_checked": 0, + "personal_environment_comparison": { + ".env": { + "exists": true, + "unchanged": true + }, + "frontend/.env.local": { + "exists": true, + "unchanged": true + } + }, + "purpose": "시험 소유 버킷·임시 ADC 복사본·남은 소유 SSH 키 정리 및 개인 환경 보존 확인", + "remaining_owned_keys_confirmed_removed": true, + "run_id": "DR-TASK-CLEANUP-20261018-R1", + "soft_delete_is_not_physical_erasure": true, + "source_adc_file_present": true, + "source_adc_revoked": false, + "started_at_utc": "2026-10-18T13:13:33.644785+00:00", + "success_criteria": "시험 버킷 3개 삭제·의도한 공개 원본 버킷 유지·임시 인증 파일 제거, 원래 자격 증명 보존", + "temporary_adc_copy_removed": true, + "trial_buckets": [ + { + "delete_http_status": 204, + "role": "source", + "soft_deleted_generations_may_remain": true + }, + { + "delete_http_status": 204, + "role": "preserved", + "soft_deleted_generations_may_remain": true + }, + { + "delete_http_status": 204, + "role": "database", + "soft_deleted_generations_may_remain": true + } + ], + "verdict": "PASS" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/checks/task-cleanup-r1/spec.json b/docs/test-results/dr-functional-recovery-20261018/checks/task-cleanup-r1/spec.json new file mode 100644 index 00000000..6c333a61 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/checks/task-cleanup-r1/spec.json @@ -0,0 +1,6 @@ +{ + "purpose": "시험 소유 버킷·임시 ADC 복사본·남은 소유 SSH 키 정리 및 개인 환경 보존 확인", + "run_id": "DR-TASK-CLEANUP-20261018-R1", + "started_at_utc": "2026-10-18T13:13:33.644785+00:00", + "success_criteria": "시험 버킷 3개 삭제·의도한 공개 원본 버킷 유지·임시 인증 파일 제거, 원래 자격 증명 보존" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/checks/wal-boundary-r1/analysis.json b/docs/test-results/dr-functional-recovery-20261018/checks/wal-boundary-r1/analysis.json new file mode 100644 index 00000000..7f997c85 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/checks/wal-boundary-r1/analysis.json @@ -0,0 +1,12 @@ +{ + "all_frozen_segment_bytes_reached": true, + "cause_of_offsite_freshness_gap": "NOT_ISOLATED", + "derived_at_utc": "2026-10-18T13:09:21.018235+00:00", + "frozen_wal_end_lsn": "0/25000000", + "last_frozen_wal": "000000060000000000000024", + "last_replay_transaction_utc": "2026-10-18T12:54:21.655964", + "method": "last WAL filename and declared 16 MiB segment length compared with pg_last_wal_replay_lsn", + "observed_last_replay_lsn": "0/25000000", + "rpo_guarantee": "NOT_EVALUATED", + "run_id": "DR-RESTORE-WAL-BOUNDARY-20261018-R1" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/checks/wal-boundary-r1/result.json b/docs/test-results/dr-functional-recovery-20261018/checks/wal-boundary-r1/result.json new file mode 100644 index 00000000..99a138db --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/checks/wal-boundary-r1/result.json @@ -0,0 +1,25 @@ +{ + "finished_at_utc": "2026-10-18T13:05:37.215284+00:00", + "frozen_wal_segments": [ + "00000006000000000000001E", + "00000006000000000000001F", + "000000060000000000000020", + "000000060000000000000021", + "000000060000000000000022", + "000000060000000000000023", + "000000060000000000000024" + ], + "location": "same isolated restored OpenSQL, explicit recovery socket", + "method": "pg_last_wal_replay_lsn / pg_last_xact_replay_timestamp read-only SQL", + "observation": { + "last_replay_lsn": "0/25000000", + "last_replay_transaction_utc": "2026-10-18T12:54:21.655964", + "standby": false, + "wal_segment_size": "16MB" + }, + "purpose": "동결 WAL 입력의 끝과 실제 복원 DB 재생 경계를 읽기 전용 대조", + "run_id": "DR-RESTORE-WAL-BOUNDARY-20261018-R1", + "started_at_utc": "2026-10-18T13:05:27.031743+00:00", + "success_criteria": "복원 DB 전용 소켓 확인·실제 재생 LSN과 입력 목록 보존; 사용자 쓰기·원본 조회 없음", + "verdict": "OBSERVED" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/checks/wal-boundary-r1/spec.json b/docs/test-results/dr-functional-recovery-20261018/checks/wal-boundary-r1/spec.json new file mode 100644 index 00000000..5de93d5a --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/checks/wal-boundary-r1/spec.json @@ -0,0 +1,8 @@ +{ + "location": "same isolated restored OpenSQL, explicit recovery socket", + "method": "pg_last_wal_replay_lsn / pg_last_xact_replay_timestamp read-only SQL", + "purpose": "동결 WAL 입력의 끝과 실제 복원 DB 재생 경계를 읽기 전용 대조", + "run_id": "DR-RESTORE-WAL-BOUNDARY-20261018-R1", + "started_at_utc": "2026-10-18T13:05:27.031743+00:00", + "success_criteria": "복원 DB 전용 소켓 확인·실제 재생 LSN과 입력 목록 보존; 사용자 쓰기·원본 조회 없음" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/export-index.json b/docs/test-results/dr-functional-recovery-20261018/export-index.json new file mode 100644 index 00000000..43c363e3 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/export-index.json @@ -0,0 +1,185 @@ +{ + "purpose": "Only evidence required by the recovery report; full private preparation material is not published", + "artifacts": [ + { + "artifact": "checks/contract-r5/events.log", + "bytes": 3698, + "sha256": "b9cca6fd4c25a8b35e79fecbf27ab1d0e642e2665fdec8c109b0a6a4444fcd59" + }, + { + "artifact": "checks/contract-r5/result.json", + "bytes": 621, + "sha256": "b7e3993b68c52fa563c28a47dc26acceee49949fe47ec8201d7a6b883a8c1777" + }, + { + "artifact": "checks/contract-r5/spec.json", + "bytes": 475, + "sha256": "0b76e84ad9cae46c76e512285dbd028dd63e27ba41a34e5c1050683f4f5549a0" + }, + { + "artifact": "checks/status-compat-r1/result.json", + "bytes": 823, + "sha256": "13d6bea528716ca35a667ecbe52d758c21ddd594fb5382f0f1b3933b7d2e31d8" + }, + { + "artifact": "checks/status-compat-r1/spec.json", + "bytes": 636, + "sha256": "8644b2da9b9a9edd164a9c93c9357149cd37b1773f9f22c3301424966ad49b6e" + }, + { + "artifact": "checks/status-compat-r2/result.json", + "bytes": 814, + "sha256": "f9b1184427e3adf35c0ca1c558453af26064e44a811f2aa87a11678c6910318b" + }, + { + "artifact": "checks/status-compat-r2/spec.json", + "bytes": 627, + "sha256": "f034358a064c051333169f550d95328d33253bc335e56d418ccaed3ee32cd67c" + }, + { + "artifact": "checks/task-cleanup-r1/result.json", + "bytes": 1304, + "sha256": "d38337dd61fb112efe2a4e10426e4d70f612f517db0ac7677880f119dc2b6de6" + }, + { + "artifact": "checks/task-cleanup-r1/spec.json", + "bytes": 371, + "sha256": "171b66b5f39f09ab2e00ee551f695e7f9ad4d5241af51f3752c54e49c073ddf7" + }, + { + "artifact": "checks/wal-boundary-r1/analysis.json", + "bytes": 541, + "sha256": "82ce943dbc2496683930f53146f755c3f5e700e475a14838190deaa92efdfcdb" + }, + { + "artifact": "checks/wal-boundary-r1/result.json", + "bytes": 1022, + "sha256": "56e85e22a72b71e8e79c898fb476c426f1828758ef5f7544e83384932a326981" + }, + { + "artifact": "checks/wal-boundary-r1/spec.json", + "bytes": 506, + "sha256": "a945b765d82218b27fc0758eab643c40c5866b40625d361e2a11fe74c629bf6f" + }, + { + "artifact": "metrics.json", + "bytes": 3140, + "sha256": "3c4ddf939768ada18efa33ec76b3296e874febee674c8ff36133e8dc882aec23" + }, + { + "artifact": "runs/DR-PG-START-DIAG-20261018-R1/result.json", + "bytes": 922, + "sha256": "c35548f9ed461502c960e0b95d3fc8e2f983da68eff89d7b1dbafdd34272f2ed" + }, + { + "artifact": "runs/DR-PG-START-DIAG-20261018-R1/source-capture.json", + "bytes": 729, + "sha256": "09cb643b0c046ae1e8f8c4aed8d1cf98af7a93a2efcb176039ae14151559318b" + }, + { + "artifact": "runs/DR-PG-START-DIAG-20261018-R1/spec.json", + "bytes": 622, + "sha256": "434ee42d5e089735aec252065bf2134a8e23078e84dc910e46073148fee58909" + }, + { + "artifact": "runs/DR-PG-START-DIAG-20261018-R2/result.json", + "bytes": 1132, + "sha256": "cb34f18d677f2db4c353d362e54f49288eb2b44eba53b59446bd11f6f67e4460" + }, + { + "artifact": "runs/DR-PG-START-DIAG-20261018-R2/spec.json", + "bytes": 720, + "sha256": "0e54b35d612a74f5ce4e2809736081eba6902f0841432be1a6a6226988d47b50" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/BACKUP-result.json", + "bytes": 541, + "sha256": "07637e7594d6368a47ab47a03260f9151082fe7c975c309401566c4d4f12264c" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/INTERRUPT-result.json", + "bytes": 620, + "sha256": "e2b332f82c67f2c15f56388f006fbade29a1ea451f56b72d460c32051c85a38c" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/RECEIVER-result.json", + "bytes": 2465, + "sha256": "20bef6bb1e90f33b642137f560432ef841c48242d35a2f1fc38890115a25a8d5" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/events.jsonl", + "bytes": 143528, + "sha256": "ff93e425516fcba5e4e504bd1da54b8a7dfd0e730eed75900bfc1f9a6d2c5f75" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/final-audit.json", + "bytes": 14971, + "sha256": "88ee63810f780eebff125d5844d06568fe8905efa41a3013af43e4aa6e90501b" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/final-snapshot.json", + "bytes": 23269, + "sha256": "6ff1af3002e9812ea86024aea8b47bf81d985cdf07caf2130bd8936103bd2058" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/formal-spec.json", + "bytes": 901, + "sha256": "bd87ef31dfd91351d58bd0aaeaa4e111a2ddc8ce963648cfe7e5ecd04b8cc2e4" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/functional-observations.json", + "bytes": 661, + "sha256": "ee57b78f4aedf70f3d6cf9763cd18631f4e80cc51c0324d0cfa474c4427af5fd" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/initial-audit.json", + "bytes": 14971, + "sha256": "88ee63810f780eebff125d5844d06568fe8905efa41a3013af43e4aa6e90501b" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/initial-snapshot.json", + "bytes": 23190, + "sha256": "a9f0d38fd7da13c17f23ca1042f8854b9a7cc104c0c3ff0c8f3941a5282d377e" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/inputs.json", + "bytes": 13135, + "sha256": "2c5afe03e8ed47c1924cc6082873d9e5ec52c5de808a98f0f351e807d9b20b95" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/manifest.json", + "bytes": 180, + "sha256": "297041f2f6ce019115950c70a034edea6f1aac6d1a61d7e2018a258ff1c1231f" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/preparation-result.json", + "bytes": 165, + "sha256": "99c1cd8b51cdeb61c77337e8b2e905a99c06be0a6c2aaa65504daaa5793b24eb" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/runtime-cleanup-observations.json", + "bytes": 941, + "sha256": "35021669a095eb43998dd44d087e2962b3847bab731d465bfe3bd5d029b3b424" + }, + { + "artifact": "runs/DR-RESTORE-20261018-FORMAL1/summary.json", + "bytes": 32991, + "sha256": "7a634d7020ef885f82cdf1e3a8789a627ef759157f8ab2c8da8f8cecbe2a1f7c" + }, + { + "artifact": "runs/DR-RESTORE-20261018-REHEARSAL1/operator-restore-error.json", + "bytes": 182, + "sha256": "ffbe5c1e7ec814163b391eb39826ee2c940349aaa75ef24debefb7e3101c8096" + }, + { + "artifact": "runs/DR-RESTORE-20261018-REHEARSAL1/ssh-cleanup-reconciliation-r2.json", + "bytes": 387, + "sha256": "6777087caf0ed94e93f6becebafa6193eedca1cfbb5df0a1c2702c41eec1c461" + }, + { + "artifact": "runs/DR-RESTORE-20261018-REHEARSAL1/summary.json", + "bytes": 285, + "sha256": "94ff5d2948841d83c758f16e1367e54a1d0357b35dce14b74f3bb035216df15c" + } + ] +} diff --git a/docs/test-results/dr-functional-recovery-20261018/metrics.json b/docs/test-results/dr-functional-recovery-20261018/metrics.json new file mode 100644 index 00000000..77cb9260 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/metrics.json @@ -0,0 +1,118 @@ +{ + "acknowledged": 65, + "cohorts": { + "functional": { + "acknowledged": 5, + "restored": 5, + "states": { + "RESTORED": 5 + } + }, + "loss": { + "acknowledged": 60, + "restored": 28, + "states": { + "DB_MISSING": 32, + "RESTORED": 28 + } + } + }, + "contiguous_prefix_acknowledged": 33, + "cutoff_minus_t0_seconds": 10.646, + "cutoff_utc": "2026-10-18T13:01:24.539+00:00", + "execution": "PASS", + "fault_t0_utc": "2026-10-18T13:01:13.892692+00:00", + "frozen_original_count": 60, + "functional_checks": { + "download": { + "at_utc": "2026-10-18T13:04:21.530770+00:00", + "elapsed_since_t0_seconds": 187.638, + "from_cutoff_seconds": 176.992, + "passed": true, + "samples": 4 + }, + "job_resume": { + "at_utc": "2026-10-18T13:05:23.022354+00:00", + "elapsed_since_t0_seconds": 249.13, + "from_cutoff_seconds": 238.483, + "passed": true, + "samples": 1 + }, + "new_download": { + "at_utc": "2026-10-18T13:10:57.724793+00:00", + "elapsed_since_t0_seconds": 583.832, + "from_cutoff_seconds": 573.186, + "passed": true, + "samples": 1 + }, + "new_indexing": { + "at_utc": "2026-10-18T13:10:55.947040+00:00", + "elapsed_since_t0_seconds": 582.054, + "from_cutoff_seconds": 571.408, + "passed": true, + "samples": 1 + }, + "new_upload": { + "at_utc": "2026-10-18T13:05:30.661209+00:00", + "elapsed_since_t0_seconds": 256.769, + "from_cutoff_seconds": 246.122, + "passed": true, + "samples": 1 + }, + "search": { + "at_utc": "2026-10-18T13:04:37.379579+00:00", + "elapsed_since_t0_seconds": 203.487, + "from_cutoff_seconds": 192.841, + "passed": true, + "samples": 1 + } + }, + "initial_states": { + "DB_MISSING": 32, + "RESTORED": 33 + }, + "latest_restored_ack_utc": "2026-10-18T12:54:21.832000+00:00", + "missing_db_with_preserved_original": 27, + "observed_prefix_boundary_gap_seconds": 412.061, + "offsite_rounds": [ + { + "complete_wals": 4, + "elapsed_seconds": 59.888, + "event_at_utc": "2026-10-18T12:44:55.930+00:00", + "ordinal": 0, + "preserved_originals": 5 + }, + { + "completed_upload_requests": 33, + "event_at_utc": "2026-10-18T12:54:21.843+00:00", + "ordinal": 1, + "scheduled_at_utc": "2026-10-18T12:54:16.828378+00:00", + "start_lag_seconds": 5.005 + }, + { + "complete_wals": 7, + "elapsed_seconds": 73.514, + "event_at_utc": "2026-10-18T12:55:35.364+00:00", + "ordinal": 1, + "preserved_originals": 33 + }, + { + "completed_upload_requests": 60, + "event_at_utc": "2026-10-18T12:59:18.878+00:00", + "ordinal": 2, + "scheduled_at_utc": "2026-10-18T12:59:16.828378+00:00", + "start_lag_seconds": 2.033 + }, + { + "complete_wals": 8, + "elapsed_seconds": 63.774, + "event_at_utc": "2026-10-18T13:00:22.660+00:00", + "ordinal": 2, + "preserved_originals": 60 + } + ], + "reason": null, + "restored_after_first_hole": 0, + "rpo_guarantee": "NOT_EVALUATED", + "run_id": "DR-RESTORE-20261018-FORMAL1" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/plan.md b/docs/test-results/dr-functional-recovery-20261018/plan.md new file mode 100644 index 00000000..f994ee7e --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/plan.md @@ -0,0 +1,62 @@ +# 업로드 성공 응답과 격리 복원 기능 계측 계획 + +관련 이슈: #560. 이 문서는 실행 전 기준이며 결과는 회차별 기록으로 구분한다. + +## 목적·합격 기준 + +업로드 성공 응답을 DB 밖에 즉시 기록하고, 장애 이전에 고정한 외부 기준 백업·완성 WAL·GCS 원본 보존본으로 복원한다. 최초 상태 감사가 통과한 뒤에만 버킷 재지정·Flyway·Worker를 허용한다. 후속 인덱싱 결과로 최초 손실을 덮어쓰지 않는다. + +| 대상 | 계획한 확인 | +|---|---| +| 기능 코호트 | 실제 BGE 인덱싱 완료 문서 4건, 실제 모델 응답 대기 중 시험 JVM을 강제 종료한 Job 1건 | +| 손실 코호트 | 고유 요청 키의 합성 업로드 60건, 목표 전송 간격 11초. 실제 전송·응답 시각을 원장에 기록 | +| 외부 보존 | 기준 백업·identity와 완성 WAL, 원본 객체의 고정 generation·SHA-256·크기 | +| 주기 비교 | 손실 코호트를 300초 주기의 두 전송 구간 이상에 배치. 전송 지연·누락은 실제 시각으로 보고 | +| 최초 감사 | 요청 키, Document·Version·FileObject·Job ID와 연결, 원본 generation·바이트 대조 | +| 기능 계측 | 기존 다운로드·검색, 중단 Job 재개, 새 업로드·인덱싱·다운로드 | +| 최종 정합성 | 관측된 복원 문서의 미완료 Job·중복 Chunk·중복/누락 Embedding 0건 | +| 정리 | 원래 리더 1대·Standby 2대 복귀, 시험 소유 프로세스·터널·자격 증명 정리, 개인·공개 설정 보존 | + +## 환경과 통제 변수 + +- 코드 기준: `3bff887a5f6abdf4155957a658599f8f0366a141`. Java 제품 코드를 변경하지 않는다. 복원 판정기의 상태 허용 목록은 실제 Java enum에 맞춰 검증한다. +- 로컬 전용 JVM·Valkey·Barman 도구 컨테이너, 실제 실행 중인 BGE-M3, 클라우드의 전용 논리 DB와 시험 버킷을 사용한다. Ollama 생성 답변은 범위 밖이다. +- 새 클라우드 프로젝트에는 이전 백업 VM·카탈로그가 없으므로 시험용 수집기와 기준 백업을 새로 준비한다. 이전 프로젝트의 결과를 이번 준비 상태로 간주하지 않는다. +- 임시 수집기는 영구 replication slot과 Patroni DCS 설정 변경 없이 사용한다. Barman의 두 archiver를 켜되 이 수집기로의 원본 `archive_command` 전송을 추가하지 않고, 실제 스트리밍 경로만 시험한다. 장기 WAL 보존·수신 재연결을 보장하는 운영 구성으로 주장하지 않는다. +- 검색은 장애 전에 완료 문서의 합성 Chunk 본문과 해시를 고정하고, 복원 뒤 기대 문서가 Top 20에 포함되는지 확인한다. 정확한 순위·유사도 점수·검색 품질 개선을 합격 기준으로 삼지 않는다. +- 전송 경계의 요청이 완료된 뒤 WAL을 전환한다. 이 시험의 명시적 제어이며, 작은 쓰기 부하에서 완성되지 않은 WAL을 300초 전송 주기만으로 복구할 수 있다고 가정하지 않는다. +- 설치된 OpenSQL의 기존 비동기 Standby 호스트를 직렬 재사용한다. 복원 중 원래 클러스터는 리더 1대와 동기 Standby 1대로 운영된다. 동기 Standby·리더는 중지하지 않는다. 따라서 완전히 별도인 물리 클러스터 시험은 아니다. +- 실제 DB 중지 전 시험용 프로세스로 자동 복귀 타이머를 검증한다. 대상 역할·프로세스 시작 정보·소유 경로를 재확인하고, 1,800초 자동 원복 장치를 건 뒤 원래 PostgreSQL을 멈춘다. 같은 호스트에서 원래 DB와 복원 DB를 동시에 실행하지 않는다. + +## 실행 순서 + +```text +그림 1 — 변경 전 최초 감사 관문 [계획] + +클라이언트 → 요청 키·파일 SHA 기록 → 업로드 HTTP 응답의 ID 4종 기록 + │ +외부 보존 → 기준 백업·identity·완성 WAL·원본 generation 고정 + │ +원본 접근 상실 T0 → 시험 API 종료·전용 DB 터널 종료 + │ 복원 드라이버에서 원본 버킷 I/O 거부 확인 + │ 원본에서 복원 자료를 추가 취득하지 않음 + │ 원본 버킷의 클라우드 IAM 차단·전체 사이트 소실은 시험하지 않음 + ▼ +빈 카탈로그 → 고정 identity 준비 → import·verify → 새 PGDATA 복원 + │ +실제 OpenSQL 기동 → SQL·보존 객체 읽기만으로 최초 상태 감사 + ├─ 부분 DB·원본 누락·해시 불일치·기능 코호트 누락 → 활성화 금지·정리 + └─ 관문 통과 → 최초 감사 fsync → 버킷 재지정·Flyway·Worker 기동 +``` + +원본 데이터 접근은 T0 이후 금지한다. 기존 호스트의 역할 확인·자동 원복 제어는 필요하므로, 원래 인프라 전체가 사라진 시험으로 표현하지 않는다. + +리허설은 합성 손실 요청 8건·30초 전송 간격의 작은 부하로 먼저 확인한다. 준비 중단·재개로 간격이 달라지면 실제 시각을 기록하고 본시험의 300초 정책 검증에 합치지 않는다. + +## 기록과 해석 + +- 리허설·본시험·재시험의 원장, 카탈로그·PGDATA, 입력 목록과 결과는 분리한다. 준비 실패와 재시도도 숨기지 않는다. +- stdout·stderr는 허용한 이벤트 유형·상태·건수만 기록한다. 원시 앱 로그·연결 문자열·인증정보·내부 자원 식별자는 남기지 않는다. 이 때문에 세부 오류 진단이 제한된 경우를 명시한다. +- T0와 판정기 `cutoff`는 별도 시각이다. 기능별로 두 기준의 경과 시간을 구분한다. +- 유실은 성공 응답별로 대조하고 중간 누락도 확인한다. 결과 불명 요청과 후속 재전송 성공을 원래 성공 응답에 합치지 않는다. +- 보고 범위는 격리 문서 다운로드·검색·인덱싱의 경과 시간이다. 공개 서비스 전환, 전체 사이트 소실, RPO 0 보장, 실제 Ollama 답변 복구를 주장하지 않는다. diff --git a/docs/test-results/dr-functional-recovery-20261018/preparation-findings.md b/docs/test-results/dr-functional-recovery-20261018/preparation-findings.md new file mode 100644 index 00000000..602686cb --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/preparation-findings.md @@ -0,0 +1,38 @@ +# 준비·재시도에서 확인된 사항 + +본시험 이전에 실제로 확인한 문제와 재시도를 구분한다. 준비가 통과했다는 사실을 기능 복원 통과나 RPO 측정으로 합치지 않는다. UTC 원본 시각은 `+00:00`, 한국 시각은 여기에 9시간을 더한다. + +## 복원 판정기 상태 계약 + +| 회차 | 목적·실행 위치 | 관측 결과 | 해석·근거 | +|---|---|---|---| +| DR-STATUS-COMPAT-20261018-R1 | 로컬 Python, 실제 Java enum을 입력 계약으로 읽음 | 40개 메서드 중 4개 실패. assertion 3건·오류 7건 | 기존 공통 상태 목록이 실제 `INDEXING`·`EMBEDDING` 등 상태를 거부하고 다른 엔티티의 상태를 허용. [원본 판정](checks/status-compat-r1/result.json) | +| DR-STATUS-COMPAT-20261018-R2 | 엔티티별 허용 목록 적용 뒤 같은 계약 | 40/40 통과, 오류 0, 1.066초 | Java 상태 18종과 잘못된 엔티티 상태 3종을 독립 입력으로 확인. [결과](checks/status-compat-r2/result.json) | +| DR-CONTRACT-20261018-R5 | 본시험 전 최종 계약 회귀 | 40/40 통과, 1.014초 | `python3 -m unittest discover -s scripts/opensql -p test_dr_recovery.py -v`. 합성 입력·제어 자식 프로세스 시험이며 클라우드 복원 성공 증거는 아님. [로그](checks/contract-r5/events.log) | + +## 첫 복원 리허설과 라이선스 경로 + +| 회차 | 실제 도달 단계 | 판정 | 원복 | +|---|---|---|---| +| DR-RESTORE-20261018-REHEARSAL1 | 외부 입력 import·검증·새 PGDATA 준비 뒤 OpenSQL 기동에서 중단 | **INCOMPLETE**. 최초 감사·기능 시험 미실행. 성공 응답 13건을 수집했지만 복원 손실 건수는 판정하지 못함 | 원래 3노드·공개 설정 복구. 만료된 SSH 키 제거의 초기 미확인을 후속 프로필 조회로 보완. [요약](runs/DR-RESTORE-20261018-REHEARSAL1/summary.json), [키 재확인](runs/DR-RESTORE-20261018-REHEARSAL1/ssh-cleanup-reconciliation-r2.json) | +| DR-PG-START-DIAG-20261018-R1 | 같은 외부 백업 바이트를 새 PGDATA에 복원하고 네이티브 오류 수집 | **INCOMPLETE**. `FATAL: license is invalid` 확인. 첫 리허설은 원시 stderr를 보존하지 않아 같은 메시지가 나왔다고 소급 단정하지 않음 | 공개 설정 보존·3노드 복귀·소유 자원 정리 모두 확인. [결과](runs/DR-PG-START-DIAG-20261018-R1/result.json) | +| DR-PG-START-DIAG-20261018-R2 | 같은 호스트·같은 백업/WAL 해시, 기존 프로세스의 `OPENSQL_LICENSE_PATH`만 런처 환경에 추가 | **기동 진단 PASS**. 실제 OpenSQL 쓰기 가능 확인. 본시험 사전 조건은 NO-GO이며 기능·RPO 통과 회차가 아님 | 세 정리 조건 모두 확인. [결과](runs/DR-PG-START-DIAG-20261018-R2/result.json) | + +확인한 원인은 임시 복원 런처가 기존 설치의 라이선스 경로 환경을 전달하지 않은 것이다. 라이선스 내용·만료·제품 결함으로 결론내리지 않았다. 파일 내용을 읽거나 수정·재발급하지 않았다. 기존 정상 프로세스의 환경 경로를 같은 호스트에서 재사용했다. 새 팀 공통 `.env` 항목이나 공개 설정 변경은 없다. + +진단용 외부 객체를 되살릴 때에는 GCS soft delete의 기존 generation을 제자리에서 사용한 것이 아니다. 복원 API가 만든 **새 generation**을 읽고 원래 동결 바이트의 SHA-256·크기와 대조했다. 이 진단 절차는 본시험 데이터 보충에 사용하지 않았다. [GCS 복원 API](https://docs.cloud.google.com/storage/docs/json_api/v1/objects/restore) + +## Barman·모델·접속 준비 + +- **백업 함수 권한:** 함수 권한은 DB별이므로 관리자 DB에 준 권한만으로 새 논리 DB의 `pg_backup_start/stop`가 허용되지 않았다. 시험 DB에만 필요한 권한을 부여한 뒤 다시 점검했다. 공개 역할 권한을 넓히지 않았다. +- **Barman 잠금:** macOS Docker 바인드 디렉터리에서 경쟁 잠금이 감지되지 않았지만 컨테이너 네이티브 잠금 디렉터리에서는 감지됐다. 잠금 위치를 분리했다. 디렉터리 fsync는 두 위치 모두 통과했으므로 fsync 결함으로 설명하지 않는다. +- **도구 누락:** 최소 클라이언트 이미지에 `file` 실행 파일이 없어 WAL 처리 단계가 막혔다. 필요한 `file`·`rsync`를 추가한 전용 이미지에서 `DONE`과 `pg_verifybackup=0`을 확인했다. PostgreSQL 서버를 다른 제품으로 바꾸지 않았다. +- **제어 프록시:** 첫 모델 프록시는 chunked HTTP body를 전달하지 못했다. 전송 처리를 수정하고 실제 BGE로 인덱싱을 재확인했다. 당시 HTTPError만 수집한 회차의 구체적 상태 코드를 사후 추정하지 않는다. +- **초기 JVM 기동:** 첫 120초 대기 한도 안에서는 readiness가 오지 않았지만 같은 프로세스가 나중에 준비됐다. 본시험 초기 준비는 600초 한도에서 검사했고, 실제 277.613초 후 readiness와 리스너 소유권을 확인했다. +- **SQL 연결:** 도구 컨테이너의 새 PostgreSQL 연결에서 SQLSTATE 없는 OperationalError가 간헐적으로 관측됐다. IPv6 가설은 확인되지 않았다(AAAA 0개, 연결 비교 4/4 통과). 원인 미확정으로 남기고 원본 제어 SQL에는 이미 검증한 SSH·Unix 소켓 경로를 사용했다. + +본시험의 receiver 준비 결과는 Barman 24/24, streaming receiver 1개다. 이 결과는 장애 뒤 재연결·장기 운영 경보·영구 WAL 보관 정책까지 검증한 것이 아니다. 각 준비·재시도의 개별 JSON과 이벤트는 해당 실행 디렉터리에 분리한다. + +## 증거의 공개 범위 + +원시 앱·SSH 출력은 허용한 이벤트·상태·건수로 사전 축약하거나 사전 마스킹한 뒤 기록했다. 공개 사본은 자격 증명·내부 주소·프로젝트·버킷·호스트 식별자·개인 경로를 다시 검사한 파생 자료다. 인증정보·라이선스·백업 바이트·비공개 접속 연산자는 업로드하지 않는다. 따라서 수집하지 않은 원시 출력으로 세부 원인을 소급 재구성할 수 없다. diff --git a/docs/test-results/dr-functional-recovery-20261018/procedure.md b/docs/test-results/dr-functional-recovery-20261018/procedure.md new file mode 100644 index 00000000..f9544158 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/procedure.md @@ -0,0 +1,58 @@ +# 격리 복원 실행·감사 방법 + +관련 판정기: [`dr_recovery.py`](../../../scripts/opensql/dr_recovery.py). 이 문서는 이번 실험의 절차이며, 공개 서비스에 그대로 적용하는 자동 복구 매뉴얼은 아니다. 비공개 연산자에는 접속 대상·인증정보가 있으므로 저장소에 포함하지 않는다. + +## 실행 전 고정한 조건 + +1. 전용 논리 DB·계정·시험 버킷을 만들고, 기존 공개 앱의 DB·버킷·Patroni 설정은 유지한다. Java 제품 코드는 `3bff887a5f6abdf4155957a658599f8f0366a141`의 JAR을 사용한다. +2. 로컬 전용 JVM과 Valkey, 실제 BGE-M3, Barman 수집기를 연결한다. 원본·보존본·DB 백업은 서로 다른 시험 버킷을 사용한다. +3. 완료 문서 4건을 인덱싱한다. 다섯 번째 Job에서는 실제 BGE 응답을 제어 프록시에서 보류한 뒤 **시험 JVM만 종료**한다. 합성 벡터나 가짜 검색 결과를 넣지 않는다. +4. Job이 `PROCESSING`, Attempt 1건, Chunk 3건, Embedding 0건인지 SQL로 확인하고 Worker를 끈 시험 API를 다시 시작한다. +5. Barman `check`의 24개 항목과 실제 streaming receiver를 확인한다. 기준 백업 `DONE`, `pg_verifybackup` 종료 코드 0을 확인한다. 사전 보존한 identity와 완성 WAL만 외부에 전송한다. +6. 완료 문서의 실제 합성 Chunk를 검색 입력으로 고정한다. 본문 해시·기대 문서 ID·Top 20 포함 조건을 장애 전에 기록한다. 이는 검색 기능 복구 확인이며 품질 평가가 아니다. +7. 요청 키·파일 해시·크기를 전송 전에, HTTP 응답과 ID 4종을 응답 수신 직후 외부 원장에 기록한다. 60건의 목표 전송 간격은 11초이고 외부 전송 주기는 300초다. 실제 지연은 원장의 시각으로 판정한다. + +## T0·입력 동결 + +- 시험 API 중지 시작 시각을 T0로 기록한다. 수집기와 전용 원본 DB 터널을 종료하고 두 리스너 포트가 닫혔는지 확인한다. +- 복원 드라이버에서 원본 GCS 접근을 거부하는지 확인한다. **버킷 IAM 차단은 시험하지 않았다.** 원본 클러스터는 남아 있으며, 역할 확인·원복·시험 DB 삭제에 필요한 제어 접근은 유지한다. +- 외부 백업·identity·WAL·원본 객체의 generation·SHA-256·크기를 동결한다. 완성되지 않은 `.partial` WAL은 복원 입력에서 제외한다. 이 시점을 판정기의 `cutoff`로 따로 기록한다. +- 입력을 고정한 뒤 source DB·source GCS에서 복원 자료를 추가 취득하지 않는다. 보존본 검증이나 복원 실패를 이유로 이 경계를 이동하지 않는다. + +## 최초 감사 전 금지 사항 + +| 단계 | 실제 연산자가 수행하는 일 | 금지한 일 | +|---|---|---| +| restore | 동결 generation을 새 디렉터리로 다운로드, SHA·크기 대조, 빈 카탈로그에 identity/import, `verify-backup`, 새 PGDATA 생성 | 원본 카탈로그 마운트·원본 DB 연결·ADC를 복원 컨테이너에 전달 | +| target start | 현재 비동기 Standby 역할과 프로세스 시작 정보를 확인, 1,800초 자동 원복 장치 설치, 원래 프로세스 일시 정지, 기존 DB 종료, 같은 라이선스 호스트에서 복원 OpenSQL 기동 | 리더·동기 Standby 정지, 원래 PGDATA 덮어쓰기, 원래 DB와 복원 DB 동시 기동 | +| snapshot | 복원 전용 소켓·포트와 `data_directory`를 확인한 뒤 읽기 전용 SQL, 보존 객체 generation·실제 바이트 대조 | Flyway·Worker·버킷 재지정·수선·재업로드 | +| initial audit | 최초 스냅샷과 감사 결과를 단독 생성·fsync | 후속 정상화 결과로 최초 손실을 덮어쓰기 | + +복원 도구 컨테이너는 네트워크를 차단하고, 동결 입력을 새로 받은 디렉터리만 마운트한다. 실제 OpenSQL 서버는 설치된 원래 라이선스 경로를 환경으로 전달받는다. 라이선스 파일을 수정·재발급·다른 호스트에 복사하지 않는다. + +## 요청별 대조와 활성화 + +읽기 전용 SQL은 `initial_upload_requests`를 **사용자 ID + 요청 키**로 찾고, 응답에 기록한 Document·Version·FileObject·EmbeddingJob ID와 관계를 대조한다. 숫자 ID가 우연히 존재하는지만 검사하지 않는다. + +- DB 기록 없음: `DB_MISSING`. +- 응답 ID나 연결 관계 불일치: `DB_PARTIAL`. +- DB가 참조하는 보존 원본 없음: `OBJECT_MISSING`. +- 동결 generation·SHA-256·크기 불일치: `HASH_MISMATCH`. +- DB 관계와 원본 바이트 일치: `RESTORED`. 아직 인덱싱 완료라는 뜻은 아니다. + +기능 코호트 5건이 모두 `RESTORED`이고 다른 구조·객체·무결성 오류가 없을 때만 활성화한다. 손실 코호트의 `DB_MISSING`은 실제 손실로 남긴다. 그 뒤 **복원 DB만** 보존 버킷으로 재지정하고, Flyway·Worker를 켠다. 복원된 사용자 레코드로 새 로그인해 인증된 HTTP를 사용한다. + +## 기능·정합성·정리 판정 + +| 기능 | 실제 판정 | +|---|---| +| 기존 다운로드 | 완료 문서 4건의 HTTP 200·크기·SHA-256 일치 | +| 기존 검색 | 사전 고정 Chunk 질의, HTTP 200, 기대 문서가 Top 20 안에 포함 | +| 중단 Job 재개 | 같은 Job `INDEXED`, 해당 버전 Job 1개, Attempt 2개 | +| 새 작업 | HTTP 업로드 → `INDEXED` → 다운로드 SHA-256 일치 | +| 최종 감사 | 관측 복원 문서의 미완료 Job·중복 Chunk·중복 Embedding·누락 Embedding 각각 0건 | +| 정리 | 복원 프로세스 종료·기존 Patroni 재개, 리더 1대·동기 Standby 1대·비동기 Standby 1대 확인, 소유 시험 자원 삭제, 원래 설정 해시 대조 | + +각 기능의 **실제로 확인한 시각**을 T0와 cutoff 양쪽 기준으로 계산한다. 기능 검사는 순차 실행되므로 그보다 앞서 준비됐을 가능성까지 측정한 것은 아니다. DB를 켠 시각을 전체 기능 복구 시간으로 대체하지 않는다. + +GCS의 활성 시험 객체 삭제와 soft delete 보존은 구분한다. 정리 뒤에도 제공자 보존 정책에 따른 삭제 세대가 남을 수 있으며, 물리 소거라고 보고하지 않는다. 기존 개인 환경 파일·기존 모델 컨테이너·공개 원본 버킷은 삭제 대상이 아니다. diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R1/result.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R1/result.json new file mode 100644 index 00000000..8f96a027 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R1/result.json @@ -0,0 +1,19 @@ +{ + "cleanup": { + "owned_resources_removed": true, + "public_config_preserved": true, + "source_cluster_restored": true + }, + "diagnostic_verdict": "INCOMPLETE", + "error_type": "RuntimeError", + "finished_at_utc": "2026-10-18T12:12:04.265169+00:00", + "functional_or_rpo_measurement": false, + "inputs": "previous offsite bytes restored from soft-deleted objects, exact SHA checked, new generations recorded", + "location": "same licensed asynchronous replica, new empty PGDATA, offline restore container", + "purpose": "첫 리허설의 실제 OpenSQL 기동 실패 원인 진단, 오류를 사전 마스킹해 보존", + "run_id": "DR-PG-START-DIAG-20261018-R1", + "source_data_requeried": false, + "stage": "LICENSED_TARGET_START_FAILED", + "started_at_utc": "2026-10-18T12:09:52.656314+00:00", + "success_criteria": "기동 성공 또는 구체적인 비밀 제거 오류 확보, 최초 노드 3대 원복" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R1/source-capture.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R1/source-capture.json new file mode 100644 index 00000000..f26323f9 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R1/source-capture.json @@ -0,0 +1,13 @@ +{ + "all_source_mtimes_precede_start": true, + "captured_after_start": true, + "captured_at_utc": "2026-10-18T12:12:52.114819+00:00", + "sha256": { + "dr_cleanup_private.py": "6871e436112ce8a944c303e09ad2f587b75a609e37047e726a1ec9e5d73958fa", + "dr_gcs_private.py": "d718eb5c761293cf6cc738014139e023f85f8177d0391140fc12a55675c69dd0", + "dr_operator_private.py": "2691974bc7d390f38448fa5bcb01dc49b0464f084d76be72d4fc733a9ce83700", + "dr_runtime_private.py": "6728715e0d982171526f6873ff873e1a1525cd18dbd12a4f7495c5a78c90d2f0", + "dr_target_private.py": "f606e7d7bf2001db6a27b15fe34c95c3e869500dbe380b6c92f28486a04d1f01", + "native_pg_filter.py": "04f7c3d3fd6c518dbfa00902a3f70078d7d1dd66b4b4adf095ed93facfad2a66" + } +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R1/spec.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R1/spec.json new file mode 100644 index 00000000..9058c94a --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R1/spec.json @@ -0,0 +1,10 @@ +{ + "functional_or_rpo_measurement": false, + "inputs": "previous offsite bytes restored from soft-deleted objects, exact SHA checked, new generations recorded", + "location": "same licensed asynchronous replica, new empty PGDATA, offline restore container", + "purpose": "첫 리허설의 실제 OpenSQL 기동 실패 원인 진단, 오류를 사전 마스킹해 보존", + "run_id": "DR-PG-START-DIAG-20261018-R1", + "source_data_requeried": false, + "started_at_utc": "2026-10-18T12:09:52.656314+00:00", + "success_criteria": "기동 성공 또는 구체적인 비밀 제거 오류 확보, 최초 노드 3대 원복" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R2/result.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R2/result.json new file mode 100644 index 00000000..0d646180 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R2/result.json @@ -0,0 +1,22 @@ +{ + "cleanup": { + "owned_resources_removed": true, + "public_config_preserved": true, + "source_cluster_restored": true + }, + "diagnostic_verdict": "PASS", + "finished_at_utc": "2026-10-18T12:23:03.669562+00:00", + "formal_preflight": "NO_GO", + "formal_preflight_reason": "no live receiver/original cohort preparation in this diagnostic", + "functional_or_rpo_measurement": false, + "inputs": "previous offsite bytes restored from soft-deleted objects, exact SHA checked, new generations recorded", + "location": "same licensed asynchronous replica, new empty PGDATA, offline restore container", + "native_raw_logs_written": false, + "only_runtime_change": "existing OPENSQL_LICENSE_PATH added to standalone process environment", + "postmaster_started": true, + "purpose": "첫 리허설의 실제 OpenSQL 기동 실패 원인 진단, 오류를 사전 마스킹해 보존", + "run_id": "DR-PG-START-DIAG-20261018-R2", + "source_data_requeried": false, + "started_at_utc": "2026-10-18T12:20:30.569381+00:00", + "success_criteria": "기동 성공 또는 구체적인 비밀 제거 오류 확보, 최초 노드 3대 원복" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R2/spec.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R2/spec.json new file mode 100644 index 00000000..2621d2b7 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-PG-START-DIAG-20261018-R2/spec.json @@ -0,0 +1,11 @@ +{ + "functional_or_rpo_measurement": false, + "inputs": "previous offsite bytes restored from soft-deleted objects, exact SHA checked, new generations recorded", + "location": "same licensed asynchronous replica, new empty PGDATA, offline restore container", + "only_runtime_change": "existing OPENSQL_LICENSE_PATH added to standalone process environment", + "purpose": "첫 리허설의 실제 OpenSQL 기동 실패 원인 진단, 오류를 사전 마스킹해 보존", + "run_id": "DR-PG-START-DIAG-20261018-R2", + "source_data_requeried": false, + "started_at_utc": "2026-10-18T12:20:30.569381+00:00", + "success_criteria": "기동 성공 또는 구체적인 비밀 제거 오류 확보, 최초 노드 3대 원복" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/BACKUP-result.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/BACKUP-result.json new file mode 100644 index 00000000..e411aaaa --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/BACKUP-result.json @@ -0,0 +1,16 @@ +{ + "elapsed_seconds": 87.7, + "finished_at_utc": "2026-10-18T12:43:25.674130+00:00", + "location": "isolated local runtime and owned cloud source database", + "observations": { + "backup_command_exit_code": 0, + "backup_id": "20261018T124224", + "final_status": "DONE", + "verify_exit_code": 0 + }, + "purpose": "새 OpenSQL 물리 기준 백업 검증", + "run_id": "DR-RESTORE-20261018-FORMAL1-BACKUP", + "started_at_utc": "2026-10-18T12:41:57.972085+00:00", + "success_criteria": "DONE·pg_verifybackup 0", + "verdict": "PASS" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/INTERRUPT-result.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/INTERRUPT-result.json new file mode 100644 index 00000000..0bb0b5b1 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/INTERRUPT-result.json @@ -0,0 +1,18 @@ +{ + "elapsed_seconds": 65.232, + "finished_at_utc": "2026-10-18T12:41:57.971341+00:00", + "location": "isolated local runtime and owned cloud source database", + "observations": { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXING", + "embedding_count": 0, + "job_status": "PROCESSING", + "version_status": "EMBEDDING" + }, + "purpose": "실제 BGE 응답 대기 중 Job 중단", + "run_id": "DR-RESTORE-20261018-FORMAL1-INTERRUPT", + "started_at_utc": "2026-10-18T12:40:52.738032+00:00", + "success_criteria": "Job PROCESSING·Attempt 1·Worker=false 재기동", + "verdict": "PASS" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/RECEIVER-result.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/RECEIVER-result.json new file mode 100644 index 00000000..91ba8d84 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/RECEIVER-result.json @@ -0,0 +1,113 @@ +{ + "elapsed_seconds": 75.116, + "finished_at_utc": "2026-10-18T12:40:52.734048+00:00", + "location": "isolated local runtime and owned cloud source database", + "observations": { + "checks": [ + { + "check": "PostgreSQL", + "status": "OK" + }, + { + "check": "superuser or standard user with backup privileges", + "status": "OK" + }, + { + "check": "PostgreSQL streaming", + "status": "OK" + }, + { + "check": "wal_level", + "status": "OK" + }, + { + "check": "directories", + "status": "OK" + }, + { + "check": "retention policy settings", + "status": "OK" + }, + { + "check": "backup maximum age", + "status": "OK" + }, + { + "check": "backup minimum size", + "status": "OK" + }, + { + "check": "wal maximum age", + "status": "OK" + }, + { + "check": "wal size", + "status": "OK" + }, + { + "check": "compression settings", + "status": "OK" + }, + { + "check": "failed backups", + "status": "OK" + }, + { + "check": "minimum redundancy requirements", + "status": "OK" + }, + { + "check": "pg_basebackup", + "status": "OK" + }, + { + "check": "pg_basebackup compatible", + "status": "OK" + }, + { + "check": "pg_basebackup supports tablespaces mapping", + "status": "OK" + }, + { + "check": "systemid coherence", + "status": "OK" + }, + { + "check": "pg_receivexlog", + "status": "OK" + }, + { + "check": "pg_receivexlog compatible", + "status": "OK" + }, + { + "check": "receive-wal running", + "status": "OK" + }, + { + "check": "archive_mode", + "status": "OK" + }, + { + "check": "archive_command", + "status": "OK" + }, + { + "check": "continuous archiving", + "status": "OK" + }, + { + "check": "archiver errors", + "status": "OK" + } + ], + "failed": 0, + "passed": 24, + "permanent_slot_created": false + }, + "purpose": "실제 WAL 수신·Barman 준비 점검", + "run_id": "DR-RESTORE-20261018-FORMAL1-RECEIVER", + "started_at_utc": "2026-10-18T12:39:37.618562+00:00", + "success_criteria": "수신기 1·완성 WAL·준비 항목 통과", + "verdict": "PASS" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/events.jsonl b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/events.jsonl new file mode 100644 index 00000000..91855532 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/events.jsonl @@ -0,0 +1,334 @@ +{"at_utc": "2026-10-18T12:31:37.287+00:00", "data": {"scope": "isolated_source_database", "source_revision": "3bff887"}, "kind": "preparation_started", "previous": "0000000000000000000000000000000000000000000000000000000000000000", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 1, "sha256": "41799609dadb47c8d06cd5fff60d6b6c94e2bdcf80a2084448c26aa7a16820c6"} +{"at_utc": "2026-10-18T12:32:07.855+00:00", "data": {"public_database_modified": false, "roles": 2, "vector_version": "0.8.1"}, "kind": "source_database_prepared", "previous": "41799609dadb47c8d06cd5fff60d6b6c94e2bdcf80a2084448c26aa7a16820c6", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 2, "sha256": "8c31dc1727f73a0ef20f2f630b1c1d0fb85ce2b88c55e82b74494c9697256568"} +{"at_utc": "2026-10-18T12:32:07.855+00:00", "data": {"elapsed_seconds": 30.567, "stage": "DATABASE"}, "kind": "preparation_stage_passed", "previous": "8c31dc1727f73a0ef20f2f630b1c1d0fb85ce2b88c55e82b74494c9697256568", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 3, "sha256": "7b78c18e737e46c4fa675b4c748f55a7eb4f88ac4631581de5e46f4e6a5caedd"} +{"at_utc": "2026-10-18T12:32:14.897+00:00", "data": {"private_endpoint_omitted": true, "purpose": "source"}, "kind": "tunnel_ready", "previous": "7b78c18e737e46c4fa675b4c748f55a7eb4f88ac4631581de5e46f4e6a5caedd", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 4, "sha256": "d4163193c233010c3b9fa13f802fcbde2f07bf5459773957a9f6d574c50fdd0b"} +{"at_utc": "2026-10-18T12:32:16.752+00:00", "data": {"database_matched": true, "standby": false}, "kind": "owned_tcp_ready", "previous": "d4163193c233010c3b9fa13f802fcbde2f07bf5459773957a9f6d574c50fdd0b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 5, "sha256": "b02c2323970e33da380e52d2d1aaad5f2fc662b862fd387d8958e5b829e27662"} +{"at_utc": "2026-10-18T12:32:16.753+00:00", "data": {"elapsed_seconds": 8.897, "stage": "TOOLING"}, "kind": "preparation_stage_passed", "previous": "b02c2323970e33da380e52d2d1aaad5f2fc662b862fd387d8958e5b829e27662", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 6, "sha256": "6ddf61e4160f30772ec5f0828a9ade44ae6d26c3847cdd5f1aa4d353e9ce7ff5"} +{"at_utc": "2026-10-18T12:32:17.009+00:00", "data": {"response": "PONG"}, "kind": "owned_cache_ready", "previous": "6ddf61e4160f30772ec5f0828a9ade44ae6d26c3847cdd5f1aa4d353e9ce7ff5", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 7, "sha256": "c2280b17f3c0478800d8b7047b533fae6e92ca694feaeb4b25b5924ca457ca7e"} +{"at_utc": "2026-10-18T12:36:54.374+00:00", "data": {"listener_ownership_verified": true, "purpose": "source", "worker_enabled": true}, "kind": "backend_ready", "previous": "c2280b17f3c0478800d8b7047b533fae6e92ca694feaeb4b25b5924ca457ca7e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 8, "sha256": "3220a078f7d0abb4c0ec32ef1788599f568702d82536f1fca8ca75b48ff551f7"} +{"at_utc": "2026-10-18T12:36:54.375+00:00", "data": {"elapsed_seconds": 277.613, "stage": "SERVICES"}, "kind": "preparation_stage_passed", "previous": "3220a078f7d0abb4c0ec32ef1788599f568702d82536f1fca8ca75b48ff551f7", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 9, "sha256": "73ed91b368a9d8a0cd34e270215f1db82d44c5567810d5c8997e32bc959e7cfb"} +{"at_utc": "2026-10-18T12:36:57.704+00:00", "data": {"login_http": 200, "signup_http": 201}, "kind": "test_account_ready", "previous": "73ed91b368a9d8a0cd34e270215f1db82d44c5567810d5c8997e32bc959e7cfb", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 10, "sha256": "71d3073a6681dc981adefcefd53007ea1a1ae16c05696ecfeed79fde275bd745"} +{"at_utc": "2026-10-18T12:36:57.726+00:00", "data": {"cohort": "functional", "request_id": "upload-965c69a074e94cacb37ed7cd2f1cfb2a", "sha256": "1a8c22f7f7acce67d4544607b38e46ab4754d71cd4413e85533acfa1c30bb027", "size": 1044}, "kind": "sent", "previous": "71d3073a6681dc981adefcefd53007ea1a1ae16c05696ecfeed79fde275bd745", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 11, "sha256": "9f0677c69747650bc0b53cbdb39ff6aa29ae687556a0455494286cedd13f7293"} +{"at_utc": "2026-10-18T12:37:05.356+00:00", "data": {"http_status": 201, "ids": {"documentId": 3, "documentVersionId": 3, "embeddingJobId": 1, "fileObjectId": 1}, "outcome": "ACKNOWLEDGED", "request_id": "upload-965c69a074e94cacb37ed7cd2f1cfb2a"}, "kind": "response", "previous": "9f0677c69747650bc0b53cbdb39ff6aa29ae687556a0455494286cedd13f7293", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 12, "sha256": "68f3293e1ead67b34b35858512f8037cac358a8bf6e2e9cf22e13a7ecec82d85"} +{"at_utc": "2026-10-18T12:37:05.358+00:00", "data": {"cohort": "functional", "elapsed_seconds": 7.63, "http_status": 201, "ordinal": 1}, "kind": "upload_observed", "previous": "68f3293e1ead67b34b35858512f8037cac358a8bf6e2e9cf22e13a7ecec82d85", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 13, "sha256": "3057bdbc72f4d6f371add1e811124f77d24ba86d50c5a72c39c43b42f7bcf50d"} +{"at_utc": "2026-10-18T12:37:06.444+00:00", "data": {"ordinal": 1, "phase": "source", "state": "PENDING"}, "kind": "index_state", "previous": "3057bdbc72f4d6f371add1e811124f77d24ba86d50c5a72c39c43b42f7bcf50d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 14, "sha256": "c8b4b06f6c094585141fd4ab3074c0f3059fb9c303ce91d9251f40e934992f6a"} +{"at_utc": "2026-10-18T12:37:12.979+00:00", "data": {"ordinal": 1, "phase": "source", "state": "PROCESSING"}, "kind": "index_state", "previous": "c8b4b06f6c094585141fd4ab3074c0f3059fb9c303ce91d9251f40e934992f6a", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 15, "sha256": "3ce0d914485e917313500453e2c78991eefc27081f8afcbe0b4917179392d3a0"} +{"at_utc": "2026-10-18T12:37:43.837+00:00", "data": {"ordinal": 1, "phase": "source", "state": "INDEXED"}, "kind": "index_state", "previous": "3ce0d914485e917313500453e2c78991eefc27081f8afcbe0b4917179392d3a0", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 16, "sha256": "fe3dbef2eb7fd3609f27454c0e355010749277a632d32e006479942aca316347"} +{"at_utc": "2026-10-18T12:37:43.839+00:00", "data": {"cohort": "functional", "request_id": "upload-3a043da834f14436b70a14733f2252b8", "sha256": "1595721465e0cfb8d633de6b2a3674e7be455007aaf278d977f983be7ce5f609", "size": 638}, "kind": "sent", "previous": "fe3dbef2eb7fd3609f27454c0e355010749277a632d32e006479942aca316347", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 17, "sha256": "aa94740eb5e8d043e53303f2ebc1c095d6678b17dc92c74041bc24db4c3e2afc"} +{"at_utc": "2026-10-18T12:37:50.492+00:00", "data": {"http_status": 201, "ids": {"documentId": 4, "documentVersionId": 4, "embeddingJobId": 2, "fileObjectId": 2}, "outcome": "ACKNOWLEDGED", "request_id": "upload-3a043da834f14436b70a14733f2252b8"}, "kind": "response", "previous": "aa94740eb5e8d043e53303f2ebc1c095d6678b17dc92c74041bc24db4c3e2afc", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 18, "sha256": "8b14d4b5bff696bfeb3e8b14f07fcaa50a0a3e701d5ee88592e0987c8f1bf78d"} +{"at_utc": "2026-10-18T12:37:50.493+00:00", "data": {"cohort": "functional", "elapsed_seconds": 6.653, "http_status": 201, "ordinal": 2}, "kind": "upload_observed", "previous": "8b14d4b5bff696bfeb3e8b14f07fcaa50a0a3e701d5ee88592e0987c8f1bf78d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 19, "sha256": "f7ab74ea18ab0332ed12d06202ee43401f90607f67d51ca592c62b623a695a4d"} +{"at_utc": "2026-10-18T12:37:51.847+00:00", "data": {"ordinal": 2, "phase": "source", "state": "PENDING"}, "kind": "index_state", "previous": "f7ab74ea18ab0332ed12d06202ee43401f90607f67d51ca592c62b623a695a4d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 20, "sha256": "6f4fb0a1e0cfe3a49ef8bde1276f8aeee74e89b05602b5930dea4f41c61ba129"} +{"at_utc": "2026-10-18T12:38:02.190+00:00", "data": {"ordinal": 2, "phase": "source", "state": "PROCESSING"}, "kind": "index_state", "previous": "6f4fb0a1e0cfe3a49ef8bde1276f8aeee74e89b05602b5930dea4f41c61ba129", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 21, "sha256": "774ec9781beb075530f4d9088d433b98976a6b2e1049c80eaaa2a3766afa5c27"} +{"at_utc": "2026-10-18T12:38:22.640+00:00", "data": {"ordinal": 2, "phase": "source", "state": "INDEXED"}, "kind": "index_state", "previous": "774ec9781beb075530f4d9088d433b98976a6b2e1049c80eaaa2a3766afa5c27", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 22, "sha256": "df3ef47da297cb7f36c176da9bc2d828e9cceea73f945297ac71c8e981babec5"} +{"at_utc": "2026-10-18T12:38:22.644+00:00", "data": {"cohort": "functional", "request_id": "upload-aa478ffa2f2a43acb43c84ce2d8ac431", "sha256": "6213e3cb37b48c59023d9bf356f7126f9f013358cb665acd23a3dc8613a50140", "size": 1045}, "kind": "sent", "previous": "df3ef47da297cb7f36c176da9bc2d828e9cceea73f945297ac71c8e981babec5", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 23, "sha256": "b7f2e5c30317631f5147cd5d53bebf9553e9f0c03a6a68116054a11641b2bccb"} +{"at_utc": "2026-10-18T12:38:29.983+00:00", "data": {"http_status": 201, "ids": {"documentId": 5, "documentVersionId": 5, "embeddingJobId": 3, "fileObjectId": 3}, "outcome": "ACKNOWLEDGED", "request_id": "upload-aa478ffa2f2a43acb43c84ce2d8ac431"}, "kind": "response", "previous": "b7f2e5c30317631f5147cd5d53bebf9553e9f0c03a6a68116054a11641b2bccb", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 24, "sha256": "cfe2b82f9865c9039f96c93c63dbf39a64795a66c51e3b9faca1fc02e9c0db46"} +{"at_utc": "2026-10-18T12:38:29.985+00:00", "data": {"cohort": "functional", "elapsed_seconds": 7.34, "http_status": 201, "ordinal": 3}, "kind": "upload_observed", "previous": "cfe2b82f9865c9039f96c93c63dbf39a64795a66c51e3b9faca1fc02e9c0db46", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 25, "sha256": "867bbce08c0d2d8a49dfb581085b74b14a274fdc6e37aa9673f57212ab8057d8"} +{"at_utc": "2026-10-18T12:38:31.039+00:00", "data": {"ordinal": 3, "phase": "source", "state": "PENDING"}, "kind": "index_state", "previous": "867bbce08c0d2d8a49dfb581085b74b14a274fdc6e37aa9673f57212ab8057d8", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 26, "sha256": "c7763fa3cd0be287e4fbaec4586d276d9446ceb6aa14390a4863fc1b9aa744b5"} +{"at_utc": "2026-10-18T12:38:41.509+00:00", "data": {"ordinal": 3, "phase": "source", "state": "PROCESSING"}, "kind": "index_state", "previous": "c7763fa3cd0be287e4fbaec4586d276d9446ceb6aa14390a4863fc1b9aa744b5", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 27, "sha256": "b015d2f6166643eaf894b6ee1e9646a97531e824638bfc85c814692246f78944"} +{"at_utc": "2026-10-18T12:39:05.363+00:00", "data": {"ordinal": 3, "phase": "source", "state": "INDEXED"}, "kind": "index_state", "previous": "b015d2f6166643eaf894b6ee1e9646a97531e824638bfc85c814692246f78944", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 28, "sha256": "621bd3d828eb3b45dceeca33bee6c710819cc12d55961158e90f3987ed2b56e8"} +{"at_utc": "2026-10-18T12:39:05.365+00:00", "data": {"cohort": "functional", "request_id": "upload-9ca175c82653458b92bed4bcd9537a5f", "sha256": "671f658102aa034a78d956f412eb24e80919a2d2a25df92ae5c58c393317500f", "size": 638}, "kind": "sent", "previous": "621bd3d828eb3b45dceeca33bee6c710819cc12d55961158e90f3987ed2b56e8", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 29, "sha256": "b923bb05d93eb2b9dc43090e8191435c5366b5af0134642cdd662e10f16c7212"} +{"at_utc": "2026-10-18T12:39:12.453+00:00", "data": {"http_status": 201, "ids": {"documentId": 6, "documentVersionId": 6, "embeddingJobId": 4, "fileObjectId": 4}, "outcome": "ACKNOWLEDGED", "request_id": "upload-9ca175c82653458b92bed4bcd9537a5f"}, "kind": "response", "previous": "b923bb05d93eb2b9dc43090e8191435c5366b5af0134642cdd662e10f16c7212", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 30, "sha256": "800f256df2a21a36c9bd0428e39e4547fd6ee9fca6de24616b3baa62d6ab689c"} +{"at_utc": "2026-10-18T12:39:12.454+00:00", "data": {"cohort": "functional", "elapsed_seconds": 7.093, "http_status": 201, "ordinal": 4}, "kind": "upload_observed", "previous": "800f256df2a21a36c9bd0428e39e4547fd6ee9fca6de24616b3baa62d6ab689c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 31, "sha256": "ab0ea220e286577510c22d1b7ea3e5c08c304f0237756049ee46b40de614d5c2"} +{"at_utc": "2026-10-18T12:39:13.517+00:00", "data": {"ordinal": 4, "phase": "source", "state": "PENDING"}, "kind": "index_state", "previous": "ab0ea220e286577510c22d1b7ea3e5c08c304f0237756049ee46b40de614d5c2", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 32, "sha256": "0e057235f950208451bc0676d265a3bb37cac93a95f63f95a345e8a6968749e9"} +{"at_utc": "2026-10-18T12:39:16.885+00:00", "data": {"ordinal": 4, "phase": "source", "state": "PROCESSING"}, "kind": "index_state", "previous": "0e057235f950208451bc0676d265a3bb37cac93a95f63f95a345e8a6968749e9", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 33, "sha256": "963b7551f4f1fbca4dcb6db317d7025f24721e3295ac85194bf9a984853f52ce"} +{"at_utc": "2026-10-18T12:39:37.614+00:00", "data": {"ordinal": 4, "phase": "source", "state": "INDEXED"}, "kind": "index_state", "previous": "963b7551f4f1fbca4dcb6db317d7025f24721e3295ac85194bf9a984853f52ce", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 34, "sha256": "d34362f826f837ab47d550e144a0f9615d8d1f0426dcb74e726e36ccffe67bca"} +{"at_utc": "2026-10-18T12:39:37.618+00:00", "data": {"elapsed_seconds": 163.252, "stage": "CORPUS"}, "kind": "preparation_stage_passed", "previous": "d34362f826f837ab47d550e144a0f9615d8d1f0426dcb74e726e36ccffe67bca", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 35, "sha256": "de8506c8f946ae92401aea48f72677699a2b7b1f085aa6a4c8e79d3a7a2aee6c"} +{"at_utc": "2026-10-18T12:39:48.685+00:00", "data": {"count": 0, "probe": 1}, "kind": "receiver_readiness_probe", "previous": "de8506c8f946ae92401aea48f72677699a2b7b1f085aa6a4c8e79d3a7a2aee6c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 36, "sha256": "b8569e3ca75e97c29e9f0b90da811c0b66dba20f87fd7bb5bbf3b8ba9f455f7e"} +{"at_utc": "2026-10-18T12:40:01.314+00:00", "data": {"count": 0, "probe": 2}, "kind": "receiver_readiness_probe", "previous": "b8569e3ca75e97c29e9f0b90da811c0b66dba20f87fd7bb5bbf3b8ba9f455f7e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 37, "sha256": "096ded95ebf0a7d859f2f31531fca3cfd0bfbd19063a82b5f6e3b6bd0f869479"} +{"at_utc": "2026-10-18T12:40:13.320+00:00", "data": {"count": 1, "probe": 3}, "kind": "receiver_readiness_probe", "previous": "096ded95ebf0a7d859f2f31531fca3cfd0bfbd19063a82b5f6e3b6bd0f869479", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 38, "sha256": "a30d68c6700294cebcdd4408e5e79a8ad9f2bb73c12c9888b4716e0d649cbfa1"} +{"at_utc": "2026-10-18T12:40:52.737+00:00", "data": {"elapsed_seconds": 75.116, "stage": "RECEIVER"}, "kind": "preparation_stage_passed", "previous": "a30d68c6700294cebcdd4408e5e79a8ad9f2bb73c12c9888b4716e0d649cbfa1", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 39, "sha256": "4edef190573c2b96ca477fb4635f9c855fb4f6f7ddc4fb758b0050486e4ecea9"} +{"at_utc": "2026-10-18T12:40:52.743+00:00", "data": {"cohort": "functional", "request_id": "upload-9e7df4353fed4e7aaa5a3c7f1408e4e5", "sha256": "c5f4b1afadfdbc24aefed3d4ef5786b495b09f83e957583be16d49fd876f7921", "size": 1045}, "kind": "sent", "previous": "4edef190573c2b96ca477fb4635f9c855fb4f6f7ddc4fb758b0050486e4ecea9", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 40, "sha256": "1a77916c7d99e8dfb016aa6b9bff540769a7c264953cdf0d6755103094056f6a"} +{"at_utc": "2026-10-18T12:41:01.038+00:00", "data": {"http_status": 201, "ids": {"documentId": 7, "documentVersionId": 7, "embeddingJobId": 5, "fileObjectId": 5}, "outcome": "ACKNOWLEDGED", "request_id": "upload-9e7df4353fed4e7aaa5a3c7f1408e4e5"}, "kind": "response", "previous": "1a77916c7d99e8dfb016aa6b9bff540769a7c264953cdf0d6755103094056f6a", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 41, "sha256": "ef6557be9442ca4d1fcbd04c42bec405e99c3bc8c825baeb416083dc37b7e2a5"} +{"at_utc": "2026-10-18T12:41:01.040+00:00", "data": {"cohort": "functional", "elapsed_seconds": 8.295, "http_status": 201, "ordinal": 5}, "kind": "upload_observed", "previous": "ef6557be9442ca4d1fcbd04c42bec405e99c3bc8c825baeb416083dc37b7e2a5", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 42, "sha256": "301d8d66adeaefa27b482f00cc9181fc5e126e1837db4f351f1efe4a7f43375c"} +{"at_utc": "2026-10-18T12:41:36.548+00:00", "data": {"attempt_count": 1, "chunk_count": 3, "document_status": "INDEXING", "embedding_count": 0, "job_status": "PROCESSING", "ordinal": 5, "source_jvm_signal": "SIGKILL", "version_status": "EMBEDDING"}, "kind": "actual_job_interrupted", "previous": "301d8d66adeaefa27b482f00cc9181fc5e126e1837db4f351f1efe4a7f43375c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 43, "sha256": "a84f8f52500f0d5f050ee57c5cdc8cfa0bb64c9626fca7eee2500601a7dd432f"} +{"at_utc": "2026-10-18T12:41:57.971+00:00", "data": {"listener_ownership_verified": true, "purpose": "source", "worker_enabled": false}, "kind": "backend_ready", "previous": "a84f8f52500f0d5f050ee57c5cdc8cfa0bb64c9626fca7eee2500601a7dd432f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 44, "sha256": "28b523aaf2b6b2ff8cb3316e62fb8c1d185c9194a71349c6490b452636bda3ca"} +{"at_utc": "2026-10-18T12:41:57.972+00:00", "data": {"elapsed_seconds": 65.232, "stage": "INTERRUPT"}, "kind": "preparation_stage_passed", "previous": "28b523aaf2b6b2ff8cb3316e62fb8c1d185c9194a71349c6490b452636bda3ca", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 45, "sha256": "5220303cf15cf44cfc4578e21620b8d2d5127d3a0fa4674e6ccebe4bab315b6f"} +{"at_utc": "2026-10-18T12:43:24.886+00:00", "data": {"archive_exit_code": 0, "done": true, "probe": 1}, "kind": "backup_finalization_probe", "previous": "5220303cf15cf44cfc4578e21620b8d2d5127d3a0fa4674e6ccebe4bab315b6f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 46, "sha256": "a3e02e41f089fb7a7447f98334dbac2a0a3307def44404b0d656b33bd4d85c98"} +{"at_utc": "2026-10-18T12:43:25.676+00:00", "data": {"elapsed_seconds": 87.7, "stage": "BACKUP"}, "kind": "preparation_stage_passed", "previous": "a3e02e41f089fb7a7447f98334dbac2a0a3307def44404b0d656b33bd4d85c98", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 47, "sha256": "e3f252e5b9c877b3ccf43f1fb7d3ebcb3158264426dd1716a200d2dd79086b16"} +{"at_utc": "2026-10-18T12:43:45.970+00:00", "data": {"captured_before_cutoff": true, "license_environment_present": true, "sha256": "5ecf3c1e11719b00332a1c2a7d5af8759fb4d90b5c193ea4124d1e5460b47b6d"}, "kind": "bootstrap_frozen", "previous": "e3f252e5b9c877b3ccf43f1fb7d3ebcb3158264426dd1716a200d2dd79086b16", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 48, "sha256": "051a863807b14c9f70f4329502323c28ff4a639cd75971217ee21b75db6c5418"} +{"at_utc": "2026-10-18T12:43:56.038+00:00", "data": {"elapsed_seconds": 30.36, "stage": "BOOTSTRAP"}, "kind": "preparation_stage_passed", "previous": "051a863807b14c9f70f4329502323c28ff4a639cd75971217ee21b75db6c5418", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 49, "sha256": "84d050f1c84ea11ce92111eada0265ac99a0c7be7b5b68bb17bc456a1937fe4c"} +{"at_utc": "2026-10-18T12:44:20.565+00:00", "data": {"fileObjectId": 1, "generation": "1792327460164362", "sha256": "1a8c22f7f7acce67d4544607b38e46ab4754d71cd4413e85533acfa1c30bb027", "size": 1044}, "kind": "original_preserved", "previous": "84d050f1c84ea11ce92111eada0265ac99a0c7be7b5b68bb17bc456a1937fe4c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 50, "sha256": "da579cb5d2a2366fbc67547597f5d7a24c2655dc86c82f02a786f65761e92903"} +{"at_utc": "2026-10-18T12:44:21.845+00:00", "data": {"fileObjectId": 2, "generation": "1792327461460810", "sha256": "1595721465e0cfb8d633de6b2a3674e7be455007aaf278d977f983be7ce5f609", "size": 638}, "kind": "original_preserved", "previous": "da579cb5d2a2366fbc67547597f5d7a24c2655dc86c82f02a786f65761e92903", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 51, "sha256": "399f8e407b6f068be03e10c69ca043f4d3c3effe096a9d7431cedbe6fa19b393"} +{"at_utc": "2026-10-18T12:44:23.145+00:00", "data": {"fileObjectId": 3, "generation": "1792327462725332", "sha256": "6213e3cb37b48c59023d9bf356f7126f9f013358cb665acd23a3dc8613a50140", "size": 1045}, "kind": "original_preserved", "previous": "399f8e407b6f068be03e10c69ca043f4d3c3effe096a9d7431cedbe6fa19b393", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 52, "sha256": "039a6ecf9a4690e5b1e299c58c4938048040144a1803db3c3ddf3c140c1521c9"} +{"at_utc": "2026-10-18T12:44:24.416+00:00", "data": {"fileObjectId": 4, "generation": "1792327464013839", "sha256": "671f658102aa034a78d956f412eb24e80919a2d2a25df92ae5c58c393317500f", "size": 638}, "kind": "original_preserved", "previous": "039a6ecf9a4690e5b1e299c58c4938048040144a1803db3c3ddf3c140c1521c9", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 53, "sha256": "912fd51463dde7a155c3136aad859a38e5b4c559070b8ac42a8ab918130fcc09"} +{"at_utc": "2026-10-18T12:44:25.680+00:00", "data": {"fileObjectId": 5, "generation": "1792327465201824", "sha256": "c5f4b1afadfdbc24aefed3d4ef5786b495b09f83e957583be16d49fd876f7921", "size": 1045}, "kind": "original_preserved", "previous": "912fd51463dde7a155c3136aad859a38e5b4c559070b8ac42a8ab918130fcc09", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 54, "sha256": "9faa6aad44e1b8e8cbd8fedbe9da03d2d591d50b4c5dfe91d74c7fc8840a0915"} +{"at_utc": "2026-10-18T12:44:44.048+00:00", "data": {"generation": "1792327483514667", "name": "00000006.history", "sha256": "7dbe25c372b72e1132dca125c614a696011f1aeb3c1dfdac9cc8e19055a6d3c6", "size": 209}, "kind": "wal_preserved", "previous": "9faa6aad44e1b8e8cbd8fedbe9da03d2d591d50b4c5dfe91d74c7fc8840a0915", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 55, "sha256": "acb2564160bc0b88760d1cdb3521f12514a84596d24bd5929bc529c574faa1e7"} +{"at_utc": "2026-10-18T12:44:47.656+00:00", "data": {"generation": "1792327486114407", "name": "00000006000000000000001E", "sha256": "f6b03616fc67281e7772e125edf5e18e65fe999409844f564158f8581456b2b5", "size": 16777216}, "kind": "wal_preserved", "previous": "acb2564160bc0b88760d1cdb3521f12514a84596d24bd5929bc529c574faa1e7", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 56, "sha256": "793705216e12ba62b1c3266b089dd5b2b3a6887ba9969bdaf2cb71d75aaaae84"} +{"at_utc": "2026-10-18T12:44:51.736+00:00", "data": {"generation": "1792327490310809", "name": "00000006000000000000001F", "sha256": "99e9c17abdeaff81a84125179fafd7f43f702d7f5a3251b9f8ca9987380c135c", "size": 16777216}, "kind": "wal_preserved", "previous": "793705216e12ba62b1c3266b089dd5b2b3a6887ba9969bdaf2cb71d75aaaae84", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 57, "sha256": "3c90fbb01def49e18a3f0a3a10ee9f1007473b8188aecd02c13f25d4722f6edc"} +{"at_utc": "2026-10-18T12:44:55.928+00:00", "data": {"generation": "1792327494374579", "name": "000000060000000000000020", "sha256": "cca03fedad9e7d5e7f9e9c585c27abb96d42cb6205c1b13d2c62e5ab7e9fb3d7", "size": 16777216}, "kind": "wal_preserved", "previous": "3c90fbb01def49e18a3f0a3a10ee9f1007473b8188aecd02c13f25d4722f6edc", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 58, "sha256": "543b4957523977cd1e268d268ac08be6d178ba85ddfbe6106be44d66655df629"} +{"at_utc": "2026-10-18T12:44:55.930+00:00", "data": {"complete_wals": 4, "elapsed_seconds": 59.888, "ordinal": 0, "preserved_originals": 5}, "kind": "offsite_round_completed", "previous": "543b4957523977cd1e268d268ac08be6d178ba85ddfbe6106be44d66655df629", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 59, "sha256": "5cd4bc2e247e90e88e31b39a82b2eed2503369211ce2d1f6940db1134a030bd5"} +{"at_utc": "2026-10-18T12:44:55.932+00:00", "data": {"elapsed_seconds": 59.89, "stage": "OFFSITE"}, "kind": "preparation_stage_passed", "previous": "5cd4bc2e247e90e88e31b39a82b2eed2503369211ce2d1f6940db1134a030bd5", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 60, "sha256": "0dd99a5c877ab131147291382f2a32baf8774ee7f8d1979dd5dfb1043820f1fe"} +{"at_utc": "2026-10-18T12:49:16.823+00:00", "data": {"chunkId": 5, "documentId": 3, "query_length": 1000, "query_sha256": "881a59da25442393ff48e7380b00aab77229648df2851e8e4dd017592fd6f69f", "top_k": 20}, "kind": "search_fixture_frozen", "previous": "0dd99a5c877ab131147291382f2a32baf8774ee7f8d1979dd5dfb1043820f1fe", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 61, "sha256": "8311c52a901f931ea35b9c9db7b29f166cbc7e9d941fab7460c447f191faa536"} +{"at_utc": "2026-10-18T12:49:16.828+00:00", "data": {"product_revision": "3bff887a5f6abdf4155957a658599f8f0366a141", "sha256": {"dr_cleanup_private.py": "efa355952fea5ccdd98c75bdb0d9399a68198d6bafb25db3181d6bf68117d6f8", "dr_gcs_private.py": "d718eb5c761293cf6cc738014139e023f85f8177d0391140fc12a55675c69dd0", "dr_keys_private.py": "b6c32f3ab7e0fc5bd77be86d7ee7f87c6d2553087cc947482f34ad69731ff1e5", "dr_operator_private.py": "436489f92bc91d80c173019907aa2a204821dfc125be6d10f6421b3059dbeda4", "dr_runtime_private.py": "6728715e0d982171526f6873ff873e1a1525cd18dbd12a4f7495c5a78c90d2f0", "dr_source_private.py": "d53b1f421f09bfafb96e5d0038c4ef3cabcbf5ac38d8586b0b13b85b1626cc96", "dr_target_private.py": "f606e7d7bf2001db6a27b15fe34c95c3e869500dbe380b6c92f28486a04d1f01", "native_pg_filter.py": "04f7c3d3fd6c518dbfa00902a3f70078d7d1dd66b4b4adf095ed93facfad2a66"}}, "kind": "operator_sources_frozen", "previous": "8311c52a901f931ea35b9c9db7b29f166cbc7e9d941fab7460c447f191faa536", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 62, "sha256": "7217955858c27cbe1f8cd3271c2339ff8076c15f22bc3997dc43733abf1c5e55"} +{"at_utc": "2026-10-18T12:49:16.841+00:00", "data": {"cohort": "loss", "request_id": "upload-c9166086f88c466d8a756e6f6d9622c5", "sha256": "6403d52e2ebc1bf1adabfd78a27dad188d674a61a25d3125864a896f3a18e3b8", "size": 1044}, "kind": "sent", "previous": "7217955858c27cbe1f8cd3271c2339ff8076c15f22bc3997dc43733abf1c5e55", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 63, "sha256": "e153df271ddd5ddf6fc5981ceb21cd34ee34d7ef52087fb4c313e6696b928680"} +{"at_utc": "2026-10-18T12:49:25.127+00:00", "data": {"http_status": 201, "ids": {"documentId": 8, "documentVersionId": 8, "embeddingJobId": 6, "fileObjectId": 6}, "outcome": "ACKNOWLEDGED", "request_id": "upload-c9166086f88c466d8a756e6f6d9622c5"}, "kind": "response", "previous": "e153df271ddd5ddf6fc5981ceb21cd34ee34d7ef52087fb4c313e6696b928680", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 64, "sha256": "74d9f25a39feb9dcda3f85430c8eec8011227439fcf6c9ac3a9bb4a36f542a34"} +{"at_utc": "2026-10-18T12:49:25.129+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.286, "http_status": 201, "ordinal": 6}, "kind": "upload_observed", "previous": "74d9f25a39feb9dcda3f85430c8eec8011227439fcf6c9ac3a9bb4a36f542a34", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 65, "sha256": "fe4454fbc2aefa92d5537c319999612ce53eeddef0bd7e4fbbd5309a4b876c25"} +{"at_utc": "2026-10-18T12:49:27.831+00:00", "data": {"cohort": "loss", "request_id": "upload-66ba7d8a8de148629607f6a26bbc8f90", "sha256": "b5533ff9dc19e8ab13eceda011d203151c304adb6c15a76f9624c40e4f727229", "size": 638}, "kind": "sent", "previous": "fe4454fbc2aefa92d5537c319999612ce53eeddef0bd7e4fbbd5309a4b876c25", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 66, "sha256": "22d9b42c379e1d905e4b20f8e25b74b318afbba6f01cdbf3ab08676cf6a6f452"} +{"at_utc": "2026-10-18T12:49:34.772+00:00", "data": {"http_status": 201, "ids": {"documentId": 9, "documentVersionId": 9, "embeddingJobId": 7, "fileObjectId": 7}, "outcome": "ACKNOWLEDGED", "request_id": "upload-66ba7d8a8de148629607f6a26bbc8f90"}, "kind": "response", "previous": "22d9b42c379e1d905e4b20f8e25b74b318afbba6f01cdbf3ab08676cf6a6f452", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 67, "sha256": "4ff2eceb84548349ae2e8d963ea43769926b549e4691691bc5c02a1ffc45d805"} +{"at_utc": "2026-10-18T12:49:34.775+00:00", "data": {"cohort": "loss", "elapsed_seconds": 6.941, "http_status": 201, "ordinal": 7}, "kind": "upload_observed", "previous": "4ff2eceb84548349ae2e8d963ea43769926b549e4691691bc5c02a1ffc45d805", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 68, "sha256": "d5e56b30f6e4aa522b7a64762433dc71ffd1a14258ad565812128d5c323fafaf"} +{"at_utc": "2026-10-18T12:49:38.839+00:00", "data": {"cohort": "loss", "request_id": "upload-99d323fbaa964760a0b71ef6412e025d", "sha256": "17ca77586adbaf5282fa290b8daa01e087a06aa263602a51f4ae6557813e6a96", "size": 1044}, "kind": "sent", "previous": "d5e56b30f6e4aa522b7a64762433dc71ffd1a14258ad565812128d5c323fafaf", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 69, "sha256": "7b016067a6a9d2356d9f533a4d866f184c9e513b6bb5cd4842ffa759f3fbbd4b"} +{"at_utc": "2026-10-18T12:49:46.064+00:00", "data": {"http_status": 201, "ids": {"documentId": 10, "documentVersionId": 10, "embeddingJobId": 8, "fileObjectId": 8}, "outcome": "ACKNOWLEDGED", "request_id": "upload-99d323fbaa964760a0b71ef6412e025d"}, "kind": "response", "previous": "7b016067a6a9d2356d9f533a4d866f184c9e513b6bb5cd4842ffa759f3fbbd4b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 70, "sha256": "34f3e219ff12d4a241a7670525cbfaeb3e33d845fdf1d8c86bf0fdeec3b33290"} +{"at_utc": "2026-10-18T12:49:46.067+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.224, "http_status": 201, "ordinal": 8}, "kind": "upload_observed", "previous": "34f3e219ff12d4a241a7670525cbfaeb3e33d845fdf1d8c86bf0fdeec3b33290", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 71, "sha256": "9ddbbaded3e3ea8d1a4211699d7923fde613cec463514fdb29f16944c1181e8a"} +{"at_utc": "2026-10-18T12:49:49.839+00:00", "data": {"cohort": "loss", "request_id": "upload-b331678e5a7d42cf8a4c4db431cd19bf", "sha256": "389fdb5425c931b59623ed2ba6888f9e120609317fe8963245d88aec7304d929", "size": 638}, "kind": "sent", "previous": "9ddbbaded3e3ea8d1a4211699d7923fde613cec463514fdb29f16944c1181e8a", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 72, "sha256": "0c09cf720bc6339d2cb0e585d35640a0df70ed31b17dbce313637857a5f925e3"} +{"at_utc": "2026-10-18T12:49:57.093+00:00", "data": {"http_status": 201, "ids": {"documentId": 11, "documentVersionId": 11, "embeddingJobId": 9, "fileObjectId": 9}, "outcome": "ACKNOWLEDGED", "request_id": "upload-b331678e5a7d42cf8a4c4db431cd19bf"}, "kind": "response", "previous": "0c09cf720bc6339d2cb0e585d35640a0df70ed31b17dbce313637857a5f925e3", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 73, "sha256": "d32078fa252580a0aa695e543ba85e4991c80fcb022af8075c3a09cb0a2c42d8"} +{"at_utc": "2026-10-18T12:49:57.097+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.254, "http_status": 201, "ordinal": 9}, "kind": "upload_observed", "previous": "d32078fa252580a0aa695e543ba85e4991c80fcb022af8075c3a09cb0a2c42d8", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 74, "sha256": "e7a0a4604399f2997127b85409117477d4ee6fd841b8671cf17a653bb32e15ff"} +{"at_utc": "2026-10-18T12:50:00.837+00:00", "data": {"cohort": "loss", "request_id": "upload-0ba3a720b65e417f807def554042667e", "sha256": "ea06cf64bc7c18fd6379ee9fbc9eade765483eadbeee54799f7add448ad4288c", "size": 1045}, "kind": "sent", "previous": "e7a0a4604399f2997127b85409117477d4ee6fd841b8671cf17a653bb32e15ff", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 75, "sha256": "c4cd5400b549f3d06ea6c9a041a36aaf427eb398af87bd44bfcf921ff02ffe8c"} +{"at_utc": "2026-10-18T12:50:08.644+00:00", "data": {"http_status": 201, "ids": {"documentId": 12, "documentVersionId": 12, "embeddingJobId": 10, "fileObjectId": 10}, "outcome": "ACKNOWLEDGED", "request_id": "upload-0ba3a720b65e417f807def554042667e"}, "kind": "response", "previous": "c4cd5400b549f3d06ea6c9a041a36aaf427eb398af87bd44bfcf921ff02ffe8c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 76, "sha256": "dd5994430bbad3752d77656cac96455aa892fee1cc678e2d6b4e9623f3e4044d"} +{"at_utc": "2026-10-18T12:50:08.647+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.808, "http_status": 201, "ordinal": 10}, "kind": "upload_observed", "previous": "dd5994430bbad3752d77656cac96455aa892fee1cc678e2d6b4e9623f3e4044d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 77, "sha256": "f1e99a0039db16b5260ca63c16e00b7702fc9acf87461113c587217cbe8939de"} +{"at_utc": "2026-10-18T12:50:11.841+00:00", "data": {"cohort": "loss", "request_id": "upload-49eb7ad416b24429afb537f16ae6333f", "sha256": "2747f698cfa86d0f8f0873b9ec34139ab9a13df1a748a441538d2f764af69c4b", "size": 639}, "kind": "sent", "previous": "f1e99a0039db16b5260ca63c16e00b7702fc9acf87461113c587217cbe8939de", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 78, "sha256": "a59cbf6c33eb69c64fd9aabe01535d116fbd3c27d870ed8aeeffb2fd56c3fea9"} +{"at_utc": "2026-10-18T12:50:20.001+00:00", "data": {"http_status": 201, "ids": {"documentId": 13, "documentVersionId": 13, "embeddingJobId": 11, "fileObjectId": 11}, "outcome": "ACKNOWLEDGED", "request_id": "upload-49eb7ad416b24429afb537f16ae6333f"}, "kind": "response", "previous": "a59cbf6c33eb69c64fd9aabe01535d116fbd3c27d870ed8aeeffb2fd56c3fea9", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 79, "sha256": "d0a39638b3e9982161c42ac882fc111f9b7460754525f0fba8fc38c0e0fca95c"} +{"at_utc": "2026-10-18T12:50:20.003+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.159, "http_status": 201, "ordinal": 11}, "kind": "upload_observed", "previous": "d0a39638b3e9982161c42ac882fc111f9b7460754525f0fba8fc38c0e0fca95c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 80, "sha256": "4b1b25078c8e83432f0175673fc43122b591072808bb39f2320fa4517d538793"} +{"at_utc": "2026-10-18T12:50:22.840+00:00", "data": {"cohort": "loss", "request_id": "upload-c7f7b9de82414e0889a3e52223279673", "sha256": "f052804612d30f76364e7c3e9103be400f0b1117eb36920e0c56de1eb80cc19d", "size": 1045}, "kind": "sent", "previous": "4b1b25078c8e83432f0175673fc43122b591072808bb39f2320fa4517d538793", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 81, "sha256": "9fab0a316ea0a7339b80560a157921f694759ec16bcb7f17efd58eb1287172b9"} +{"at_utc": "2026-10-18T12:50:31.133+00:00", "data": {"http_status": 201, "ids": {"documentId": 14, "documentVersionId": 14, "embeddingJobId": 12, "fileObjectId": 12}, "outcome": "ACKNOWLEDGED", "request_id": "upload-c7f7b9de82414e0889a3e52223279673"}, "kind": "response", "previous": "9fab0a316ea0a7339b80560a157921f694759ec16bcb7f17efd58eb1287172b9", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 82, "sha256": "ecfaf9ddd730f926fd9927fe9c4ed3ad0b90992bfa3f4805578872a2adc74618"} +{"at_utc": "2026-10-18T12:50:31.136+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.292, "http_status": 201, "ordinal": 12}, "kind": "upload_observed", "previous": "ecfaf9ddd730f926fd9927fe9c4ed3ad0b90992bfa3f4805578872a2adc74618", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 83, "sha256": "e6c96dceff036df813c2a1871d145cdb0db3b20aed6e46992da507f6e0aa501d"} +{"at_utc": "2026-10-18T12:50:33.841+00:00", "data": {"cohort": "loss", "request_id": "upload-3332040234024f5fab7f4545b4bb1f9f", "sha256": "172c782cc9e625e2e54344a27890565cc8461a2aacc7c38e1b29d4197322a243", "size": 639}, "kind": "sent", "previous": "e6c96dceff036df813c2a1871d145cdb0db3b20aed6e46992da507f6e0aa501d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 84, "sha256": "305b1eced907942fd1de30c2c76498b6f3f588260337608f9c9d5775bdfc9d28"} +{"at_utc": "2026-10-18T12:50:41.632+00:00", "data": {"http_status": 201, "ids": {"documentId": 15, "documentVersionId": 15, "embeddingJobId": 13, "fileObjectId": 13}, "outcome": "ACKNOWLEDGED", "request_id": "upload-3332040234024f5fab7f4545b4bb1f9f"}, "kind": "response", "previous": "305b1eced907942fd1de30c2c76498b6f3f588260337608f9c9d5775bdfc9d28", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 85, "sha256": "82ab9baf5dc0f9e085cf01b03febacb8feb4d2ed8ad6486b60633b46d61b98b8"} +{"at_utc": "2026-10-18T12:50:41.636+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.792, "http_status": 201, "ordinal": 13}, "kind": "upload_observed", "previous": "82ab9baf5dc0f9e085cf01b03febacb8feb4d2ed8ad6486b60633b46d61b98b8", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 86, "sha256": "fcc6dd2edefbe4ff9a772f29a2d18d835b81a51dcc4f59782a29af4445b756f6"} +{"at_utc": "2026-10-18T12:50:44.847+00:00", "data": {"cohort": "loss", "request_id": "upload-0935757b0cd5475ab423f04174c1a17b", "sha256": "f29454ce42ad1c2047303fb13400aba5043e7483c49ab66c6294ac7677c5b566", "size": 1045}, "kind": "sent", "previous": "fcc6dd2edefbe4ff9a772f29a2d18d835b81a51dcc4f59782a29af4445b756f6", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 87, "sha256": "4a8fe039b6c00a36c8b13e3bd18635407ea54e62c755dc1fe54c1c801aa4d38f"} +{"at_utc": "2026-10-18T12:50:52.682+00:00", "data": {"http_status": 201, "ids": {"documentId": 16, "documentVersionId": 16, "embeddingJobId": 14, "fileObjectId": 14}, "outcome": "ACKNOWLEDGED", "request_id": "upload-0935757b0cd5475ab423f04174c1a17b"}, "kind": "response", "previous": "4a8fe039b6c00a36c8b13e3bd18635407ea54e62c755dc1fe54c1c801aa4d38f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 88, "sha256": "72fa9b31e6da414f4e561c49630187605e2fec3fc38c08566aa81f8682e1f804"} +{"at_utc": "2026-10-18T12:50:52.687+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.835, "http_status": 201, "ordinal": 14}, "kind": "upload_observed", "previous": "72fa9b31e6da414f4e561c49630187605e2fec3fc38c08566aa81f8682e1f804", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 89, "sha256": "527c225a53498a1a2ecf96d80f9f51e7c524e845806b863f94fa58ec2aeb4e6f"} +{"at_utc": "2026-10-18T12:50:55.845+00:00", "data": {"cohort": "loss", "request_id": "upload-dd0d63abe92b4a9192a897b7af26e314", "sha256": "c5f84d6ccff80425b9dd6a301379c50c6c0ba91185f968280bf6eff1d36f758a", "size": 639}, "kind": "sent", "previous": "527c225a53498a1a2ecf96d80f9f51e7c524e845806b863f94fa58ec2aeb4e6f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 90, "sha256": "a8afb4e790f5ee1f5ebd4f73d39bde0c91c42b573ffd90116c1fda19d04443b4"} +{"at_utc": "2026-10-18T12:51:03.643+00:00", "data": {"http_status": 201, "ids": {"documentId": 17, "documentVersionId": 17, "embeddingJobId": 15, "fileObjectId": 15}, "outcome": "ACKNOWLEDGED", "request_id": "upload-dd0d63abe92b4a9192a897b7af26e314"}, "kind": "response", "previous": "a8afb4e790f5ee1f5ebd4f73d39bde0c91c42b573ffd90116c1fda19d04443b4", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 91, "sha256": "cc27af655fdeb3e5122198b2e84459fa19c8a98493cea3d3cb9f1af4159c7f7e"} +{"at_utc": "2026-10-18T12:51:03.647+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.798, "http_status": 201, "ordinal": 15}, "kind": "upload_observed", "previous": "cc27af655fdeb3e5122198b2e84459fa19c8a98493cea3d3cb9f1af4159c7f7e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 92, "sha256": "ec503353e690bb9608566ab2c21451cf3c771e11f4abac116dd50efd95fefa62"} +{"at_utc": "2026-10-18T12:51:06.842+00:00", "data": {"cohort": "loss", "request_id": "upload-4ae3860380ef4bfcb24944535960303e", "sha256": "e23b60a24d1b2f413ac0b5a9d712b9f9df7a7c256b58344588fb5dc8b8a3baf2", "size": 1045}, "kind": "sent", "previous": "ec503353e690bb9608566ab2c21451cf3c771e11f4abac116dd50efd95fefa62", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 93, "sha256": "8e00cd15e6d4e72f6702174f02f17af6553af626437259266477cd13a1967a32"} +{"at_utc": "2026-10-18T12:51:16.097+00:00", "data": {"http_status": 201, "ids": {"documentId": 18, "documentVersionId": 18, "embeddingJobId": 16, "fileObjectId": 16}, "outcome": "ACKNOWLEDGED", "request_id": "upload-4ae3860380ef4bfcb24944535960303e"}, "kind": "response", "previous": "8e00cd15e6d4e72f6702174f02f17af6553af626437259266477cd13a1967a32", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 94, "sha256": "8a17fd31ca065d7efd21eb748b6ecd755d7eccecfc5c2e641841b7a6276aa08e"} +{"at_utc": "2026-10-18T12:51:16.102+00:00", "data": {"cohort": "loss", "elapsed_seconds": 9.255, "http_status": 201, "ordinal": 16}, "kind": "upload_observed", "previous": "8a17fd31ca065d7efd21eb748b6ecd755d7eccecfc5c2e641841b7a6276aa08e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 95, "sha256": "3f6f6d998f7b24c7bcb952724616adf4ce0c1fe9e3d21acacaa47623b3008ead"} +{"at_utc": "2026-10-18T12:51:17.843+00:00", "data": {"cohort": "loss", "request_id": "upload-6192da40bdbd4619821046d81292d3c7", "sha256": "7a4d5693fe950d8499727512a0c01f613c912b09ac020f4bfea558a347dc104f", "size": 639}, "kind": "sent", "previous": "3f6f6d998f7b24c7bcb952724616adf4ce0c1fe9e3d21acacaa47623b3008ead", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 96, "sha256": "1d79940dea62e830e400f01138ea9b8f997c087609a01c02b8c60f7bdb5a4897"} +{"at_utc": "2026-10-18T12:51:25.209+00:00", "data": {"http_status": 201, "ids": {"documentId": 19, "documentVersionId": 19, "embeddingJobId": 17, "fileObjectId": 17}, "outcome": "ACKNOWLEDGED", "request_id": "upload-6192da40bdbd4619821046d81292d3c7"}, "kind": "response", "previous": "1d79940dea62e830e400f01138ea9b8f997c087609a01c02b8c60f7bdb5a4897", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 97, "sha256": "9084b9a530a5caf14d629077c070476442ae9544e3ccea0b74ba6a20a447c912"} +{"at_utc": "2026-10-18T12:51:25.212+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.365, "http_status": 201, "ordinal": 17}, "kind": "upload_observed", "previous": "9084b9a530a5caf14d629077c070476442ae9544e3ccea0b74ba6a20a447c912", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 98, "sha256": "a23cdb0c7432f5b601b9997b25cdf64a586e30ce1273ab613ce270334d5c9558"} +{"at_utc": "2026-10-18T12:51:28.842+00:00", "data": {"cohort": "loss", "request_id": "upload-b29f691bda084cf783174e730f6e277f", "sha256": "fdf9d7977772c42a6e4954dab9df0260a963f11c4475794d6d457279af48717a", "size": 1045}, "kind": "sent", "previous": "a23cdb0c7432f5b601b9997b25cdf64a586e30ce1273ab613ce270334d5c9558", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 99, "sha256": "b605bcee331b8d94205f4dcd6edd37eed6b5d8a18c7626ac66fba74cff309211"} +{"at_utc": "2026-10-18T12:51:37.046+00:00", "data": {"http_status": 201, "ids": {"documentId": 20, "documentVersionId": 20, "embeddingJobId": 18, "fileObjectId": 18}, "outcome": "ACKNOWLEDGED", "request_id": "upload-b29f691bda084cf783174e730f6e277f"}, "kind": "response", "previous": "b605bcee331b8d94205f4dcd6edd37eed6b5d8a18c7626ac66fba74cff309211", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 100, "sha256": "b8331611a16692d906873eaa2e2e2450803134d12c06ee8e1d5dd88a4fc592e8"} +{"at_utc": "2026-10-18T12:51:37.050+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.203, "http_status": 201, "ordinal": 18}, "kind": "upload_observed", "previous": "b8331611a16692d906873eaa2e2e2450803134d12c06ee8e1d5dd88a4fc592e8", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 101, "sha256": "5a5679ef4d93c404ae7368187f890897ca45485da30ea3b8b59872e0fffff262"} +{"at_utc": "2026-10-18T12:51:39.840+00:00", "data": {"cohort": "loss", "request_id": "upload-abee4f99bb7a48bb9c763fdd49bd16f8", "sha256": "409f56aa647a363708e7dfff63a795da589aba643c16578b7b8e8cf1c2b30010", "size": 639}, "kind": "sent", "previous": "5a5679ef4d93c404ae7368187f890897ca45485da30ea3b8b59872e0fffff262", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 102, "sha256": "b723aee68cb2001a12d2d416e7e3cffb51b6fc10df13782bc08c32021e50954a"} +{"at_utc": "2026-10-18T12:51:47.680+00:00", "data": {"http_status": 201, "ids": {"documentId": 21, "documentVersionId": 21, "embeddingJobId": 19, "fileObjectId": 19}, "outcome": "ACKNOWLEDGED", "request_id": "upload-abee4f99bb7a48bb9c763fdd49bd16f8"}, "kind": "response", "previous": "b723aee68cb2001a12d2d416e7e3cffb51b6fc10df13782bc08c32021e50954a", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 103, "sha256": "b025bb2e3230810db2ab6829748038dd6489dece9e1934cd1697f3b83b0cdc4e"} +{"at_utc": "2026-10-18T12:51:47.684+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.84, "http_status": 201, "ordinal": 19}, "kind": "upload_observed", "previous": "b025bb2e3230810db2ab6829748038dd6489dece9e1934cd1697f3b83b0cdc4e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 104, "sha256": "b5e09aa47416893e740d1fef82f2c0248e1053d5a240769b5abb84a770c83e7d"} +{"at_utc": "2026-10-18T12:51:50.839+00:00", "data": {"cohort": "loss", "request_id": "upload-d73359977ccb452089a0bee66a0103ec", "sha256": "5e5ef2b830e149e931c7284c1b4ff0daf9d491c4deb3464b8bf8bf95a69da16f", "size": 1045}, "kind": "sent", "previous": "b5e09aa47416893e740d1fef82f2c0248e1053d5a240769b5abb84a770c83e7d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 105, "sha256": "8ec1bb4466f39d8fe001e392a2dfa8d0f47de83fe2185f930de540f2e9dd470d"} +{"at_utc": "2026-10-18T12:51:58.226+00:00", "data": {"http_status": 201, "ids": {"documentId": 22, "documentVersionId": 22, "embeddingJobId": 20, "fileObjectId": 20}, "outcome": "ACKNOWLEDGED", "request_id": "upload-d73359977ccb452089a0bee66a0103ec"}, "kind": "response", "previous": "8ec1bb4466f39d8fe001e392a2dfa8d0f47de83fe2185f930de540f2e9dd470d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 106, "sha256": "ad9c866466d952970c151d77803373707025d5298083b4dee955b51421be6ca1"} +{"at_utc": "2026-10-18T12:51:58.229+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.386, "http_status": 201, "ordinal": 20}, "kind": "upload_observed", "previous": "ad9c866466d952970c151d77803373707025d5298083b4dee955b51421be6ca1", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 107, "sha256": "bede9ee494c745636d948f484db7a0beb98513b1f78f1182e8dd2d72fcd623d1"} +{"at_utc": "2026-10-18T12:52:01.841+00:00", "data": {"cohort": "loss", "request_id": "upload-1cf0c03bbe594bb5827f2e273bc9cb14", "sha256": "42701bbc7c882a301ccdc99ab8419a1ef05573f743a09d77cc4061e50d980643", "size": 639}, "kind": "sent", "previous": "bede9ee494c745636d948f484db7a0beb98513b1f78f1182e8dd2d72fcd623d1", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 108, "sha256": "7efbc606a59e86d08ed6bcff8782c7efdb7351b4f37c5143de0ac1f8d87bf0b5"} +{"at_utc": "2026-10-18T12:52:09.841+00:00", "data": {"http_status": 201, "ids": {"documentId": 23, "documentVersionId": 23, "embeddingJobId": 21, "fileObjectId": 21}, "outcome": "ACKNOWLEDGED", "request_id": "upload-1cf0c03bbe594bb5827f2e273bc9cb14"}, "kind": "response", "previous": "7efbc606a59e86d08ed6bcff8782c7efdb7351b4f37c5143de0ac1f8d87bf0b5", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 109, "sha256": "a42fca53c78ac06b9b55be195df4fa51f410fc72f89210bc8d6a7fe23dbe9181"} +{"at_utc": "2026-10-18T12:52:09.846+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.0, "http_status": 201, "ordinal": 21}, "kind": "upload_observed", "previous": "a42fca53c78ac06b9b55be195df4fa51f410fc72f89210bc8d6a7fe23dbe9181", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 110, "sha256": "49afb8e5a7090cc81d2ffe89396bc74e6f193590a27db5c83ea248361bf2ec99"} +{"at_utc": "2026-10-18T12:52:12.847+00:00", "data": {"cohort": "loss", "request_id": "upload-491bc1dea1d34d1fb175a4a78130b40d", "sha256": "6d5d165797b911105c7313c58ad614efbc16e70ba568b99ea261d31929f807cd", "size": 1045}, "kind": "sent", "previous": "49afb8e5a7090cc81d2ffe89396bc74e6f193590a27db5c83ea248361bf2ec99", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 111, "sha256": "823e2c88a84ef76c912d5ac162818e23e99ced15f6043b10324376042dda4e62"} +{"at_utc": "2026-10-18T12:52:21.081+00:00", "data": {"http_status": 201, "ids": {"documentId": 24, "documentVersionId": 24, "embeddingJobId": 22, "fileObjectId": 22}, "outcome": "ACKNOWLEDGED", "request_id": "upload-491bc1dea1d34d1fb175a4a78130b40d"}, "kind": "response", "previous": "823e2c88a84ef76c912d5ac162818e23e99ced15f6043b10324376042dda4e62", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 112, "sha256": "2a78e4ab8758b830c9ffa254643b05802ff79d532bfda25807e12c2ec091ac0e"} +{"at_utc": "2026-10-18T12:52:21.086+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.234, "http_status": 201, "ordinal": 22}, "kind": "upload_observed", "previous": "2a78e4ab8758b830c9ffa254643b05802ff79d532bfda25807e12c2ec091ac0e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 113, "sha256": "607176d666a2f0aa98fa795f144203b472551464cd8e526c02cc4bc426504968"} +{"at_utc": "2026-10-18T12:52:23.843+00:00", "data": {"cohort": "loss", "request_id": "upload-1528b4c8ea56434286fd24803372273f", "sha256": "6c5b22d9c0bc5e4d8b53e680fad8c1ffcdfdfed3f2077bfdaca176e81010409b", "size": 639}, "kind": "sent", "previous": "607176d666a2f0aa98fa795f144203b472551464cd8e526c02cc4bc426504968", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 114, "sha256": "f2be90fc8ce1de558d00c8571f61a580c289aa5aa9b059f540b7cb82e0c116df"} +{"at_utc": "2026-10-18T12:52:31.532+00:00", "data": {"http_status": 201, "ids": {"documentId": 25, "documentVersionId": 25, "embeddingJobId": 23, "fileObjectId": 23}, "outcome": "ACKNOWLEDGED", "request_id": "upload-1528b4c8ea56434286fd24803372273f"}, "kind": "response", "previous": "f2be90fc8ce1de558d00c8571f61a580c289aa5aa9b059f540b7cb82e0c116df", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 115, "sha256": "774becbb20f33d4fa56bf4b50f5646ef54ec2e18c98fcb54712dadbebdef1121"} +{"at_utc": "2026-10-18T12:52:31.540+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.689, "http_status": 201, "ordinal": 23}, "kind": "upload_observed", "previous": "774becbb20f33d4fa56bf4b50f5646ef54ec2e18c98fcb54712dadbebdef1121", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 116, "sha256": "2785213b5b3be638594b0c4ff737852be05288a2b191a9a6ef0532c272113c3f"} +{"at_utc": "2026-10-18T12:52:34.847+00:00", "data": {"cohort": "loss", "request_id": "upload-5bbe80c1080d48dea97625043e1d6cb0", "sha256": "4adfd5f972546b1dfa778665f20ffab54a1b862e8327f1b8a027b5c35ed41e90", "size": 1045}, "kind": "sent", "previous": "2785213b5b3be638594b0c4ff737852be05288a2b191a9a6ef0532c272113c3f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 117, "sha256": "f700bd189a6bf64c7189a59ba029be2a48476d4669717596f7028762a60aaa98"} +{"at_utc": "2026-10-18T12:52:42.854+00:00", "data": {"http_status": 201, "ids": {"documentId": 26, "documentVersionId": 26, "embeddingJobId": 24, "fileObjectId": 24}, "outcome": "ACKNOWLEDGED", "request_id": "upload-5bbe80c1080d48dea97625043e1d6cb0"}, "kind": "response", "previous": "f700bd189a6bf64c7189a59ba029be2a48476d4669717596f7028762a60aaa98", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 118, "sha256": "7b0365e77aeb684de39d9a395d706e1d2ef34a0b3b3e148b6808a46a5482dac0"} +{"at_utc": "2026-10-18T12:52:42.859+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.007, "http_status": 201, "ordinal": 24}, "kind": "upload_observed", "previous": "7b0365e77aeb684de39d9a395d706e1d2ef34a0b3b3e148b6808a46a5482dac0", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 119, "sha256": "70a9252b8a9f7479b366a2b24105884634b4b4c9618e803b988b46b1f36a506d"} +{"at_utc": "2026-10-18T12:52:45.844+00:00", "data": {"cohort": "loss", "request_id": "upload-134f03b858e8450b8b08c854a839ac90", "sha256": "d80005d1c6c406a53d135ce7728671f5a50fa6497ffbbf969e7cb032629f2caa", "size": 639}, "kind": "sent", "previous": "70a9252b8a9f7479b366a2b24105884634b4b4c9618e803b988b46b1f36a506d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 120, "sha256": "3e2fea88c6eed616b7a85217340628e283575661de9fe9368642013fa70f698e"} +{"at_utc": "2026-10-18T12:52:53.221+00:00", "data": {"http_status": 201, "ids": {"documentId": 27, "documentVersionId": 27, "embeddingJobId": 25, "fileObjectId": 25}, "outcome": "ACKNOWLEDGED", "request_id": "upload-134f03b858e8450b8b08c854a839ac90"}, "kind": "response", "previous": "3e2fea88c6eed616b7a85217340628e283575661de9fe9368642013fa70f698e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 121, "sha256": "110fe933e55de23fa4aac8379d6934b5e879558e5793f942d9efa6347a303d2c"} +{"at_utc": "2026-10-18T12:52:53.226+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.377, "http_status": 201, "ordinal": 25}, "kind": "upload_observed", "previous": "110fe933e55de23fa4aac8379d6934b5e879558e5793f942d9efa6347a303d2c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 122, "sha256": "6010652d5aaf415f95beb1c3f79881004974102bd2c2db5082cba29ac7cde095"} +{"at_utc": "2026-10-18T12:52:56.848+00:00", "data": {"cohort": "loss", "request_id": "upload-1289f0ebaa3f4751a0ad788c9b648ea3", "sha256": "cf77c01b509b1129086a7b0aa93d64069b0bec875466228e24a5155c4677fa9b", "size": 1045}, "kind": "sent", "previous": "6010652d5aaf415f95beb1c3f79881004974102bd2c2db5082cba29ac7cde095", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 123, "sha256": "050675d5c29c124f8fa4c8ba0ebc9e1a8d4f17fceae53efe5f703d069456ca0c"} +{"at_utc": "2026-10-18T12:53:04.581+00:00", "data": {"http_status": 201, "ids": {"documentId": 28, "documentVersionId": 28, "embeddingJobId": 26, "fileObjectId": 26}, "outcome": "ACKNOWLEDGED", "request_id": "upload-1289f0ebaa3f4751a0ad788c9b648ea3"}, "kind": "response", "previous": "050675d5c29c124f8fa4c8ba0ebc9e1a8d4f17fceae53efe5f703d069456ca0c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 124, "sha256": "f1847754fa9d71fc0ea775eff202b3d9adb423c8ef1de420afa16ba1176f6678"} +{"at_utc": "2026-10-18T12:53:04.589+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.733, "http_status": 201, "ordinal": 26}, "kind": "upload_observed", "previous": "f1847754fa9d71fc0ea775eff202b3d9adb423c8ef1de420afa16ba1176f6678", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 125, "sha256": "3fc9977dfde2aea42c3242c4bf4c57cdf7907b841363245eb87024bd098113b1"} +{"at_utc": "2026-10-18T12:53:07.846+00:00", "data": {"cohort": "loss", "request_id": "upload-2fb35853a420491c971f6c971b63896b", "sha256": "7f923c7e4b2be2a18a9f4324f9c1d6269f2d01c74062645f5918b2e035b93076", "size": 639}, "kind": "sent", "previous": "3fc9977dfde2aea42c3242c4bf4c57cdf7907b841363245eb87024bd098113b1", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 126, "sha256": "42a84e3595aac7a34511e1a572ae48be38f204d3e0249bb734dfbdb9a7b4ff83"} +{"at_utc": "2026-10-18T12:53:15.508+00:00", "data": {"http_status": 201, "ids": {"documentId": 29, "documentVersionId": 29, "embeddingJobId": 27, "fileObjectId": 27}, "outcome": "ACKNOWLEDGED", "request_id": "upload-2fb35853a420491c971f6c971b63896b"}, "kind": "response", "previous": "42a84e3595aac7a34511e1a572ae48be38f204d3e0249bb734dfbdb9a7b4ff83", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 127, "sha256": "904044d5249644ccbe7b139bc758f48b688bca6565f96731aac3110934296bdf"} +{"at_utc": "2026-10-18T12:53:15.514+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.662, "http_status": 201, "ordinal": 27}, "kind": "upload_observed", "previous": "904044d5249644ccbe7b139bc758f48b688bca6565f96731aac3110934296bdf", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 128, "sha256": "87ec1cb74edb02b23e21e7fa617d108b2d6570c8cf6206161a43e9d3d6f8a60c"} +{"at_utc": "2026-10-18T12:53:18.848+00:00", "data": {"cohort": "loss", "request_id": "upload-dd1a5e718ce24ce8b4508229da0951aa", "sha256": "7630822014249f6f05b8fcc99ade977ac1ed1db491bd43a5198b8af046a8bcb5", "size": 1045}, "kind": "sent", "previous": "87ec1cb74edb02b23e21e7fa617d108b2d6570c8cf6206161a43e9d3d6f8a60c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 129, "sha256": "f100de8a55cab9958bf676551c85c4a20ee2af5f25e02e39a50cf3120cb41c12"} +{"at_utc": "2026-10-18T12:53:27.678+00:00", "data": {"http_status": 201, "ids": {"documentId": 30, "documentVersionId": 30, "embeddingJobId": 28, "fileObjectId": 28}, "outcome": "ACKNOWLEDGED", "request_id": "upload-dd1a5e718ce24ce8b4508229da0951aa"}, "kind": "response", "previous": "f100de8a55cab9958bf676551c85c4a20ee2af5f25e02e39a50cf3120cb41c12", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 130, "sha256": "786611ff18544b33fdd804a1ed3c5bca97cbfc8153cbc9ab6e2211cdbba4edab"} +{"at_utc": "2026-10-18T12:53:27.683+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.83, "http_status": 201, "ordinal": 28}, "kind": "upload_observed", "previous": "786611ff18544b33fdd804a1ed3c5bca97cbfc8153cbc9ab6e2211cdbba4edab", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 131, "sha256": "ef2bc5d8c91111c0f4aede46b38eb7f2edef70a32126e2fb38f31d686617a2fd"} +{"at_utc": "2026-10-18T12:53:29.847+00:00", "data": {"cohort": "loss", "request_id": "upload-c01c1678cb594cd993a9a0170bb18cd2", "sha256": "0a05be12c9ee25da1b2bb3bc24fb51bed25970f9896f473067769bd4ac987680", "size": 639}, "kind": "sent", "previous": "ef2bc5d8c91111c0f4aede46b38eb7f2edef70a32126e2fb38f31d686617a2fd", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 132, "sha256": "7c6f813a3c5b278bf069570979e61fad18da6e89c4297d08d4f8fbc898371a5f"} +{"at_utc": "2026-10-18T12:53:37.267+00:00", "data": {"http_status": 201, "ids": {"documentId": 31, "documentVersionId": 31, "embeddingJobId": 29, "fileObjectId": 29}, "outcome": "ACKNOWLEDGED", "request_id": "upload-c01c1678cb594cd993a9a0170bb18cd2"}, "kind": "response", "previous": "7c6f813a3c5b278bf069570979e61fad18da6e89c4297d08d4f8fbc898371a5f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 133, "sha256": "585a24f020b4f15aad18883d5d3b2eb59881881d381c3d6f011744b76edb40d8"} +{"at_utc": "2026-10-18T12:53:37.272+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.42, "http_status": 201, "ordinal": 29}, "kind": "upload_observed", "previous": "585a24f020b4f15aad18883d5d3b2eb59881881d381c3d6f011744b76edb40d8", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 134, "sha256": "0282e180dd296d18f8aa3a54c99236d1bd74ce20ac2d56da96d80d556a8611ef"} +{"at_utc": "2026-10-18T12:53:40.849+00:00", "data": {"cohort": "loss", "request_id": "upload-83040d6892d74b63ab768e9d90bb9d04", "sha256": "d3d1f41587d7a417b5c9d6e1f7affd062395787a32b6e75043fb8f15f84e22ce", "size": 1045}, "kind": "sent", "previous": "0282e180dd296d18f8aa3a54c99236d1bd74ce20ac2d56da96d80d556a8611ef", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 135, "sha256": "a1876e234e18868cf6b62b92a6f05e933bd36a7ad3131a564bcca2efc955cc9d"} +{"at_utc": "2026-10-18T12:53:48.798+00:00", "data": {"http_status": 201, "ids": {"documentId": 32, "documentVersionId": 32, "embeddingJobId": 30, "fileObjectId": 30}, "outcome": "ACKNOWLEDGED", "request_id": "upload-83040d6892d74b63ab768e9d90bb9d04"}, "kind": "response", "previous": "a1876e234e18868cf6b62b92a6f05e933bd36a7ad3131a564bcca2efc955cc9d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 136, "sha256": "05cc3a68d9dbdfc0dd88bc36261a8bc90715487e335ea610b944d73834a55583"} +{"at_utc": "2026-10-18T12:53:48.803+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.948, "http_status": 201, "ordinal": 30}, "kind": "upload_observed", "previous": "05cc3a68d9dbdfc0dd88bc36261a8bc90715487e335ea610b944d73834a55583", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 137, "sha256": "9362826003e7f1e0b713a10caefda57920295f8bf758dbb74e42b1d1d91ea75f"} +{"at_utc": "2026-10-18T12:53:51.847+00:00", "data": {"cohort": "loss", "request_id": "upload-603a571ae0c147bea35927c304f9d772", "sha256": "a871ca01a920e8762991610ff7e7c81ff7874d90214cc30ff20a95bdaba83899", "size": 639}, "kind": "sent", "previous": "9362826003e7f1e0b713a10caefda57920295f8bf758dbb74e42b1d1d91ea75f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 138, "sha256": "f1b89bf714d360c85ae8c31431997932e2fb2072c4e3f589d4cca7b8f83e120f"} +{"at_utc": "2026-10-18T12:53:59.902+00:00", "data": {"http_status": 201, "ids": {"documentId": 33, "documentVersionId": 33, "embeddingJobId": 31, "fileObjectId": 31}, "outcome": "ACKNOWLEDGED", "request_id": "upload-603a571ae0c147bea35927c304f9d772"}, "kind": "response", "previous": "f1b89bf714d360c85ae8c31431997932e2fb2072c4e3f589d4cca7b8f83e120f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 139, "sha256": "13e0435b929c9486120efecd34f8e9a08fb32668b335ec330938d58c167d4a3f"} +{"at_utc": "2026-10-18T12:53:59.911+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.055, "http_status": 201, "ordinal": 31}, "kind": "upload_observed", "previous": "13e0435b929c9486120efecd34f8e9a08fb32668b335ec330938d58c167d4a3f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 140, "sha256": "14f3f898ecdb9aba9b3c483ed1a5818ab0fa1c0d05adca93ead80487ff897245"} +{"at_utc": "2026-10-18T12:54:02.853+00:00", "data": {"cohort": "loss", "request_id": "upload-1a65c1542f684feeb034c3b1a6b92c37", "sha256": "4588a9b4f22011102e8ffcca351041c4ad02e784d45cb753f1f312f25d5c0d5b", "size": 1045}, "kind": "sent", "previous": "14f3f898ecdb9aba9b3c483ed1a5818ab0fa1c0d05adca93ead80487ff897245", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 141, "sha256": "06eee29a917134e6bbb995eb7c18076a62f92bf03a1f0ad190fb80ad66be079d"} +{"at_utc": "2026-10-18T12:54:10.932+00:00", "data": {"http_status": 201, "ids": {"documentId": 34, "documentVersionId": 34, "embeddingJobId": 32, "fileObjectId": 32}, "outcome": "ACKNOWLEDGED", "request_id": "upload-1a65c1542f684feeb034c3b1a6b92c37"}, "kind": "response", "previous": "06eee29a917134e6bbb995eb7c18076a62f92bf03a1f0ad190fb80ad66be079d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 142, "sha256": "b5e6b4387d55239cc01a220b2b9818284678b2ae4dbb193dad027f0d87cc0911"} +{"at_utc": "2026-10-18T12:54:10.938+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.079, "http_status": 201, "ordinal": 32}, "kind": "upload_observed", "previous": "b5e6b4387d55239cc01a220b2b9818284678b2ae4dbb193dad027f0d87cc0911", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 143, "sha256": "1d4469f52626393a1fab54245fd710061281a1ff3792362412d067f3ed503cf2"} +{"at_utc": "2026-10-18T12:54:13.842+00:00", "data": {"cohort": "loss", "request_id": "upload-0e631fa0a6ea4ee1afac2ce5f99f4327", "sha256": "38ea0ae6af7f1af5b8acc0bc844f34a8a8e05e7621a89725fc26c78f9ab6475e", "size": 639}, "kind": "sent", "previous": "1d4469f52626393a1fab54245fd710061281a1ff3792362412d067f3ed503cf2", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 144, "sha256": "fa0ea64ac43b850f5e02f5d2d43e7266677bea83e2e29215d12de55e671e6223"} +{"at_utc": "2026-10-18T12:54:21.832+00:00", "data": {"http_status": 201, "ids": {"documentId": 35, "documentVersionId": 35, "embeddingJobId": 33, "fileObjectId": 33}, "outcome": "ACKNOWLEDGED", "request_id": "upload-0e631fa0a6ea4ee1afac2ce5f99f4327"}, "kind": "response", "previous": "fa0ea64ac43b850f5e02f5d2d43e7266677bea83e2e29215d12de55e671e6223", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 145, "sha256": "d0f34340cae709f2914016a658b4d60634aa10e730b541fdd601abe616efc48c"} +{"at_utc": "2026-10-18T12:54:21.839+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.99, "http_status": 201, "ordinal": 33}, "kind": "upload_observed", "previous": "d0f34340cae709f2914016a658b4d60634aa10e730b541fdd601abe616efc48c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 146, "sha256": "0917ca0cd02b8e9cd79f40f52e0c32584275b4ced22ca77f60da4e68fa8893f8"} +{"at_utc": "2026-10-18T12:54:21.843+00:00", "data": {"completed_upload_requests": 33, "ordinal": 1, "scheduled_at_utc": "2026-10-18T12:54:16.828378+00:00", "start_lag_seconds": 5.005}, "kind": "offsite_schedule_observed", "previous": "0917ca0cd02b8e9cd79f40f52e0c32584275b4ced22ca77f60da4e68fa8893f8", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 147, "sha256": "edd29afd361f6603a1c258c01c1b526416ca548e645f715595d5904c9a7a8675"} +{"at_utc": "2026-10-18T12:54:46.502+00:00", "data": {"fileObjectId": 6, "generation": "1792328086019431", "sha256": "6403d52e2ebc1bf1adabfd78a27dad188d674a61a25d3125864a896f3a18e3b8", "size": 1044}, "kind": "original_preserved", "previous": "edd29afd361f6603a1c258c01c1b526416ca548e645f715595d5904c9a7a8675", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 148, "sha256": "851761cb70663b573bf90d1eab6ec66ca377b18ee7e9e4167abf2123b48df166"} +{"at_utc": "2026-10-18T12:54:48.017+00:00", "data": {"fileObjectId": 7, "generation": "1792328087449527", "sha256": "b5533ff9dc19e8ab13eceda011d203151c304adb6c15a76f9624c40e4f727229", "size": 638}, "kind": "original_preserved", "previous": "851761cb70663b573bf90d1eab6ec66ca377b18ee7e9e4167abf2123b48df166", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 149, "sha256": "8a4fca84088ddfd15534746efe8161023690aa1f3988f365eb0bc5e5a68f399b"} +{"at_utc": "2026-10-18T12:54:49.372+00:00", "data": {"fileObjectId": 8, "generation": "1792328088900545", "sha256": "17ca77586adbaf5282fa290b8daa01e087a06aa263602a51f4ae6557813e6a96", "size": 1044}, "kind": "original_preserved", "previous": "8a4fca84088ddfd15534746efe8161023690aa1f3988f365eb0bc5e5a68f399b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 150, "sha256": "4265dbca2a6f6541736f1d19c7c93be2d21c8c5b2a0b465ef240be6698724b5d"} +{"at_utc": "2026-10-18T12:54:50.617+00:00", "data": {"fileObjectId": 9, "generation": "1792328090136180", "sha256": "389fdb5425c931b59623ed2ba6888f9e120609317fe8963245d88aec7304d929", "size": 638}, "kind": "original_preserved", "previous": "4265dbca2a6f6541736f1d19c7c93be2d21c8c5b2a0b465ef240be6698724b5d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 151, "sha256": "53dea9a8a19b2dabf0d8ad876ede01bebb94b0f5b6bcb078048eb06a95c4dc52"} +{"at_utc": "2026-10-18T12:54:51.975+00:00", "data": {"fileObjectId": 10, "generation": "1792328091427646", "sha256": "ea06cf64bc7c18fd6379ee9fbc9eade765483eadbeee54799f7add448ad4288c", "size": 1045}, "kind": "original_preserved", "previous": "53dea9a8a19b2dabf0d8ad876ede01bebb94b0f5b6bcb078048eb06a95c4dc52", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 152, "sha256": "9db218ff05bb86e7c7d4ab2103634dbaad8d06ab5d30851372f56e5be6dc08b8"} +{"at_utc": "2026-10-18T12:54:53.262+00:00", "data": {"fileObjectId": 11, "generation": "1792328092722116", "sha256": "2747f698cfa86d0f8f0873b9ec34139ab9a13df1a748a441538d2f764af69c4b", "size": 639}, "kind": "original_preserved", "previous": "9db218ff05bb86e7c7d4ab2103634dbaad8d06ab5d30851372f56e5be6dc08b8", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 153, "sha256": "e4d977bc1aebc167a5d95b3e05297475005ef2ad699b525d0b4fa3ce7331be3b"} +{"at_utc": "2026-10-18T12:54:54.458+00:00", "data": {"fileObjectId": 12, "generation": "1792328094048871", "sha256": "f052804612d30f76364e7c3e9103be400f0b1117eb36920e0c56de1eb80cc19d", "size": 1045}, "kind": "original_preserved", "previous": "e4d977bc1aebc167a5d95b3e05297475005ef2ad699b525d0b4fa3ce7331be3b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 154, "sha256": "82733d69e86d1fd0683e3079df5a9a5200385774fcd20649c00dcca1b8733fba"} +{"at_utc": "2026-10-18T12:54:55.862+00:00", "data": {"fileObjectId": 13, "generation": "1792328095388030", "sha256": "172c782cc9e625e2e54344a27890565cc8461a2aacc7c38e1b29d4197322a243", "size": 639}, "kind": "original_preserved", "previous": "82733d69e86d1fd0683e3079df5a9a5200385774fcd20649c00dcca1b8733fba", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 155, "sha256": "44a36e0c8e2e1ca1b7bf5f145cab7c103e69d323459501659645601f1312c0ca"} +{"at_utc": "2026-10-18T12:54:57.241+00:00", "data": {"fileObjectId": 14, "generation": "1792328096826330", "sha256": "f29454ce42ad1c2047303fb13400aba5043e7483c49ab66c6294ac7677c5b566", "size": 1045}, "kind": "original_preserved", "previous": "44a36e0c8e2e1ca1b7bf5f145cab7c103e69d323459501659645601f1312c0ca", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 156, "sha256": "00834fac126a8512c41c2182bd709556f4c2590c312362ecc4f4f2ae5f6f1882"} +{"at_utc": "2026-10-18T12:54:58.484+00:00", "data": {"fileObjectId": 15, "generation": "1792328097997840", "sha256": "c5f84d6ccff80425b9dd6a301379c50c6c0ba91185f968280bf6eff1d36f758a", "size": 639}, "kind": "original_preserved", "previous": "00834fac126a8512c41c2182bd709556f4c2590c312362ecc4f4f2ae5f6f1882", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 157, "sha256": "635a4192bce7d1caf839d9e5cccd6efb86cc56fdc522f99e252177c44a93957d"} +{"at_utc": "2026-10-18T12:55:00.125+00:00", "data": {"fileObjectId": 16, "generation": "1792328099653018", "sha256": "e23b60a24d1b2f413ac0b5a9d712b9f9df7a7c256b58344588fb5dc8b8a3baf2", "size": 1045}, "kind": "original_preserved", "previous": "635a4192bce7d1caf839d9e5cccd6efb86cc56fdc522f99e252177c44a93957d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 158, "sha256": "38482068672f381aa1ec3437a6e3ba39e57188676e20aae92e8c4a37ebb85fc3"} +{"at_utc": "2026-10-18T12:55:01.485+00:00", "data": {"fileObjectId": 17, "generation": "1792328101064159", "sha256": "7a4d5693fe950d8499727512a0c01f613c912b09ac020f4bfea558a347dc104f", "size": 639}, "kind": "original_preserved", "previous": "38482068672f381aa1ec3437a6e3ba39e57188676e20aae92e8c4a37ebb85fc3", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 159, "sha256": "40550dbd0423bc44bfd65f1ead2d0d96fbb060f17734d7459e48c70c19065e8e"} +{"at_utc": "2026-10-18T12:55:02.753+00:00", "data": {"fileObjectId": 18, "generation": "1792328102247810", "sha256": "fdf9d7977772c42a6e4954dab9df0260a963f11c4475794d6d457279af48717a", "size": 1045}, "kind": "original_preserved", "previous": "40550dbd0423bc44bfd65f1ead2d0d96fbb060f17734d7459e48c70c19065e8e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 160, "sha256": "44e1d33877cd1888ee8b6cf1f20691a3c0e9bcdbc5bf4ac2bac46950d35ebb29"} +{"at_utc": "2026-10-18T12:55:03.916+00:00", "data": {"fileObjectId": 19, "generation": "1792328103504958", "sha256": "409f56aa647a363708e7dfff63a795da589aba643c16578b7b8e8cf1c2b30010", "size": 639}, "kind": "original_preserved", "previous": "44e1d33877cd1888ee8b6cf1f20691a3c0e9bcdbc5bf4ac2bac46950d35ebb29", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 161, "sha256": "e580849f53f904702a803bb5a0404b6cc204e90cb5bb84de06a65b1348cd932c"} +{"at_utc": "2026-10-18T12:55:05.344+00:00", "data": {"fileObjectId": 20, "generation": "1792328104848915", "sha256": "5e5ef2b830e149e931c7284c1b4ff0daf9d491c4deb3464b8bf8bf95a69da16f", "size": 1045}, "kind": "original_preserved", "previous": "e580849f53f904702a803bb5a0404b6cc204e90cb5bb84de06a65b1348cd932c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 162, "sha256": "f4667bb99d645d493dc0f6d7d218b6b91966b6d74eebe5def01b2a79bfd7a9e3"} +{"at_utc": "2026-10-18T12:55:06.721+00:00", "data": {"fileObjectId": 21, "generation": "1792328106308712", "sha256": "42701bbc7c882a301ccdc99ab8419a1ef05573f743a09d77cc4061e50d980643", "size": 639}, "kind": "original_preserved", "previous": "f4667bb99d645d493dc0f6d7d218b6b91966b6d74eebe5def01b2a79bfd7a9e3", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 163, "sha256": "75832769b691ede372ac8d2ab0de2ebafcb00c2351519f45b5bc63fd20232c28"} +{"at_utc": "2026-10-18T12:55:07.993+00:00", "data": {"fileObjectId": 22, "generation": "1792328107545458", "sha256": "6d5d165797b911105c7313c58ad614efbc16e70ba568b99ea261d31929f807cd", "size": 1045}, "kind": "original_preserved", "previous": "75832769b691ede372ac8d2ab0de2ebafcb00c2351519f45b5bc63fd20232c28", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 164, "sha256": "f927560c1b9d67006f1f6f5cec780dcf7297477c268c96bfc9fb3465e5e96c42"} +{"at_utc": "2026-10-18T12:55:09.386+00:00", "data": {"fileObjectId": 23, "generation": "1792328108969233", "sha256": "6c5b22d9c0bc5e4d8b53e680fad8c1ffcdfdfed3f2077bfdaca176e81010409b", "size": 639}, "kind": "original_preserved", "previous": "f927560c1b9d67006f1f6f5cec780dcf7297477c268c96bfc9fb3465e5e96c42", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 165, "sha256": "08646b596ec4de370451f2d9e0c2715d77a2e387ddf0388f2853e0b6badc0811"} +{"at_utc": "2026-10-18T12:55:10.658+00:00", "data": {"fileObjectId": 24, "generation": "1792328110244934", "sha256": "4adfd5f972546b1dfa778665f20ffab54a1b862e8327f1b8a027b5c35ed41e90", "size": 1045}, "kind": "original_preserved", "previous": "08646b596ec4de370451f2d9e0c2715d77a2e387ddf0388f2853e0b6badc0811", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 166, "sha256": "d657e59c2747564f9ac08251c3c6c4aeb2d42514fd3a12d58f443504c0dd1796"} +{"at_utc": "2026-10-18T12:55:12.298+00:00", "data": {"fileObjectId": 25, "generation": "1792328111630812", "sha256": "d80005d1c6c406a53d135ce7728671f5a50fa6497ffbbf969e7cb032629f2caa", "size": 639}, "kind": "original_preserved", "previous": "d657e59c2747564f9ac08251c3c6c4aeb2d42514fd3a12d58f443504c0dd1796", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 167, "sha256": "702bcb0cee9ffe6224bc0a19bb95907d7136ab92d2a30a1b6918ca431608421e"} +{"at_utc": "2026-10-18T12:55:13.797+00:00", "data": {"fileObjectId": 26, "generation": "1792328113382947", "sha256": "cf77c01b509b1129086a7b0aa93d64069b0bec875466228e24a5155c4677fa9b", "size": 1045}, "kind": "original_preserved", "previous": "702bcb0cee9ffe6224bc0a19bb95907d7136ab92d2a30a1b6918ca431608421e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 168, "sha256": "acd1718dab8b1266cf78c09fe6fcb0561333bb26ee37a987c2774891433c72ce"} +{"at_utc": "2026-10-18T12:55:15.159+00:00", "data": {"fileObjectId": 27, "generation": "1792328114737996", "sha256": "7f923c7e4b2be2a18a9f4324f9c1d6269f2d01c74062645f5918b2e035b93076", "size": 639}, "kind": "original_preserved", "previous": "acd1718dab8b1266cf78c09fe6fcb0561333bb26ee37a987c2774891433c72ce", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 169, "sha256": "de54f586835f5c68d46e50fd40b7c27bed78ea4429d3e4b95df816a1f04921a9"} +{"at_utc": "2026-10-18T12:55:16.420+00:00", "data": {"fileObjectId": 28, "generation": "1792328116002594", "sha256": "7630822014249f6f05b8fcc99ade977ac1ed1db491bd43a5198b8af046a8bcb5", "size": 1045}, "kind": "original_preserved", "previous": "de54f586835f5c68d46e50fd40b7c27bed78ea4429d3e4b95df816a1f04921a9", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 170, "sha256": "e41328cc87bfbffb118c8e237f0948d32d5c62c9c4be1a5ea0d75843f5cd2d73"} +{"at_utc": "2026-10-18T12:55:17.690+00:00", "data": {"fileObjectId": 29, "generation": "1792328117286515", "sha256": "0a05be12c9ee25da1b2bb3bc24fb51bed25970f9896f473067769bd4ac987680", "size": 639}, "kind": "original_preserved", "previous": "e41328cc87bfbffb118c8e237f0948d32d5c62c9c4be1a5ea0d75843f5cd2d73", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 171, "sha256": "0027532e112ec323856d7bfdb421afea7d77c46fdf26a41e0bf6317a26806d8b"} +{"at_utc": "2026-10-18T12:55:18.997+00:00", "data": {"fileObjectId": 30, "generation": "1792328118488397", "sha256": "d3d1f41587d7a417b5c9d6e1f7affd062395787a32b6e75043fb8f15f84e22ce", "size": 1045}, "kind": "original_preserved", "previous": "0027532e112ec323856d7bfdb421afea7d77c46fdf26a41e0bf6317a26806d8b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 172, "sha256": "ea9bed00ee4d0b7f889d1bf45a18ec54d98cc68095176277542e5326948ac07d"} +{"at_utc": "2026-10-18T12:55:20.186+00:00", "data": {"fileObjectId": 31, "generation": "1792328119765199", "sha256": "a871ca01a920e8762991610ff7e7c81ff7874d90214cc30ff20a95bdaba83899", "size": 639}, "kind": "original_preserved", "previous": "ea9bed00ee4d0b7f889d1bf45a18ec54d98cc68095176277542e5326948ac07d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 173, "sha256": "731b0d43a71217c9b99a06bdc9393f47bb042bc472a5e3a3d479ed34f1654551"} +{"at_utc": "2026-10-18T12:55:21.547+00:00", "data": {"fileObjectId": 32, "generation": "1792328121047737", "sha256": "4588a9b4f22011102e8ffcca351041c4ad02e784d45cb753f1f312f25d5c0d5b", "size": 1045}, "kind": "original_preserved", "previous": "731b0d43a71217c9b99a06bdc9393f47bb042bc472a5e3a3d479ed34f1654551", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 174, "sha256": "c5bc5ff3036e14cce85216ef955059f3e99b6733774ecc78c8629a479fae9068"} +{"at_utc": "2026-10-18T12:55:22.764+00:00", "data": {"fileObjectId": 33, "generation": "1792328122347029", "sha256": "38ea0ae6af7f1af5b8acc0bc844f34a8a8e05e7621a89725fc26c78f9ab6475e", "size": 639}, "kind": "original_preserved", "previous": "c5bc5ff3036e14cce85216ef955059f3e99b6733774ecc78c8629a479fae9068", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 175, "sha256": "4fac65b186b4b2c762646e4bda57090202b7a88530f52a35e9b4475d3b442d9d"} +{"at_utc": "2026-10-18T12:55:27.169+00:00", "data": {"generation": "1792328125375427", "name": "000000060000000000000021", "sha256": "7b20f95786aa9238fbb93dfd399216c034d93392d22559fea2fda917551d9e8b", "size": 16777216}, "kind": "wal_preserved", "previous": "4fac65b186b4b2c762646e4bda57090202b7a88530f52a35e9b4475d3b442d9d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 176, "sha256": "786cd95c7ca91966daeeee29c50868a95fff7ed42b2dee8f8753906f06f9156b"} +{"at_utc": "2026-10-18T12:55:31.207+00:00", "data": {"generation": "1792328129379945", "name": "000000060000000000000022", "sha256": "d3172ba08c364acda7721e4cbab32bdff67735550cf47002648dce40aac6d50f", "size": 16777216}, "kind": "wal_preserved", "previous": "786cd95c7ca91966daeeee29c50868a95fff7ed42b2dee8f8753906f06f9156b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 177, "sha256": "1538f7ff3612a7517a30bd6afd887f344a58d38a99078f431b9f4060a8ddd538"} +{"at_utc": "2026-10-18T12:55:35.358+00:00", "data": {"generation": "1792328133749894", "name": "000000060000000000000023", "sha256": "852b7974ecf0ba081e258df1c44f6419469f97184617452a38b9e4e7f5840015", "size": 16777216}, "kind": "wal_preserved", "previous": "1538f7ff3612a7517a30bd6afd887f344a58d38a99078f431b9f4060a8ddd538", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 178, "sha256": "1c789ad16af81565fa44137229bb6972c2ba8ad637dfdbc779382816006acdc2"} +{"at_utc": "2026-10-18T12:55:35.364+00:00", "data": {"complete_wals": 7, "elapsed_seconds": 73.514, "ordinal": 1, "preserved_originals": 33}, "kind": "offsite_round_completed", "previous": "1c789ad16af81565fa44137229bb6972c2ba8ad637dfdbc779382816006acdc2", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 179, "sha256": "d82c639973302b595f0e7a16ef167f8ca7cd6dcc3eaf896deb1e55bd0397b997"} +{"at_utc": "2026-10-18T12:55:35.372+00:00", "data": {"cohort": "loss", "request_id": "upload-80ba1cb578334f68b4712ec46271edab", "sha256": "2b2330ff3f21d26f156091669add4e90615d0364196fa79bb1e6b43ad9d7b621", "size": 1045}, "kind": "sent", "previous": "d82c639973302b595f0e7a16ef167f8ca7cd6dcc3eaf896deb1e55bd0397b997", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 180, "sha256": "5170cd58dd4e7bb84d903c9dbe1d27795208cdd8a3bc57614a82fc193f04a8ca"} +{"at_utc": "2026-10-18T12:55:43.909+00:00", "data": {"http_status": 201, "ids": {"documentId": 36, "documentVersionId": 36, "embeddingJobId": 34, "fileObjectId": 34}, "outcome": "ACKNOWLEDGED", "request_id": "upload-80ba1cb578334f68b4712ec46271edab"}, "kind": "response", "previous": "5170cd58dd4e7bb84d903c9dbe1d27795208cdd8a3bc57614a82fc193f04a8ca", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 181, "sha256": "8bf14e13ce3e05f5bb106cc272fa3078edac72bb509359c0d80e6839c47937fe"} +{"at_utc": "2026-10-18T12:55:43.917+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.537, "http_status": 201, "ordinal": 34}, "kind": "upload_observed", "previous": "8bf14e13ce3e05f5bb106cc272fa3078edac72bb509359c0d80e6839c47937fe", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 182, "sha256": "5cc1662b9f8ba55f2acc048225bca7c7439242bb58c7fe7bd38811fb42ab4353"} +{"at_utc": "2026-10-18T12:55:43.925+00:00", "data": {"cohort": "loss", "request_id": "upload-fbaf3249ead041aab8c8729f7cfe6bb5", "sha256": "33cff912087abdd6949f053df3efd3ee4cc3dc9f4410a9d610186511bf56a8da", "size": 639}, "kind": "sent", "previous": "5cc1662b9f8ba55f2acc048225bca7c7439242bb58c7fe7bd38811fb42ab4353", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 183, "sha256": "65d055908ad18954a15be36d98c2c3abf98ede0252f2a3fbd68c3b763b5c5260"} +{"at_utc": "2026-10-18T12:55:50.968+00:00", "data": {"http_status": 201, "ids": {"documentId": 37, "documentVersionId": 37, "embeddingJobId": 35, "fileObjectId": 35}, "outcome": "ACKNOWLEDGED", "request_id": "upload-fbaf3249ead041aab8c8729f7cfe6bb5"}, "kind": "response", "previous": "65d055908ad18954a15be36d98c2c3abf98ede0252f2a3fbd68c3b763b5c5260", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 184, "sha256": "65574da8756178d6794e375fe723b9b7f367f90ea8956a26ede8fc51848fc94e"} +{"at_utc": "2026-10-18T12:55:50.975+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.043, "http_status": 201, "ordinal": 35}, "kind": "upload_observed", "previous": "65574da8756178d6794e375fe723b9b7f367f90ea8956a26ede8fc51848fc94e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 185, "sha256": "67c13cccb8c20583dfd7b1c31c56cee1d5ff051f2cce499413545fa6f512a378"} +{"at_utc": "2026-10-18T12:55:50.984+00:00", "data": {"cohort": "loss", "request_id": "upload-f9cecc4bc6e645ebae7a7d8f0b60833f", "sha256": "bb8222fa45c741a91a26739414b68b4cab720cc2d938b0075016bc1627de5962", "size": 1045}, "kind": "sent", "previous": "67c13cccb8c20583dfd7b1c31c56cee1d5ff051f2cce499413545fa6f512a378", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 186, "sha256": "bd80398c614a49feb8b1e066e31df230910751e5f8cd99cea2edf9d87102b691"} +{"at_utc": "2026-10-18T12:55:58.082+00:00", "data": {"http_status": 201, "ids": {"documentId": 38, "documentVersionId": 38, "embeddingJobId": 36, "fileObjectId": 36}, "outcome": "ACKNOWLEDGED", "request_id": "upload-f9cecc4bc6e645ebae7a7d8f0b60833f"}, "kind": "response", "previous": "bd80398c614a49feb8b1e066e31df230910751e5f8cd99cea2edf9d87102b691", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 187, "sha256": "ed7b8e017925fbd268ef9dc573dc23464c58144ff579a25312def6c3232e9a79"} +{"at_utc": "2026-10-18T12:55:58.089+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.099, "http_status": 201, "ordinal": 36}, "kind": "upload_observed", "previous": "ed7b8e017925fbd268ef9dc573dc23464c58144ff579a25312def6c3232e9a79", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 188, "sha256": "59f9b115bb1aedcc9d65b3f576600fc03120b05a0b4cf3fb48b5f84948d9d4a2"} +{"at_utc": "2026-10-18T12:55:58.098+00:00", "data": {"cohort": "loss", "request_id": "upload-266e9cd10bf44c4099c45351aff5a213", "sha256": "09e6f22bc526b38bf65db51e22bdcd4c9c6ab2679644ab17a21b07cb6237e6dc", "size": 639}, "kind": "sent", "previous": "59f9b115bb1aedcc9d65b3f576600fc03120b05a0b4cf3fb48b5f84948d9d4a2", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 189, "sha256": "a17e1c7638b1b63ddeeabe756875d34974ad912c90b896df1ccc4b7ad3a5a62f"} +{"at_utc": "2026-10-18T12:56:05.900+00:00", "data": {"http_status": 201, "ids": {"documentId": 39, "documentVersionId": 39, "embeddingJobId": 37, "fileObjectId": 37}, "outcome": "ACKNOWLEDGED", "request_id": "upload-266e9cd10bf44c4099c45351aff5a213"}, "kind": "response", "previous": "a17e1c7638b1b63ddeeabe756875d34974ad912c90b896df1ccc4b7ad3a5a62f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 190, "sha256": "7ee22861b2cc0e2a4c11fd57377d51da484ff4b0bfe1dc010dcfadc270d2f585"} +{"at_utc": "2026-10-18T12:56:05.903+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.802, "http_status": 201, "ordinal": 37}, "kind": "upload_observed", "previous": "7ee22861b2cc0e2a4c11fd57377d51da484ff4b0bfe1dc010dcfadc270d2f585", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 191, "sha256": "0868b748fe679f58e0d9a59d907537fa3c053f86893b07d17465585e22b5f585"} +{"at_utc": "2026-10-18T12:56:05.907+00:00", "data": {"cohort": "loss", "request_id": "upload-f385c020c7394b5993fc0d23dd0c4314", "sha256": "c6d0c699ab27b05a52f159dcd884648f6b7a504b1fe203139e47570379b634f6", "size": 1045}, "kind": "sent", "previous": "0868b748fe679f58e0d9a59d907537fa3c053f86893b07d17465585e22b5f585", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 192, "sha256": "d86df62dd7575c9152c3f45bbdc8b92b33bdb5a6f6b3b2bdb000cf57959f199c"} +{"at_utc": "2026-10-18T12:56:13.259+00:00", "data": {"http_status": 201, "ids": {"documentId": 40, "documentVersionId": 40, "embeddingJobId": 38, "fileObjectId": 38}, "outcome": "ACKNOWLEDGED", "request_id": "upload-f385c020c7394b5993fc0d23dd0c4314"}, "kind": "response", "previous": "d86df62dd7575c9152c3f45bbdc8b92b33bdb5a6f6b3b2bdb000cf57959f199c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 193, "sha256": "52394fc65eac5b27c9df0ffe4e944e6be3cbfd77b94050a3d0940fdb9a097d68"} +{"at_utc": "2026-10-18T12:56:13.265+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.352, "http_status": 201, "ordinal": 38}, "kind": "upload_observed", "previous": "52394fc65eac5b27c9df0ffe4e944e6be3cbfd77b94050a3d0940fdb9a097d68", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 194, "sha256": "5098a90912e65c8f5960f285bb47d274a9f2615c239f59dc6f5a8db13878b8d1"} +{"at_utc": "2026-10-18T12:56:13.273+00:00", "data": {"cohort": "loss", "request_id": "upload-cd52989eefdc4d01993e077360b6e844", "sha256": "848b879592449d6780759eb458312a4118a971e0896bbbddb44d84fa7cf4cf42", "size": 639}, "kind": "sent", "previous": "5098a90912e65c8f5960f285bb47d274a9f2615c239f59dc6f5a8db13878b8d1", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 195, "sha256": "2b55d8a91f78505fa0a1fb4098be974787d50b286083a5b2e75f9d437919c46d"} +{"at_utc": "2026-10-18T12:56:20.688+00:00", "data": {"http_status": 201, "ids": {"documentId": 41, "documentVersionId": 41, "embeddingJobId": 39, "fileObjectId": 39}, "outcome": "ACKNOWLEDGED", "request_id": "upload-cd52989eefdc4d01993e077360b6e844"}, "kind": "response", "previous": "2b55d8a91f78505fa0a1fb4098be974787d50b286083a5b2e75f9d437919c46d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 196, "sha256": "4612cd97db7d6aebd73cf1f6f396880c08f58613a136ed8bfa1e3b3e61e4b4a1"} +{"at_utc": "2026-10-18T12:56:20.694+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.414, "http_status": 201, "ordinal": 39}, "kind": "upload_observed", "previous": "4612cd97db7d6aebd73cf1f6f396880c08f58613a136ed8bfa1e3b3e61e4b4a1", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 197, "sha256": "1d9fbd3f56c3ad63c2f9d19aa77802669e5c32464eda6869631c8cd8f0408d21"} +{"at_utc": "2026-10-18T12:56:20.703+00:00", "data": {"cohort": "loss", "request_id": "upload-d008622649b04e9f871b3bd2296cc053", "sha256": "df0f390b9cdaa5d6b0c69e1b438912bcc94ada13e9b0083fdde37bc743b69e09", "size": 1045}, "kind": "sent", "previous": "1d9fbd3f56c3ad63c2f9d19aa77802669e5c32464eda6869631c8cd8f0408d21", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 198, "sha256": "57d846d3e2c85185707854e85d1a1d97fa3bd1cb271cca0316ba2f7460589fff"} +{"at_utc": "2026-10-18T12:56:28.515+00:00", "data": {"http_status": 201, "ids": {"documentId": 42, "documentVersionId": 42, "embeddingJobId": 40, "fileObjectId": 40}, "outcome": "ACKNOWLEDGED", "request_id": "upload-d008622649b04e9f871b3bd2296cc053"}, "kind": "response", "previous": "57d846d3e2c85185707854e85d1a1d97fa3bd1cb271cca0316ba2f7460589fff", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 199, "sha256": "3098ab42525daa05019eed2b4d59f108bb26064d9fc2dde2de63e89222f65613"} +{"at_utc": "2026-10-18T12:56:28.522+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.813, "http_status": 201, "ordinal": 40}, "kind": "upload_observed", "previous": "3098ab42525daa05019eed2b4d59f108bb26064d9fc2dde2de63e89222f65613", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 200, "sha256": "cda1004f7aa0144627a620486eaa25b8c3dfa333996ebf613444991ef31e702b"} +{"at_utc": "2026-10-18T12:56:28.532+00:00", "data": {"cohort": "loss", "request_id": "upload-d3b7eec6c2394e2495f06d66006c6f00", "sha256": "cfed5717eb7e18c65982136a31b8663eef977b9967baf0c3407d3f13239035b3", "size": 639}, "kind": "sent", "previous": "cda1004f7aa0144627a620486eaa25b8c3dfa333996ebf613444991ef31e702b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 201, "sha256": "623df4e92c69c6757b56a8eda9e207bf0a49cbbe99ba18392447da4a65739ad2"} +{"at_utc": "2026-10-18T12:56:36.324+00:00", "data": {"http_status": 201, "ids": {"documentId": 43, "documentVersionId": 43, "embeddingJobId": 41, "fileObjectId": 41}, "outcome": "ACKNOWLEDGED", "request_id": "upload-d3b7eec6c2394e2495f06d66006c6f00"}, "kind": "response", "previous": "623df4e92c69c6757b56a8eda9e207bf0a49cbbe99ba18392447da4a65739ad2", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 202, "sha256": "07a6fcf43efd9cc1f6522a4bfe0c23f64af37ae27c38472fa99be6bfe138f3d7"} +{"at_utc": "2026-10-18T12:56:36.331+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.793, "http_status": 201, "ordinal": 41}, "kind": "upload_observed", "previous": "07a6fcf43efd9cc1f6522a4bfe0c23f64af37ae27c38472fa99be6bfe138f3d7", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 203, "sha256": "4bac2fbafa46f65f115a690cb4ec4d9880a0d71a68813a4349fb86118d0d920e"} +{"at_utc": "2026-10-18T12:56:36.340+00:00", "data": {"cohort": "loss", "request_id": "upload-665e96a9ada843efa78379b0e921ec48", "sha256": "32cf92e6f18e615ea091b1beb7694cb399ed4b136f08846395c60f8575867572", "size": 1045}, "kind": "sent", "previous": "4bac2fbafa46f65f115a690cb4ec4d9880a0d71a68813a4349fb86118d0d920e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 204, "sha256": "d4ef7378fcf14da5360924871f1ab33607fc1c4dfa22b2ceb9d2aaf0ce288fb2"} +{"at_utc": "2026-10-18T12:56:44.441+00:00", "data": {"http_status": 201, "ids": {"documentId": 44, "documentVersionId": 44, "embeddingJobId": 42, "fileObjectId": 42}, "outcome": "ACKNOWLEDGED", "request_id": "upload-665e96a9ada843efa78379b0e921ec48"}, "kind": "response", "previous": "d4ef7378fcf14da5360924871f1ab33607fc1c4dfa22b2ceb9d2aaf0ce288fb2", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 205, "sha256": "b472085a81693ad53d3f6b8e84a2801fa97db80781930a299efe2f6ba7dafff1"} +{"at_utc": "2026-10-18T12:56:44.448+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.1, "http_status": 201, "ordinal": 42}, "kind": "upload_observed", "previous": "b472085a81693ad53d3f6b8e84a2801fa97db80781930a299efe2f6ba7dafff1", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 206, "sha256": "e339b172b8bd3fcfc7f54c3a22f2d2edc08072bc4527170b9c2cd20786e29c2d"} +{"at_utc": "2026-10-18T12:56:44.457+00:00", "data": {"cohort": "loss", "request_id": "upload-046c77ba2e4a4b88ad0d1c6f7bf11bb9", "sha256": "71936f694c856a446c6d291fee8592f6d00e0ec41782c14c9b2c43c78c99911f", "size": 639}, "kind": "sent", "previous": "e339b172b8bd3fcfc7f54c3a22f2d2edc08072bc4527170b9c2cd20786e29c2d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 207, "sha256": "aa73449e85786133b3649d0f333e15c6931588061c333bf4f3340dc4daa1871d"} +{"at_utc": "2026-10-18T12:56:51.804+00:00", "data": {"http_status": 201, "ids": {"documentId": 45, "documentVersionId": 45, "embeddingJobId": 43, "fileObjectId": 43}, "outcome": "ACKNOWLEDGED", "request_id": "upload-046c77ba2e4a4b88ad0d1c6f7bf11bb9"}, "kind": "response", "previous": "aa73449e85786133b3649d0f333e15c6931588061c333bf4f3340dc4daa1871d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 208, "sha256": "fa7b3759b79274e2952781f6bed0626c98be3ea131a4013a81dd55185c3ae1f6"} +{"at_utc": "2026-10-18T12:56:51.812+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.347, "http_status": 201, "ordinal": 43}, "kind": "upload_observed", "previous": "fa7b3759b79274e2952781f6bed0626c98be3ea131a4013a81dd55185c3ae1f6", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 209, "sha256": "b2a05585639b1cd65b1cd4a4f4b9c6a7708d0aaba1cf947cc6e17c76d917af60"} +{"at_utc": "2026-10-18T12:56:51.820+00:00", "data": {"cohort": "loss", "request_id": "upload-607f1c2f64d64fb7b6632acc5f472d59", "sha256": "602c72f4ccbba0a7902589bdb314791cdda40ab0bc2b955a3cc08de13b634fda", "size": 1045}, "kind": "sent", "previous": "b2a05585639b1cd65b1cd4a4f4b9c6a7708d0aaba1cf947cc6e17c76d917af60", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 210, "sha256": "e19f5dab085fb3f55c743787937913fb9bdc85161c7b018499f419782ee278ae"} +{"at_utc": "2026-10-18T12:56:59.481+00:00", "data": {"http_status": 201, "ids": {"documentId": 46, "documentVersionId": 46, "embeddingJobId": 44, "fileObjectId": 44}, "outcome": "ACKNOWLEDGED", "request_id": "upload-607f1c2f64d64fb7b6632acc5f472d59"}, "kind": "response", "previous": "e19f5dab085fb3f55c743787937913fb9bdc85161c7b018499f419782ee278ae", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 211, "sha256": "bebd684bb2e6a2d76917e8573dfb46c66aeae0532262bec529d8ece0e001b37c"} +{"at_utc": "2026-10-18T12:56:59.497+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.662, "http_status": 201, "ordinal": 44}, "kind": "upload_observed", "previous": "bebd684bb2e6a2d76917e8573dfb46c66aeae0532262bec529d8ece0e001b37c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 212, "sha256": "e30d0e896aae11ba17c9b692930a19ff6e73c058b4757e87517c935df33ecfba"} +{"at_utc": "2026-10-18T12:56:59.508+00:00", "data": {"cohort": "loss", "request_id": "upload-3ff63ce9c7b4402fbcbee28935a5d414", "sha256": "b056a7db567d451a920fdbf7ccb00d57ed6e663ff4ab33a723354823ffdf0602", "size": 639}, "kind": "sent", "previous": "e30d0e896aae11ba17c9b692930a19ff6e73c058b4757e87517c935df33ecfba", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 213, "sha256": "cb200a57e1739bc7641fd5babe6f054dafaeaf04eef0dac4dddef7457cce6b90"} +{"at_utc": "2026-10-18T12:57:08.320+00:00", "data": {"http_status": 201, "ids": {"documentId": 47, "documentVersionId": 47, "embeddingJobId": 45, "fileObjectId": 45}, "outcome": "ACKNOWLEDGED", "request_id": "upload-3ff63ce9c7b4402fbcbee28935a5d414"}, "kind": "response", "previous": "cb200a57e1739bc7641fd5babe6f054dafaeaf04eef0dac4dddef7457cce6b90", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 214, "sha256": "88b3e9798f055ca5b8881ee5e00d556e82b68a216fda27329fc7f86d301c141b"} +{"at_utc": "2026-10-18T12:57:08.326+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.812, "http_status": 201, "ordinal": 45}, "kind": "upload_observed", "previous": "88b3e9798f055ca5b8881ee5e00d556e82b68a216fda27329fc7f86d301c141b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 215, "sha256": "c4be3692c959ea4505dae4b8e42162f7b8e02abf612df5b65b73f5fabdac71e0"} +{"at_utc": "2026-10-18T12:57:08.333+00:00", "data": {"cohort": "loss", "request_id": "upload-3a2ac04a9ac346d2b292f89ba0b096b3", "sha256": "3978a9d63e633c2b1cfa2514a6e4ea14839c757483acf2fa8017babdd62b431b", "size": 1045}, "kind": "sent", "previous": "c4be3692c959ea4505dae4b8e42162f7b8e02abf612df5b65b73f5fabdac71e0", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 216, "sha256": "7ef1bdf47e93f63df7c490f1743de91df13fa882482b3361ab532b1658cf192a"} +{"at_utc": "2026-10-18T12:57:16.165+00:00", "data": {"http_status": 201, "ids": {"documentId": 48, "documentVersionId": 48, "embeddingJobId": 46, "fileObjectId": 46}, "outcome": "ACKNOWLEDGED", "request_id": "upload-3a2ac04a9ac346d2b292f89ba0b096b3"}, "kind": "response", "previous": "7ef1bdf47e93f63df7c490f1743de91df13fa882482b3361ab532b1658cf192a", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 217, "sha256": "dae53251410c0ffd6833c92bb4949fcd1746bcaf216d09b12d0c30b70d444e20"} +{"at_utc": "2026-10-18T12:57:16.173+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.832, "http_status": 201, "ordinal": 46}, "kind": "upload_observed", "previous": "dae53251410c0ffd6833c92bb4949fcd1746bcaf216d09b12d0c30b70d444e20", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 218, "sha256": "fe466c953659988430e543b5137d96923acb43be52b1861dc1d245c1fce3b54d"} +{"at_utc": "2026-10-18T12:57:16.182+00:00", "data": {"cohort": "loss", "request_id": "upload-24735b8cf98347f091b106994e210a7b", "sha256": "aa206070315b0964462c62379da5396667d4b96260c3e89d9f73a7dff03f59a1", "size": 639}, "kind": "sent", "previous": "fe466c953659988430e543b5137d96923acb43be52b1861dc1d245c1fce3b54d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 219, "sha256": "51e59fc83d232014b09862dc4d2d1751f38eb141538f689ec88060ff3e56ea98"} +{"at_utc": "2026-10-18T12:57:24.035+00:00", "data": {"http_status": 201, "ids": {"documentId": 49, "documentVersionId": 49, "embeddingJobId": 47, "fileObjectId": 47}, "outcome": "ACKNOWLEDGED", "request_id": "upload-24735b8cf98347f091b106994e210a7b"}, "kind": "response", "previous": "51e59fc83d232014b09862dc4d2d1751f38eb141538f689ec88060ff3e56ea98", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 220, "sha256": "6a1a3a4656d2d9000e7d88b9a0f8c0396cb5429e78ab702164137e273796ac3b"} +{"at_utc": "2026-10-18T12:57:24.041+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.853, "http_status": 201, "ordinal": 47}, "kind": "upload_observed", "previous": "6a1a3a4656d2d9000e7d88b9a0f8c0396cb5429e78ab702164137e273796ac3b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 221, "sha256": "66c884405c440cac3b392bc4f21608d51b37d0148f7282b483a50d1bf1fbba89"} +{"at_utc": "2026-10-18T12:57:24.050+00:00", "data": {"cohort": "loss", "request_id": "upload-b0e8c7e2aea24562b5813170fa6a816d", "sha256": "a6dcbc76355bcc39db7c2d9d126c7545b1916931348cb7275c6233aa8489a8a9", "size": 1045}, "kind": "sent", "previous": "66c884405c440cac3b392bc4f21608d51b37d0148f7282b483a50d1bf1fbba89", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 222, "sha256": "be46507ccc75a887874398a3a09fedc4d4766a7abc810e29da2e2ca80179cbc5"} +{"at_utc": "2026-10-18T12:57:31.231+00:00", "data": {"http_status": 201, "ids": {"documentId": 50, "documentVersionId": 50, "embeddingJobId": 48, "fileObjectId": 48}, "outcome": "ACKNOWLEDGED", "request_id": "upload-b0e8c7e2aea24562b5813170fa6a816d"}, "kind": "response", "previous": "be46507ccc75a887874398a3a09fedc4d4766a7abc810e29da2e2ca80179cbc5", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 223, "sha256": "e22c11bfaf00c99f6212960cf6d3305f07d7a818cbc957e65003b0becc00d365"} +{"at_utc": "2026-10-18T12:57:31.239+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.182, "http_status": 201, "ordinal": 48}, "kind": "upload_observed", "previous": "e22c11bfaf00c99f6212960cf6d3305f07d7a818cbc957e65003b0becc00d365", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 224, "sha256": "02d091053aff69b290e8f2c2cc1684bf5293817b0cfb561f464f91f4a7b0db54"} +{"at_utc": "2026-10-18T12:57:31.249+00:00", "data": {"cohort": "loss", "request_id": "upload-04233cdad305433abee26af93d8356e6", "sha256": "2adfe330ad106f99f0a774e0582caa3d56731285a6202b2bb3bc24724f830938", "size": 639}, "kind": "sent", "previous": "02d091053aff69b290e8f2c2cc1684bf5293817b0cfb561f464f91f4a7b0db54", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 225, "sha256": "0194faa5b47fbf840c265af67c91c6def820e8358fd669a597d4e9f76b103061"} +{"at_utc": "2026-10-18T12:57:38.977+00:00", "data": {"http_status": 201, "ids": {"documentId": 51, "documentVersionId": 51, "embeddingJobId": 49, "fileObjectId": 49}, "outcome": "ACKNOWLEDGED", "request_id": "upload-04233cdad305433abee26af93d8356e6"}, "kind": "response", "previous": "0194faa5b47fbf840c265af67c91c6def820e8358fd669a597d4e9f76b103061", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 226, "sha256": "e89ec3f1e87bdfea2557b9379a82d980f3a78ed0cb4fd36efedae0a4cfd5a5fd"} +{"at_utc": "2026-10-18T12:57:38.981+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.727, "http_status": 201, "ordinal": 49}, "kind": "upload_observed", "previous": "e89ec3f1e87bdfea2557b9379a82d980f3a78ed0cb4fd36efedae0a4cfd5a5fd", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 227, "sha256": "1b884e6f3212c68405f8330e767664d84f046401c57017ba952c14b79a375b2e"} +{"at_utc": "2026-10-18T12:57:38.986+00:00", "data": {"cohort": "loss", "request_id": "upload-3697e70376084b83a4129e6cc3d6d33f", "sha256": "c60d6cf690c9e6ca2455ee2124f5b96bb59320da7a327d49b4fb5ddc1664f5fe", "size": 1045}, "kind": "sent", "previous": "1b884e6f3212c68405f8330e767664d84f046401c57017ba952c14b79a375b2e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 228, "sha256": "806fbbcc2eb6e703761950a6747d27d4be1610acb685897731cd4ad32bbba450"} +{"at_utc": "2026-10-18T12:57:47.528+00:00", "data": {"http_status": 201, "ids": {"documentId": 52, "documentVersionId": 52, "embeddingJobId": 50, "fileObjectId": 50}, "outcome": "ACKNOWLEDGED", "request_id": "upload-3697e70376084b83a4129e6cc3d6d33f"}, "kind": "response", "previous": "806fbbcc2eb6e703761950a6747d27d4be1610acb685897731cd4ad32bbba450", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 229, "sha256": "ef10167f36d7902d8aba3d53ea89ba9b7f0836001ace9fc9fef3d4cd246086d9"} +{"at_utc": "2026-10-18T12:57:47.535+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.543, "http_status": 201, "ordinal": 50}, "kind": "upload_observed", "previous": "ef10167f36d7902d8aba3d53ea89ba9b7f0836001ace9fc9fef3d4cd246086d9", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 230, "sha256": "0ebbe4e1711d4117d4cc040f45af9e9d39093a2c58bb3a0d39e5a3e89c4452f1"} +{"at_utc": "2026-10-18T12:57:47.545+00:00", "data": {"cohort": "loss", "request_id": "upload-23e0f579e82146679e7c3a58a1605aba", "sha256": "20fbdeab3b3b91276cb3300ce4b5b414c8f75b4d2d218bd92f446476188cc24c", "size": 639}, "kind": "sent", "previous": "0ebbe4e1711d4117d4cc040f45af9e9d39093a2c58bb3a0d39e5a3e89c4452f1", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 231, "sha256": "fe4a88df30ee5778b7eab8d5b57cf0656c776a9cad37949f631f268c3048a5d7"} +{"at_utc": "2026-10-18T12:57:56.017+00:00", "data": {"http_status": 201, "ids": {"documentId": 53, "documentVersionId": 53, "embeddingJobId": 51, "fileObjectId": 51}, "outcome": "ACKNOWLEDGED", "request_id": "upload-23e0f579e82146679e7c3a58a1605aba"}, "kind": "response", "previous": "fe4a88df30ee5778b7eab8d5b57cf0656c776a9cad37949f631f268c3048a5d7", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 232, "sha256": "c602f091845cc1691a1612945480d89c42bed63e9d3fef783c49abd9b5b73f3d"} +{"at_utc": "2026-10-18T12:57:56.024+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.472, "http_status": 201, "ordinal": 51}, "kind": "upload_observed", "previous": "c602f091845cc1691a1612945480d89c42bed63e9d3fef783c49abd9b5b73f3d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 233, "sha256": "bab674233554679c99b2caca6f2737e2fca010b8164b9928293bea19ddc71183"} +{"at_utc": "2026-10-18T12:57:56.033+00:00", "data": {"cohort": "loss", "request_id": "upload-cf72758cb0624f18a5632f23c4419586", "sha256": "077ca2cd195eca9abb971513cb7cae2129b4ae233ac03adf4cac21fb6330c34e", "size": 1045}, "kind": "sent", "previous": "bab674233554679c99b2caca6f2737e2fca010b8164b9928293bea19ddc71183", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 234, "sha256": "fcd4ac38b4c65aa401fd80449ea3880ead309e575f598be3dff0d2c95ddb317f"} +{"at_utc": "2026-10-18T12:58:03.981+00:00", "data": {"http_status": 201, "ids": {"documentId": 54, "documentVersionId": 54, "embeddingJobId": 52, "fileObjectId": 52}, "outcome": "ACKNOWLEDGED", "request_id": "upload-cf72758cb0624f18a5632f23c4419586"}, "kind": "response", "previous": "fcd4ac38b4c65aa401fd80449ea3880ead309e575f598be3dff0d2c95ddb317f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 235, "sha256": "3c49e8c0cb66b3e9db42438f378c5a213526d38097a1435b7b63d1878104ceb2"} +{"at_utc": "2026-10-18T12:58:03.990+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.949, "http_status": 201, "ordinal": 52}, "kind": "upload_observed", "previous": "3c49e8c0cb66b3e9db42438f378c5a213526d38097a1435b7b63d1878104ceb2", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 236, "sha256": "7e6c1b56f5ea8a6c5e6559d4e2deb9c6815963f0790201396222bc0f44ce1a73"} +{"at_utc": "2026-10-18T12:58:04.000+00:00", "data": {"cohort": "loss", "request_id": "upload-29c885e4f88d4e57aa18663fad3b240f", "sha256": "0b20c2871e345692b751a8e04705ddbced533266963b8f9a7b89fa678cfc1776", "size": 639}, "kind": "sent", "previous": "7e6c1b56f5ea8a6c5e6559d4e2deb9c6815963f0790201396222bc0f44ce1a73", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 237, "sha256": "5cf3dff987b3a39d5cb3abc02ea97489225bf14bd1fdf4a1c1640433fdb89b5f"} +{"at_utc": "2026-10-18T12:58:11.999+00:00", "data": {"http_status": 201, "ids": {"documentId": 55, "documentVersionId": 55, "embeddingJobId": 53, "fileObjectId": 53}, "outcome": "ACKNOWLEDGED", "request_id": "upload-29c885e4f88d4e57aa18663fad3b240f"}, "kind": "response", "previous": "5cf3dff987b3a39d5cb3abc02ea97489225bf14bd1fdf4a1c1640433fdb89b5f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 238, "sha256": "affd8a1f9b60e4a4c1a8d1a217cec85615ca1e795ece5f70b44d1099c10a6b23"} +{"at_utc": "2026-10-18T12:58:12.002+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.999, "http_status": 201, "ordinal": 53}, "kind": "upload_observed", "previous": "affd8a1f9b60e4a4c1a8d1a217cec85615ca1e795ece5f70b44d1099c10a6b23", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 239, "sha256": "a2d2a554f34687d42467f5675de06981e34c64aff872296dc28b4eb1b315c4d2"} +{"at_utc": "2026-10-18T12:58:12.007+00:00", "data": {"cohort": "loss", "request_id": "upload-0ac416309fb24af199ca7e931bb45a7b", "sha256": "394d49de7b07c54a9b04801b1f57dbe1549ec0ec93f6048a831a1d3bf6daa923", "size": 1046}, "kind": "sent", "previous": "a2d2a554f34687d42467f5675de06981e34c64aff872296dc28b4eb1b315c4d2", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 240, "sha256": "d2d64e48c1ec0ba90444d9fa276d9391f6de2bb899b7a7398afe62e2fbd99af2"} +{"at_utc": "2026-10-18T12:58:19.128+00:00", "data": {"http_status": 201, "ids": {"documentId": 56, "documentVersionId": 56, "embeddingJobId": 54, "fileObjectId": 54}, "outcome": "ACKNOWLEDGED", "request_id": "upload-0ac416309fb24af199ca7e931bb45a7b"}, "kind": "response", "previous": "d2d64e48c1ec0ba90444d9fa276d9391f6de2bb899b7a7398afe62e2fbd99af2", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 241, "sha256": "6aa3fce65574d6438d10f26a88d8b14a664e8216648efa3f48c317965584d0c1"} +{"at_utc": "2026-10-18T12:58:19.134+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.121, "http_status": 201, "ordinal": 54}, "kind": "upload_observed", "previous": "6aa3fce65574d6438d10f26a88d8b14a664e8216648efa3f48c317965584d0c1", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 242, "sha256": "e9f87775ea6e504c497574d7ff600d9aa0f5613bb7367f5926a0eddabcb14df0"} +{"at_utc": "2026-10-18T12:58:19.146+00:00", "data": {"cohort": "loss", "request_id": "upload-1daf93dff41c468f85bfdbbe70d6fcc0", "sha256": "202241ea917fa4c556f9a12f7f4828bdb13243c693ecc7602955653ee1ae0afc", "size": 639}, "kind": "sent", "previous": "e9f87775ea6e504c497574d7ff600d9aa0f5613bb7367f5926a0eddabcb14df0", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 243, "sha256": "3b08c72af309f8e278327a1e1150925bfb690c25384512618fb9e15c705df8f4"} +{"at_utc": "2026-10-18T12:58:27.247+00:00", "data": {"http_status": 201, "ids": {"documentId": 57, "documentVersionId": 57, "embeddingJobId": 55, "fileObjectId": 55}, "outcome": "ACKNOWLEDGED", "request_id": "upload-1daf93dff41c468f85bfdbbe70d6fcc0"}, "kind": "response", "previous": "3b08c72af309f8e278327a1e1150925bfb690c25384512618fb9e15c705df8f4", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 244, "sha256": "2f42345195de7d49f2dfe11c0addefdc8861d12c9822e61b607d6933e5497934"} +{"at_utc": "2026-10-18T12:58:27.256+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.101, "http_status": 201, "ordinal": 55}, "kind": "upload_observed", "previous": "2f42345195de7d49f2dfe11c0addefdc8861d12c9822e61b607d6933e5497934", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 245, "sha256": "a0a4688c0be30274f356556de43a38344687fafc3187db40e6dd5eb329e8df34"} +{"at_utc": "2026-10-18T12:58:27.267+00:00", "data": {"cohort": "loss", "request_id": "upload-4f5d46c32ba04d24957cc1780db113b4", "sha256": "d4a894bed85f5339e721740e2163666e88d52743d15b4026953c242ad520c3ca", "size": 1045}, "kind": "sent", "previous": "a0a4688c0be30274f356556de43a38344687fafc3187db40e6dd5eb329e8df34", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 246, "sha256": "8500ed7caea3070ad7d3e8f07c1cdaa64d24b5f89c3a3f794b884791346ac2f6"} +{"at_utc": "2026-10-18T12:58:35.374+00:00", "data": {"http_status": 201, "ids": {"documentId": 58, "documentVersionId": 58, "embeddingJobId": 56, "fileObjectId": 56}, "outcome": "ACKNOWLEDGED", "request_id": "upload-4f5d46c32ba04d24957cc1780db113b4"}, "kind": "response", "previous": "8500ed7caea3070ad7d3e8f07c1cdaa64d24b5f89c3a3f794b884791346ac2f6", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 247, "sha256": "95a2e9098619274d4084fb283bdcca4602e9c028c40acc58349bccd405f32b61"} +{"at_utc": "2026-10-18T12:58:35.383+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.108, "http_status": 201, "ordinal": 56}, "kind": "upload_observed", "previous": "95a2e9098619274d4084fb283bdcca4602e9c028c40acc58349bccd405f32b61", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 248, "sha256": "49781783ab281369079738ed137ff25619ddf4dcb33ed4c903c799555788c174"} +{"at_utc": "2026-10-18T12:58:37.862+00:00", "data": {"cohort": "loss", "request_id": "upload-e1cbe18e321045978d12d708efb76b72", "sha256": "636cabda006f3854054325b933df56c549af856d1b7287ddbfd23f02d7558463", "size": 639}, "kind": "sent", "previous": "49781783ab281369079738ed137ff25619ddf4dcb33ed4c903c799555788c174", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 249, "sha256": "cfc94dfeb49328837bebdde52de326c8f94aa1136394aae9fb50747125d00267"} +{"at_utc": "2026-10-18T12:58:46.615+00:00", "data": {"http_status": 201, "ids": {"documentId": 59, "documentVersionId": 59, "embeddingJobId": 57, "fileObjectId": 57}, "outcome": "ACKNOWLEDGED", "request_id": "upload-e1cbe18e321045978d12d708efb76b72"}, "kind": "response", "previous": "cfc94dfeb49328837bebdde52de326c8f94aa1136394aae9fb50747125d00267", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 250, "sha256": "909b19f2898b0093eeffaa378e9949bdf19fd1ca270608a026d9758f6f2828e6"} +{"at_utc": "2026-10-18T12:58:46.622+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.753, "http_status": 201, "ordinal": 57}, "kind": "upload_observed", "previous": "909b19f2898b0093eeffaa378e9949bdf19fd1ca270608a026d9758f6f2828e6", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 251, "sha256": "f3ab855d4b1723266f5cd59010c2b2e9e3fbcf925cba973df094e0e1c8aa4603"} +{"at_utc": "2026-10-18T12:58:48.862+00:00", "data": {"cohort": "loss", "request_id": "upload-e871accf911d4c9c9c1345f38362122b", "sha256": "5428bf31e08c974068911906aa9eb2bedcaacf95e032e1cfbe7244fbbf95514e", "size": 1045}, "kind": "sent", "previous": "f3ab855d4b1723266f5cd59010c2b2e9e3fbcf925cba973df094e0e1c8aa4603", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 252, "sha256": "02ad88638a99d32eb4ebe20b51b51a3c699fe287ce4d2a0ea53f45d97ec4391e"} +{"at_utc": "2026-10-18T12:58:56.309+00:00", "data": {"http_status": 201, "ids": {"documentId": 60, "documentVersionId": 60, "embeddingJobId": 58, "fileObjectId": 58}, "outcome": "ACKNOWLEDGED", "request_id": "upload-e871accf911d4c9c9c1345f38362122b"}, "kind": "response", "previous": "02ad88638a99d32eb4ebe20b51b51a3c699fe287ce4d2a0ea53f45d97ec4391e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 253, "sha256": "98c31be127c26ae493cbb85ba6fcaa1bfdf616014b2ee6cf422fb2ff4df28f29"} +{"at_utc": "2026-10-18T12:58:56.316+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.446, "http_status": 201, "ordinal": 58}, "kind": "upload_observed", "previous": "98c31be127c26ae493cbb85ba6fcaa1bfdf616014b2ee6cf422fb2ff4df28f29", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 254, "sha256": "dbf8f2e31049895110c542a7e10ce91ce9cb7b42f404a8306fa2c631975efcf5"} +{"at_utc": "2026-10-18T12:58:59.860+00:00", "data": {"cohort": "loss", "request_id": "upload-04c4842d79894d72903e7b423e0d593a", "sha256": "3a39dcb938ccaa58f17bb27ea56d5c6e905607193b2ad872b2b3bd5005e3a26f", "size": 639}, "kind": "sent", "previous": "dbf8f2e31049895110c542a7e10ce91ce9cb7b42f404a8306fa2c631975efcf5", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 255, "sha256": "225862deac6de07af3ff1a47af5dfa1bd9497da30d36508b2aa91678e798612f"} +{"at_utc": "2026-10-18T12:59:07.914+00:00", "data": {"http_status": 201, "ids": {"documentId": 61, "documentVersionId": 61, "embeddingJobId": 59, "fileObjectId": 59}, "outcome": "ACKNOWLEDGED", "request_id": "upload-04c4842d79894d72903e7b423e0d593a"}, "kind": "response", "previous": "225862deac6de07af3ff1a47af5dfa1bd9497da30d36508b2aa91678e798612f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 256, "sha256": "6c90592e8ae72b33f99be5bb18ad484a024138f8241ccc8029db0c9b66cd2b9d"} +{"at_utc": "2026-10-18T12:59:07.918+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.053, "http_status": 201, "ordinal": 59}, "kind": "upload_observed", "previous": "6c90592e8ae72b33f99be5bb18ad484a024138f8241ccc8029db0c9b66cd2b9d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 257, "sha256": "9e9ecca0757f3787122e2a18d1e5dbb73e5608c37851734a518019690c22e619"} +{"at_utc": "2026-10-18T12:59:10.861+00:00", "data": {"cohort": "loss", "request_id": "upload-09b3714d1ed542ecad7affd1258265e4", "sha256": "b2dd6bb46d8bc4eaa85c4b896744b8cb5203cef0a46d978a49b346aec7a21d3d", "size": 1045}, "kind": "sent", "previous": "9e9ecca0757f3787122e2a18d1e5dbb73e5608c37851734a518019690c22e619", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 258, "sha256": "5281c6fefa7d411e58ffa8e1077354041fc15de79c671f51797f254acf32a1b9"} +{"at_utc": "2026-10-18T12:59:18.865+00:00", "data": {"http_status": 201, "ids": {"documentId": 62, "documentVersionId": 62, "embeddingJobId": 60, "fileObjectId": 60}, "outcome": "ACKNOWLEDGED", "request_id": "upload-09b3714d1ed542ecad7affd1258265e4"}, "kind": "response", "previous": "5281c6fefa7d411e58ffa8e1077354041fc15de79c671f51797f254acf32a1b9", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 259, "sha256": "f129d8ce25b74bad3af700f8af4f51201831e40fae27639326936f5699e635f9"} +{"at_utc": "2026-10-18T12:59:18.873+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.003, "http_status": 201, "ordinal": 60}, "kind": "upload_observed", "previous": "f129d8ce25b74bad3af700f8af4f51201831e40fae27639326936f5699e635f9", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 260, "sha256": "4e91f55bb81e218efc88ec2e11868053100de29873eca2ec745d086fcb62a13f"} +{"at_utc": "2026-10-18T12:59:18.878+00:00", "data": {"completed_upload_requests": 60, "ordinal": 2, "scheduled_at_utc": "2026-10-18T12:59:16.828378+00:00", "start_lag_seconds": 2.033}, "kind": "offsite_schedule_observed", "previous": "4e91f55bb81e218efc88ec2e11868053100de29873eca2ec745d086fcb62a13f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 261, "sha256": "1192317335c9c6aa336c15e372d89773ad59d2ed4f086bd1cd5c370112df6d8d"} +{"at_utc": "2026-10-18T12:59:44.536+00:00", "data": {"fileObjectId": 34, "generation": "1792328384031780", "sha256": "2b2330ff3f21d26f156091669add4e90615d0364196fa79bb1e6b43ad9d7b621", "size": 1045}, "kind": "original_preserved", "previous": "1192317335c9c6aa336c15e372d89773ad59d2ed4f086bd1cd5c370112df6d8d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 262, "sha256": "3cf0361d52a5346281142e7cef8b24d0df17a7bb7a0d1ca3e678c1c79b3a3bc8"} +{"at_utc": "2026-10-18T12:59:45.916+00:00", "data": {"fileObjectId": 35, "generation": "1792328385296471", "sha256": "33cff912087abdd6949f053df3efd3ee4cc3dc9f4410a9d610186511bf56a8da", "size": 639}, "kind": "original_preserved", "previous": "3cf0361d52a5346281142e7cef8b24d0df17a7bb7a0d1ca3e678c1c79b3a3bc8", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 263, "sha256": "f0630a78c8df8c9b1ea4eef23dccf10a73af0eca6aeb92630b8b30d74de2439b"} +{"at_utc": "2026-10-18T12:59:47.418+00:00", "data": {"fileObjectId": 36, "generation": "1792328386910770", "sha256": "bb8222fa45c741a91a26739414b68b4cab720cc2d938b0075016bc1627de5962", "size": 1045}, "kind": "original_preserved", "previous": "f0630a78c8df8c9b1ea4eef23dccf10a73af0eca6aeb92630b8b30d74de2439b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 264, "sha256": "551328bd47a08f6d7220266bcea02b4f678a75a85e1603a8b043d316c2845d46"} +{"at_utc": "2026-10-18T12:59:48.619+00:00", "data": {"fileObjectId": 37, "generation": "1792328388171503", "sha256": "09e6f22bc526b38bf65db51e22bdcd4c9c6ab2679644ab17a21b07cb6237e6dc", "size": 639}, "kind": "original_preserved", "previous": "551328bd47a08f6d7220266bcea02b4f678a75a85e1603a8b043d316c2845d46", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 265, "sha256": "0d9f7065547c70790083a14e08631ba171fee957ed0d98eed1493875f4c1ab9b"} +{"at_utc": "2026-10-18T12:59:50.010+00:00", "data": {"fileObjectId": 38, "generation": "1792328389485824", "sha256": "c6d0c699ab27b05a52f159dcd884648f6b7a504b1fe203139e47570379b634f6", "size": 1045}, "kind": "original_preserved", "previous": "0d9f7065547c70790083a14e08631ba171fee957ed0d98eed1493875f4c1ab9b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 266, "sha256": "4cdd1f4dbbf11511839f83072685c08d28ee067783a4070be4219a39f3416265"} +{"at_utc": "2026-10-18T12:59:51.209+00:00", "data": {"fileObjectId": 39, "generation": "1792328390799793", "sha256": "848b879592449d6780759eb458312a4118a971e0896bbbddb44d84fa7cf4cf42", "size": 639}, "kind": "original_preserved", "previous": "4cdd1f4dbbf11511839f83072685c08d28ee067783a4070be4219a39f3416265", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 267, "sha256": "4d7469bae9c7d9f9049afac7b0fa05016e3c2b6d2713b1224487add1b4d02205"} +{"at_utc": "2026-10-18T12:59:52.481+00:00", "data": {"fileObjectId": 40, "generation": "1792328392073698", "sha256": "df0f390b9cdaa5d6b0c69e1b438912bcc94ada13e9b0083fdde37bc743b69e09", "size": 1045}, "kind": "original_preserved", "previous": "4d7469bae9c7d9f9049afac7b0fa05016e3c2b6d2713b1224487add1b4d02205", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 268, "sha256": "4924be6cf5c255d11420e328390555085f69f61bbd0fde783187d195c55a4200"} +{"at_utc": "2026-10-18T12:59:53.725+00:00", "data": {"fileObjectId": 41, "generation": "1792328393237369", "sha256": "cfed5717eb7e18c65982136a31b8663eef977b9967baf0c3407d3f13239035b3", "size": 639}, "kind": "original_preserved", "previous": "4924be6cf5c255d11420e328390555085f69f61bbd0fde783187d195c55a4200", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 269, "sha256": "4e55c29f6bdeaa4640d0372fc477aa24b4f8a1de31919f51ac5d83ab8b8ca485"} +{"at_utc": "2026-10-18T12:59:54.883+00:00", "data": {"fileObjectId": 42, "generation": "1792328394464378", "sha256": "32cf92e6f18e615ea091b1beb7694cb399ed4b136f08846395c60f8575867572", "size": 1045}, "kind": "original_preserved", "previous": "4e55c29f6bdeaa4640d0372fc477aa24b4f8a1de31919f51ac5d83ab8b8ca485", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 270, "sha256": "824f832aff71444ab5ac1175e2888842704889f8a176cd0167b2062fbdaff3eb"} +{"at_utc": "2026-10-18T12:59:56.160+00:00", "data": {"fileObjectId": 43, "generation": "1792328395738784", "sha256": "71936f694c856a446c6d291fee8592f6d00e0ec41782c14c9b2c43c78c99911f", "size": 639}, "kind": "original_preserved", "previous": "824f832aff71444ab5ac1175e2888842704889f8a176cd0167b2062fbdaff3eb", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 271, "sha256": "64d4262dabb32fd9d14084ba07660d89e129dd2d45a6baf1fed24085f0403aa6"} +{"at_utc": "2026-10-18T12:59:57.397+00:00", "data": {"fileObjectId": 44, "generation": "1792328396953352", "sha256": "602c72f4ccbba0a7902589bdb314791cdda40ab0bc2b955a3cc08de13b634fda", "size": 1045}, "kind": "original_preserved", "previous": "64d4262dabb32fd9d14084ba07660d89e129dd2d45a6baf1fed24085f0403aa6", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 272, "sha256": "11ead8f2c0d4d858b3b78ed4e461a5de8c6cbc5b8035fdf5f913bb6c64f5c117"} +{"at_utc": "2026-10-18T12:59:58.689+00:00", "data": {"fileObjectId": 45, "generation": "1792328398174154", "sha256": "b056a7db567d451a920fdbf7ccb00d57ed6e663ff4ab33a723354823ffdf0602", "size": 639}, "kind": "original_preserved", "previous": "11ead8f2c0d4d858b3b78ed4e461a5de8c6cbc5b8035fdf5f913bb6c64f5c117", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 273, "sha256": "e75c47b5cb840ff5ec356d527c08e3ea9cb7d186adc8e47934e6f87e190381f6"} +{"at_utc": "2026-10-18T12:59:59.952+00:00", "data": {"fileObjectId": 46, "generation": "1792328399451108", "sha256": "3978a9d63e633c2b1cfa2514a6e4ea14839c757483acf2fa8017babdd62b431b", "size": 1045}, "kind": "original_preserved", "previous": "e75c47b5cb840ff5ec356d527c08e3ea9cb7d186adc8e47934e6f87e190381f6", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 274, "sha256": "99bafbb9be5dd22eec8d5cffe7f52a6427f5d09c500d6a4c6dc701a6b7ab4d75"} +{"at_utc": "2026-10-18T13:00:01.310+00:00", "data": {"fileObjectId": 47, "generation": "1792328400811689", "sha256": "aa206070315b0964462c62379da5396667d4b96260c3e89d9f73a7dff03f59a1", "size": 639}, "kind": "original_preserved", "previous": "99bafbb9be5dd22eec8d5cffe7f52a6427f5d09c500d6a4c6dc701a6b7ab4d75", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 275, "sha256": "17c346880e8b56501daf5ccf0150e5e12d5366ae1aac3f5da0ff5a63c793f5d2"} +{"at_utc": "2026-10-18T13:00:02.581+00:00", "data": {"fileObjectId": 48, "generation": "1792328402070421", "sha256": "a6dcbc76355bcc39db7c2d9d126c7545b1916931348cb7275c6233aa8489a8a9", "size": 1045}, "kind": "original_preserved", "previous": "17c346880e8b56501daf5ccf0150e5e12d5366ae1aac3f5da0ff5a63c793f5d2", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 276, "sha256": "4d3f41b4bb5c7cff7afc353c3907b2ecac39a7c7dd07a2deedced4d936a32689"} +{"at_utc": "2026-10-18T13:00:03.776+00:00", "data": {"fileObjectId": 49, "generation": "1792328403368873", "sha256": "2adfe330ad106f99f0a774e0582caa3d56731285a6202b2bb3bc24724f830938", "size": 639}, "kind": "original_preserved", "previous": "4d3f41b4bb5c7cff7afc353c3907b2ecac39a7c7dd07a2deedced4d936a32689", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 277, "sha256": "7e673688c54b85a12b0b089d24b944af3b25a8d220555c7e50d59a62cace4df1"} +{"at_utc": "2026-10-18T13:00:05.102+00:00", "data": {"fileObjectId": 50, "generation": "1792328404687457", "sha256": "c60d6cf690c9e6ca2455ee2124f5b96bb59320da7a327d49b4fb5ddc1664f5fe", "size": 1045}, "kind": "original_preserved", "previous": "7e673688c54b85a12b0b089d24b944af3b25a8d220555c7e50d59a62cace4df1", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 278, "sha256": "9baac7f17830a68a50a2a8c5396f702ca407d19a8d983f5190d7b44124764a46"} +{"at_utc": "2026-10-18T13:00:06.377+00:00", "data": {"fileObjectId": 51, "generation": "1792328405929490", "sha256": "20fbdeab3b3b91276cb3300ce4b5b414c8f75b4d2d218bd92f446476188cc24c", "size": 639}, "kind": "original_preserved", "previous": "9baac7f17830a68a50a2a8c5396f702ca407d19a8d983f5190d7b44124764a46", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 279, "sha256": "1436474bdaddeb096cb4df39f3cb3bac44b91d6c27f162fd79f498aa6945c90e"} +{"at_utc": "2026-10-18T13:00:07.637+00:00", "data": {"fileObjectId": 52, "generation": "1792328407203767", "sha256": "077ca2cd195eca9abb971513cb7cae2129b4ae233ac03adf4cac21fb6330c34e", "size": 1045}, "kind": "original_preserved", "previous": "1436474bdaddeb096cb4df39f3cb3bac44b91d6c27f162fd79f498aa6945c90e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 280, "sha256": "ab96e8ed88acd0e1df10a45051426c89d6cbbc2b5e4722a624204a60bc194333"} +{"at_utc": "2026-10-18T13:00:08.936+00:00", "data": {"fileObjectId": 53, "generation": "1792328408450431", "sha256": "0b20c2871e345692b751a8e04705ddbced533266963b8f9a7b89fa678cfc1776", "size": 639}, "kind": "original_preserved", "previous": "ab96e8ed88acd0e1df10a45051426c89d6cbbc2b5e4722a624204a60bc194333", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 281, "sha256": "a414cebeeca9a07a30dd8d3d2de0cf5b382ded76df8c6ea1229bc52c7e4aba5d"} +{"at_utc": "2026-10-18T13:00:10.269+00:00", "data": {"fileObjectId": 54, "generation": "1792328409828439", "sha256": "394d49de7b07c54a9b04801b1f57dbe1549ec0ec93f6048a831a1d3bf6daa923", "size": 1046}, "kind": "original_preserved", "previous": "a414cebeeca9a07a30dd8d3d2de0cf5b382ded76df8c6ea1229bc52c7e4aba5d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 282, "sha256": "1780dac63d01d10eb54d07833a83d43958446381d259288226b6bc3fc916d53e"} +{"at_utc": "2026-10-18T13:00:11.655+00:00", "data": {"fileObjectId": 55, "generation": "1792328411052970", "sha256": "202241ea917fa4c556f9a12f7f4828bdb13243c693ecc7602955653ee1ae0afc", "size": 639}, "kind": "original_preserved", "previous": "1780dac63d01d10eb54d07833a83d43958446381d259288226b6bc3fc916d53e", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 283, "sha256": "43bc636fb73699042b9b1b47399e58664bf9a9c7ec8872e29982f101deeee8e0"} +{"at_utc": "2026-10-18T13:00:13.074+00:00", "data": {"fileObjectId": 56, "generation": "1792328412581686", "sha256": "d4a894bed85f5339e721740e2163666e88d52743d15b4026953c242ad520c3ca", "size": 1045}, "kind": "original_preserved", "previous": "43bc636fb73699042b9b1b47399e58664bf9a9c7ec8872e29982f101deeee8e0", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 284, "sha256": "f3373704dbfb35e0a2661bf516ab62d215cc749af027e8dacf91d6bcbac423ab"} +{"at_utc": "2026-10-18T13:00:14.277+00:00", "data": {"fileObjectId": 57, "generation": "1792328413860674", "sha256": "636cabda006f3854054325b933df56c549af856d1b7287ddbfd23f02d7558463", "size": 639}, "kind": "original_preserved", "previous": "f3373704dbfb35e0a2661bf516ab62d215cc749af027e8dacf91d6bcbac423ab", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 285, "sha256": "a6eace225c757ccd0315bd2a66d727e0d9eec62a583de068952835b6a6fa2581"} +{"at_utc": "2026-10-18T13:00:15.556+00:00", "data": {"fileObjectId": 58, "generation": "1792328415146526", "sha256": "5428bf31e08c974068911906aa9eb2bedcaacf95e032e1cfbe7244fbbf95514e", "size": 1045}, "kind": "original_preserved", "previous": "a6eace225c757ccd0315bd2a66d727e0d9eec62a583de068952835b6a6fa2581", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 286, "sha256": "fb448efc5ed618cc697cb9c0c04258a5b360efc3079157b4f710887869a5766f"} +{"at_utc": "2026-10-18T13:00:17.078+00:00", "data": {"fileObjectId": 59, "generation": "1792328416443892", "sha256": "3a39dcb938ccaa58f17bb27ea56d5c6e905607193b2ad872b2b3bd5005e3a26f", "size": 639}, "kind": "original_preserved", "previous": "fb448efc5ed618cc697cb9c0c04258a5b360efc3079157b4f710887869a5766f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 287, "sha256": "e4365044f9834d7fef320ef43a9f7bc83d8f0ac58560a633cf87e7da7613ef75"} +{"at_utc": "2026-10-18T13:00:18.356+00:00", "data": {"fileObjectId": 60, "generation": "1792328417847807", "sha256": "b2dd6bb46d8bc4eaa85c4b896744b8cb5203cef0a46d978a49b346aec7a21d3d", "size": 1045}, "kind": "original_preserved", "previous": "e4365044f9834d7fef320ef43a9f7bc83d8f0ac58560a633cf87e7da7613ef75", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 288, "sha256": "778bd6148b45d9bd2455ecd75937cf18e0e26862856100e96dd4d79524bb4f40"} +{"at_utc": "2026-10-18T13:00:22.654+00:00", "data": {"generation": "1792328420319071", "name": "000000060000000000000024", "sha256": "e204570a27cdb84ec75831c7626e4e2874c0051683ed1180368ca7f3c40f8db7", "size": 16777216}, "kind": "wal_preserved", "previous": "778bd6148b45d9bd2455ecd75937cf18e0e26862856100e96dd4d79524bb4f40", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 289, "sha256": "127afa2e112f85031ff75b59d60d3d9420b85c8adcb68c14ed511f167262c84d"} +{"at_utc": "2026-10-18T13:00:22.660+00:00", "data": {"complete_wals": 8, "elapsed_seconds": 63.774, "ordinal": 2, "preserved_originals": 60}, "kind": "offsite_round_completed", "previous": "127afa2e112f85031ff75b59d60d3d9420b85c8adcb68c14ed511f167262c84d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 290, "sha256": "a34158330679a05e6fccc241ebf5fa43d95f782e7bd99cd1cf1b889f18ad864c"} +{"at_utc": "2026-10-18T13:00:22.673+00:00", "data": {"cohort": "loss", "request_id": "upload-66a683aeca184c1a9fef9e2c5430947e", "sha256": "517ee15eba97ef5b7106968530512d31ddd3c7006efe346f9d7c54221395fd31", "size": 639}, "kind": "sent", "previous": "a34158330679a05e6fccc241ebf5fa43d95f782e7bd99cd1cf1b889f18ad864c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 291, "sha256": "59758edacc4f0d16fd6cf867d880a1db22e47f87a4e453774ed3d73e248da693"} +{"at_utc": "2026-10-18T13:00:30.969+00:00", "data": {"http_status": 201, "ids": {"documentId": 63, "documentVersionId": 63, "embeddingJobId": 61, "fileObjectId": 61}, "outcome": "ACKNOWLEDGED", "request_id": "upload-66a683aeca184c1a9fef9e2c5430947e"}, "kind": "response", "previous": "59758edacc4f0d16fd6cf867d880a1db22e47f87a4e453774ed3d73e248da693", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 292, "sha256": "65100955d1a9ee63c1872dfee60b226e5acaf49eab0c0a34cb4f0d4596b63619"} +{"at_utc": "2026-10-18T13:00:30.975+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.296, "http_status": 201, "ordinal": 61}, "kind": "upload_observed", "previous": "65100955d1a9ee63c1872dfee60b226e5acaf49eab0c0a34cb4f0d4596b63619", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 293, "sha256": "88627826deac278d2b7aeb8d0b6d86c1ac08ab0dbc80a464216a0c199861ce22"} +{"at_utc": "2026-10-18T13:00:30.983+00:00", "data": {"cohort": "loss", "request_id": "upload-c278df614f444546967f1e1fa5ee1845", "sha256": "638affbdd36d67686c9515e48fabbe06685270a377adb22c3e26b0750e8bbffd", "size": 1045}, "kind": "sent", "previous": "88627826deac278d2b7aeb8d0b6d86c1ac08ab0dbc80a464216a0c199861ce22", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 294, "sha256": "11e0a012d6fe69cf88804dde2e5ee166499a9450296a3473790c50ae512acb55"} +{"at_utc": "2026-10-18T13:00:38.813+00:00", "data": {"http_status": 201, "ids": {"documentId": 64, "documentVersionId": 64, "embeddingJobId": 62, "fileObjectId": 62}, "outcome": "ACKNOWLEDGED", "request_id": "upload-c278df614f444546967f1e1fa5ee1845"}, "kind": "response", "previous": "11e0a012d6fe69cf88804dde2e5ee166499a9450296a3473790c50ae512acb55", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 295, "sha256": "b43282a9537a96a746edae8b131be42d2628a883325aa3bb4fc84a8e5d79d3f4"} +{"at_utc": "2026-10-18T13:00:38.821+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.83, "http_status": 201, "ordinal": 62}, "kind": "upload_observed", "previous": "b43282a9537a96a746edae8b131be42d2628a883325aa3bb4fc84a8e5d79d3f4", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 296, "sha256": "866e0d3f6a67cb6f414e042b8cedc85eedf15ece070de72946341c2b6e14e9d1"} +{"at_utc": "2026-10-18T13:00:38.832+00:00", "data": {"cohort": "loss", "request_id": "upload-24f9a22d31904c4585030acce87e0473", "sha256": "769e75df1b7fb735d177bc05bf127cf9c7618321579ad1c7d5602c9e6ae321a8", "size": 639}, "kind": "sent", "previous": "866e0d3f6a67cb6f414e042b8cedc85eedf15ece070de72946341c2b6e14e9d1", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 297, "sha256": "0fe261119d55cc47f79e341f9f82904b995e77787c5c42ee24ebf59e32926518"} +{"at_utc": "2026-10-18T13:00:46.695+00:00", "data": {"http_status": 201, "ids": {"documentId": 65, "documentVersionId": 65, "embeddingJobId": 63, "fileObjectId": 63}, "outcome": "ACKNOWLEDGED", "request_id": "upload-24f9a22d31904c4585030acce87e0473"}, "kind": "response", "previous": "0fe261119d55cc47f79e341f9f82904b995e77787c5c42ee24ebf59e32926518", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 298, "sha256": "439c79e410d9dce761b51ed308b46a08efe7c17f752d7ceffbe288b809db0412"} +{"at_utc": "2026-10-18T13:00:46.704+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.863, "http_status": 201, "ordinal": 63}, "kind": "upload_observed", "previous": "439c79e410d9dce761b51ed308b46a08efe7c17f752d7ceffbe288b809db0412", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 299, "sha256": "d221ef6c5288c42bcae23a26e2b3dc85ac4cd4cc97b12ca4bf655e1f12529123"} +{"at_utc": "2026-10-18T13:00:46.719+00:00", "data": {"cohort": "loss", "request_id": "upload-28392863101e4285a08246980789e32f", "sha256": "3949b46078d4b30cca6b7d5701929ced60553b456cc3bd69067fc943d7049190", "size": 1045}, "kind": "sent", "previous": "d221ef6c5288c42bcae23a26e2b3dc85ac4cd4cc97b12ca4bf655e1f12529123", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 300, "sha256": "45b106960adede1370bb7feb883c861b1b6e59aa15209bed48b4338c5768b93b"} +{"at_utc": "2026-10-18T13:00:54.537+00:00", "data": {"http_status": 201, "ids": {"documentId": 66, "documentVersionId": 66, "embeddingJobId": 64, "fileObjectId": 64}, "outcome": "ACKNOWLEDGED", "request_id": "upload-28392863101e4285a08246980789e32f"}, "kind": "response", "previous": "45b106960adede1370bb7feb883c861b1b6e59aa15209bed48b4338c5768b93b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 301, "sha256": "fa4d0829cbecaf37e78dd7edbbad82533fea52cf98c17df44091050fd850a623"} +{"at_utc": "2026-10-18T13:00:54.546+00:00", "data": {"cohort": "loss", "elapsed_seconds": 7.818, "http_status": 201, "ordinal": 64}, "kind": "upload_observed", "previous": "fa4d0829cbecaf37e78dd7edbbad82533fea52cf98c17df44091050fd850a623", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 302, "sha256": "2ad358cde0076faa3f852986f4124e05351f9ba087a52fac3e0c44cda5d53ba3"} +{"at_utc": "2026-10-18T13:00:54.557+00:00", "data": {"cohort": "loss", "request_id": "upload-3a06ef2536ca481fb0df95fd45d69bc6", "sha256": "e2a4fa83f61443eca8063b4b03e3f8955ebb433b1bf89275a1972f48ea6508f9", "size": 639}, "kind": "sent", "previous": "2ad358cde0076faa3f852986f4124e05351f9ba087a52fac3e0c44cda5d53ba3", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 303, "sha256": "67c301628067654aca7f857c01c9dc64266f644ee41f98ab760b94b3c8dbd4c1"} +{"at_utc": "2026-10-18T13:01:03.210+00:00", "data": {"http_status": 201, "ids": {"documentId": 67, "documentVersionId": 67, "embeddingJobId": 65, "fileObjectId": 65}, "outcome": "ACKNOWLEDGED", "request_id": "upload-3a06ef2536ca481fb0df95fd45d69bc6"}, "kind": "response", "previous": "67c301628067654aca7f857c01c9dc64266f644ee41f98ab760b94b3c8dbd4c1", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 304, "sha256": "481d5a1e597a469ea9db97b65b0e9eadb53d8f2998ef5449c97bea1faed2d3de"} +{"at_utc": "2026-10-18T13:01:03.214+00:00", "data": {"cohort": "loss", "elapsed_seconds": 8.652, "http_status": 201, "ordinal": 65}, "kind": "upload_observed", "previous": "481d5a1e597a469ea9db97b65b0e9eadb53d8f2998ef5449c97bea1faed2d3de", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 305, "sha256": "10a6940023f57865909b0e2459caa9378b8e1d482d4a135673be6887b7ea9b38"} +{"at_utc": "2026-10-18T13:01:24.539+00:00", "data": {"inputs_sha256": "2c5afe03e8ed47c1924cc6082873d9e5ec52c5de808a98f0f351e807d9b20b95", "wal_objects": 8}, "kind": "cutoff", "previous": "10a6940023f57865909b0e2459caa9378b8e1d482d4a135673be6887b7ea9b38", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 306, "sha256": "399add44ec6d801dfaee982163ed00381424dfb0b655c209935541cbf8d99ede"} +{"at_utc": "2026-10-18T13:01:24.544+00:00", "data": {"cloud_bucket_iam_denial_tested": false, "source_api_stop_started_at_utc": "2026-10-18T13:01:13.892692+00:00", "source_data_paths_stopped_at_utc": "2026-10-18T13:01:24.539551+00:00", "source_gcs_guard_enforced": true, "source_listener_ports_closed": 2}, "kind": "fault_boundary", "previous": "399add44ec6d801dfaee982163ed00381424dfb0b655c209935541cbf8d99ede", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 307, "sha256": "853d0cffdbed93064775c1966fdb8de7dc1600b85f124fa44cf6556cc6af109b"} +{"at_utc": "2026-10-18T13:01:24.555+00:00", "data": {}, "kind": "restore_started", "previous": "853d0cffdbed93064775c1966fdb8de7dc1600b85f124fa44cf6556cc6af109b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 308, "sha256": "56d8a03543e33bfd055a3b9e0279d2d591ff5c77743e9b25c374a5d635d05c3c"} +{"at_utc": "2026-10-18T13:03:05.216+00:00", "data": {"frozen_identity_used": true, "guard_seconds": 1800, "licensed_opensql_started": true, "network": "none", "original_catalog_mounted": false, "stage_archive_sha256": "fd09cb7683049800245fac94ba50fd8db7b6f3ca155ddafe703be6607ebb21f1"}, "kind": "offline_restore_observed", "previous": "56d8a03543e33bfd055a3b9e0279d2d591ff5c77743e9b25c374a5d635d05c3c", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 309, "sha256": "d8a8b2221ca57cdb4b3f60934930cc7fb73679e7dadda7cdc440f89584661711"} +{"at_utc": "2026-10-18T13:03:05.227+00:00", "data": {"elapsed_seconds": 100.667}, "kind": "restored", "previous": "d8a8b2221ca57cdb4b3f60934930cc7fb73679e7dadda7cdc440f89584661711", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 310, "sha256": "6218471df78df924cb08d8ec2cf4b93c50605aac51a4cc177cb1d68d6aa05b9f"} +{"at_utc": "2026-10-18T13:03:28.438+00:00", "data": {"acknowledged": 65, "activation_allowed": true, "audit_sha256": "88ee63810f780eebff125d5844d06568fe8905efa41a3013af43e4aa6e90501b", "lost_or_incomplete_acknowledged": 32, "snapshot_sha256": "a9f0d38fd7da13c17f23ca1042f8854b9a7cc104c0c3ff0c8f3941a5282d377e"}, "kind": "initial_audit", "previous": "6218471df78df924cb08d8ec2cf4b93c50605aac51a4cc177cb1d68d6aa05b9f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 311, "sha256": "2820b8574567239a86babd7ba0f81a46cae61eece265cad14b1c101d06aaa469"} +{"at_utc": "2026-10-18T13:03:38.955+00:00", "data": {"initial_audit_already_durable": true, "public_bucket_changed": false}, "kind": "isolated_bucket_rebound", "previous": "2820b8574567239a86babd7ba0f81a46cae61eece265cad14b1c101d06aaa469", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 312, "sha256": "d8294a0b33b8bbb97b5ca70db78a12f21b77c4fabc0dbde1d877ea0089bd757d"} +{"at_utc": "2026-10-18T13:03:46.231+00:00", "data": {"private_endpoint_omitted": true, "purpose": "recovery"}, "kind": "tunnel_ready", "previous": "d8294a0b33b8bbb97b5ca70db78a12f21b77c4fabc0dbde1d877ea0089bd757d", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 313, "sha256": "476b2b6a46acbe6aea7e908a163ef441d16210c8d59a6ef70e73bd3f128bd897"} +{"at_utc": "2026-10-18T13:04:10.484+00:00", "data": {"purpose": "recovery", "worker_enabled": true}, "kind": "backend_ready", "previous": "476b2b6a46acbe6aea7e908a163ef441d16210c8d59a6ef70e73bd3f128bd897", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 314, "sha256": "5423e2ac9dd423842a672780a84873cb043d209a6836183f222c1c21303dbc04"} +{"at_utc": "2026-10-18T13:04:12.013+00:00", "data": {"account_recreated": false, "http_status": 200}, "kind": "restored_login_observed", "previous": "5423e2ac9dd423842a672780a84873cb043d209a6836183f222c1c21303dbc04", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 315, "sha256": "2bbc475f2581cec1682d509211c274a9bdc77e4c8fb535a1d79b3b1b4d0b44cf"} +{"at_utc": "2026-10-18T13:04:12.025+00:00", "data": {}, "kind": "activated", "previous": "2bbc475f2581cec1682d509211c274a9bdc77e4c8fb535a1d79b3b1b4d0b44cf", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 316, "sha256": "f25efd06b02e14012d45d6923ec87c1dfbd5eb55b0c0ec2fd8f9ae7af46bc1e3"} +{"at_utc": "2026-10-18T13:04:21.541+00:00", "data": {"function": "download", "passed": true, "samples": 4}, "kind": "functional_observed", "previous": "f25efd06b02e14012d45d6923ec87c1dfbd5eb55b0c0ec2fd8f9ae7af46bc1e3", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 317, "sha256": "7fb01d9067494d251cf89c975539966051b3db0a4c3c9ba770b3ce512717c85a"} +{"at_utc": "2026-10-18T13:04:37.384+00:00", "data": {"expected_document_in_top_k": true, "function": "search", "samples": 1, "top_k": 20}, "kind": "functional_observed", "previous": "7fb01d9067494d251cf89c975539966051b3db0a4c3c9ba770b3ce512717c85a", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 318, "sha256": "68217f40d22d07789019e2b2844b330a10662efaa793ed53666eccac95f745a5"} +{"at_utc": "2026-10-18T13:04:38.536+00:00", "data": {"ordinal": 5, "phase": "recovery", "state": "PROCESSING"}, "kind": "index_state", "previous": "68217f40d22d07789019e2b2844b330a10662efaa793ed53666eccac95f745a5", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 319, "sha256": "56943522a78cb27cd373c91eb9a565d0b84b5b599da80877022dfc0ea1239e08"} +{"at_utc": "2026-10-18T13:05:12.529+00:00", "data": {"ordinal": 5, "phase": "recovery", "state": "INDEXED"}, "kind": "index_state", "previous": "56943522a78cb27cd373c91eb9a565d0b84b5b599da80877022dfc0ea1239e08", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 320, "sha256": "498a003d1b3ef5edf7fda4935b936cca6b622403c458853b02f068143bbfba8f"} +{"at_utc": "2026-10-18T13:05:23.026+00:00", "data": {"attempt_count": 2, "function": "job_resume", "job_count_for_version": 1, "job_status": "INDEXED"}, "kind": "functional_observed", "previous": "498a003d1b3ef5edf7fda4935b936cca6b622403c458853b02f068143bbfba8f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 321, "sha256": "28d35faf8b95c666d93669468cd1f13aefae2f6721ec0a71fcd4d6bcf7dc2b2f"} +{"at_utc": "2026-10-18T13:05:30.660+00:00", "data": {"cohort": "functional", "elapsed_seconds": 7.615, "http_status": 201, "ordinal": 1001}, "kind": "upload_observed", "previous": "28d35faf8b95c666d93669468cd1f13aefae2f6721ec0a71fcd4d6bcf7dc2b2f", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 322, "sha256": "f72798fd6428a423e5f3cdc8377b7c6148d01524b561d8e7130d8d1d0ba3e580"} +{"at_utc": "2026-10-18T13:05:31.770+00:00", "data": {"ordinal": 1001, "phase": "recovery", "state": "PENDING"}, "kind": "index_state", "previous": "f72798fd6428a423e5f3cdc8377b7c6148d01524b561d8e7130d8d1d0ba3e580", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 323, "sha256": "ca07407260b1dbbccc7773ad14a92163ab9f7e49c32319a2511dc4a6c735f7e8"} +{"at_utc": "2026-10-18T13:10:32.087+00:00", "data": {"ordinal": 1001, "phase": "recovery", "state": "PROCESSING"}, "kind": "index_state", "previous": "ca07407260b1dbbccc7773ad14a92163ab9f7e49c32319a2511dc4a6c735f7e8", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 324, "sha256": "6266c6227f474621145b29682a62a98dba02902e615a9b79899f370ebf97bcab"} +{"at_utc": "2026-10-18T13:10:55.946+00:00", "data": {"ordinal": 1001, "phase": "recovery", "state": "INDEXED"}, "kind": "index_state", "previous": "6266c6227f474621145b29682a62a98dba02902e615a9b79899f370ebf97bcab", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 325, "sha256": "ab9cd49f2fb7e6f7784ba88444f9706b1020595a09b89ba13add77cef275f492"} +{"at_utc": "2026-10-18T13:10:57.751+00:00", "data": {"from_cutoff_seconds": 176.992, "function": "download", "recovered_at_utc": "2026-10-18T13:04:21.530770+00:00", "samples": 4}, "kind": "functional", "previous": "ab9cd49f2fb7e6f7784ba88444f9706b1020595a09b89ba13add77cef275f492", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 326, "sha256": "a316c2ddd6885c366bb0317402f2c24081572fbac4928bd95ff8b8dccaab8f9a"} +{"at_utc": "2026-10-18T13:10:57.757+00:00", "data": {"from_cutoff_seconds": 192.841, "function": "search", "recovered_at_utc": "2026-10-18T13:04:37.379579+00:00", "samples": 1}, "kind": "functional", "previous": "a316c2ddd6885c366bb0317402f2c24081572fbac4928bd95ff8b8dccaab8f9a", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 327, "sha256": "5a7622d0d26d4b5d83d6a6a9c635971821d284d2d800e640e18cb32f60e98343"} +{"at_utc": "2026-10-18T13:10:57.763+00:00", "data": {"from_cutoff_seconds": 238.483, "function": "job_resume", "recovered_at_utc": "2026-10-18T13:05:23.022354+00:00", "samples": 1}, "kind": "functional", "previous": "5a7622d0d26d4b5d83d6a6a9c635971821d284d2d800e640e18cb32f60e98343", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 328, "sha256": "263d1e16be5202024106ee8fc9cb362640eb9f649e8eb909ec85ef0eaf8785af"} +{"at_utc": "2026-10-18T13:10:57.768+00:00", "data": {"from_cutoff_seconds": 246.122, "function": "new_upload", "recovered_at_utc": "2026-10-18T13:05:30.661209+00:00", "samples": 1}, "kind": "functional", "previous": "263d1e16be5202024106ee8fc9cb362640eb9f649e8eb909ec85ef0eaf8785af", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 329, "sha256": "a1f713a5d25880ae398da824a2a99be3863809b0edb70614ae1465b54c5c72bd"} +{"at_utc": "2026-10-18T13:10:57.773+00:00", "data": {"from_cutoff_seconds": 571.408, "function": "new_indexing", "recovered_at_utc": "2026-10-18T13:10:55.947040+00:00", "samples": 1}, "kind": "functional", "previous": "a1f713a5d25880ae398da824a2a99be3863809b0edb70614ae1465b54c5c72bd", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 330, "sha256": "6f7dbf39c5818a4ce9a16afb5252d09826188925ac03ccabf6d2c813bcc1c353"} +{"at_utc": "2026-10-18T13:10:57.777+00:00", "data": {"from_cutoff_seconds": 573.186, "function": "new_download", "recovered_at_utc": "2026-10-18T13:10:57.724793+00:00", "samples": 1}, "kind": "functional", "previous": "6f7dbf39c5818a4ce9a16afb5252d09826188925ac03ccabf6d2c813bcc1c353", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 331, "sha256": "f8579dc3b9da7c8f6cf306a877636b7fb3b4f2ea1df33cbc72131eecb2eb26f8"} +{"at_utc": "2026-10-18T13:11:30.178+00:00", "data": {"audit_sha256": "88ee63810f780eebff125d5844d06568fe8905efa41a3013af43e4aa6e90501b", "consistency": {"checked_documents": 34, "duplicate_chunks": 0, "duplicate_embeddings": 0, "incomplete_jobs": 0, "missing_embeddings": 0}, "snapshot_sha256": "6ff1af3002e9812ea86024aea8b47bf81d985cdf07caf2130bd8936103bd2058"}, "kind": "final_audit", "previous": "f8579dc3b9da7c8f6cf306a877636b7fb3b4f2ea1df33cbc72131eecb2eb26f8", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 332, "sha256": "6e8ea425d294d0abc9e6ba8806df2a11d541bba4ff786766b5ee3674b00c6f89"} +{"at_utc": "2026-10-18T13:13:05.688+00:00", "data": {"owned_resources_removed": true, "public_config_preserved": true, "source_cluster_restored": true}, "kind": "cleanup", "previous": "6e8ea425d294d0abc9e6ba8806df2a11d541bba4ff786766b5ee3674b00c6f89", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 333, "sha256": "d31df0a19edf44c91456ea7618d70ae79e4e0a0ed6f2b498ec08dd9702274c5b"} +{"at_utc": "2026-10-18T13:13:05.692+00:00", "data": {"execution": "PASS", "summary_sha256": "7a634d7020ef885f82cdf1e3a8789a627ef759157f8ab2c8da8f8cecbe2a1f7c"}, "kind": "finished", "previous": "d31df0a19edf44c91456ea7618d70ae79e4e0a0ed6f2b498ec08dd9702274c5b", "run_id": "DR-RESTORE-20261018-FORMAL1", "sequence": 334, "sha256": "8c3639db1a67ee655719d42aa3efd58ee55e87714a91d981ae66a38616107b0f"} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/final-audit.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/final-audit.json new file mode 100644 index 00000000..6508b46c --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/final-audit.json @@ -0,0 +1,465 @@ +{ + "acknowledged": 65, + "activation_allowed": true, + "http_error": 0, + "lost_or_incomplete_acknowledged": 32, + "requests": [ + { + "ack_at_utc": "2026-10-18T12:37:05.356+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-965c69a074e94cacb37ed7cd2f1cfb2a", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:37:50.492+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3a043da834f14436b70a14733f2252b8", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:38:29.983+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-aa478ffa2f2a43acb43c84ce2d8ac431", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:39:12.453+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-9ca175c82653458b92bed4bcd9537a5f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:41:01.038+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-9e7df4353fed4e7aaa5a3c7f1408e4e5", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:25.127+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c9166086f88c466d8a756e6f6d9622c5", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:34.772+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-66ba7d8a8de148629607f6a26bbc8f90", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:46.064+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-99d323fbaa964760a0b71ef6412e025d", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:57.093+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-b331678e5a7d42cf8a4c4db431cd19bf", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:08.644+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0ba3a720b65e417f807def554042667e", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:20.001+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-49eb7ad416b24429afb537f16ae6333f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:31.133+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c7f7b9de82414e0889a3e52223279673", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:41.632+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3332040234024f5fab7f4545b4bb1f9f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:52.682+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0935757b0cd5475ab423f04174c1a17b", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:03.643+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-dd0d63abe92b4a9192a897b7af26e314", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:16.097+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-4ae3860380ef4bfcb24944535960303e", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:25.209+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-6192da40bdbd4619821046d81292d3c7", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:37.046+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-b29f691bda084cf783174e730f6e277f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:47.680+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-abee4f99bb7a48bb9c763fdd49bd16f8", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:58.226+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-d73359977ccb452089a0bee66a0103ec", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:09.841+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1cf0c03bbe594bb5827f2e273bc9cb14", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:21.081+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-491bc1dea1d34d1fb175a4a78130b40d", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:31.532+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1528b4c8ea56434286fd24803372273f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:42.854+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-5bbe80c1080d48dea97625043e1d6cb0", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:53.221+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-134f03b858e8450b8b08c854a839ac90", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:04.581+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1289f0ebaa3f4751a0ad788c9b648ea3", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:15.508+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-2fb35853a420491c971f6c971b63896b", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:27.678+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-dd1a5e718ce24ce8b4508229da0951aa", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:37.267+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c01c1678cb594cd993a9a0170bb18cd2", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:48.798+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-83040d6892d74b63ab768e9d90bb9d04", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:59.902+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-603a571ae0c147bea35927c304f9d772", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:54:10.932+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1a65c1542f684feeb034c3b1a6b92c37", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:54:21.832+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0e631fa0a6ea4ee1afac2ce5f99f4327", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:55:43.909+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-80ba1cb578334f68b4712ec46271edab", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:55:50.968+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-fbaf3249ead041aab8c8729f7cfe6bb5", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:55:58.082+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-f9cecc4bc6e645ebae7a7d8f0b60833f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:05.900+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-266e9cd10bf44c4099c45351aff5a213", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:13.259+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-f385c020c7394b5993fc0d23dd0c4314", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:20.688+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-cd52989eefdc4d01993e077360b6e844", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:28.515+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-d008622649b04e9f871b3bd2296cc053", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:36.324+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-d3b7eec6c2394e2495f06d66006c6f00", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:44.441+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-665e96a9ada843efa78379b0e921ec48", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:51.804+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-046c77ba2e4a4b88ad0d1c6f7bf11bb9", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:59.481+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-607f1c2f64d64fb7b6632acc5f472d59", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:08.320+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3ff63ce9c7b4402fbcbee28935a5d414", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:16.165+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3a2ac04a9ac346d2b292f89ba0b096b3", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:24.035+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-24735b8cf98347f091b106994e210a7b", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:31.231+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-b0e8c7e2aea24562b5813170fa6a816d", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:38.977+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-04233cdad305433abee26af93d8356e6", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:47.528+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3697e70376084b83a4129e6cc3d6d33f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:56.017+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-23e0f579e82146679e7c3a58a1605aba", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:03.981+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-cf72758cb0624f18a5632f23c4419586", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:11.999+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-29c885e4f88d4e57aa18663fad3b240f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:19.128+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0ac416309fb24af199ca7e931bb45a7b", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:27.247+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1daf93dff41c468f85bfdbbe70d6fcc0", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:35.374+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-4f5d46c32ba04d24957cc1780db113b4", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:46.615+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-e1cbe18e321045978d12d708efb76b72", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:56.309+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-e871accf911d4c9c9c1345f38362122b", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:59:07.914+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-04c4842d79894d72903e7b423e0d593a", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:59:18.865+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-09b3714d1ed542ecad7affd1258265e4", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:30.969+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-66a683aeca184c1a9fef9e2c5430947e", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:38.813+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c278df614f444546967f1e1fa5ee1845", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:46.695+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-24f9a22d31904c4585030acce87e0473", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:54.537+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-28392863101e4285a08246980789e32f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:01:03.210+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3a06ef2536ca481fb0df95fd45d69bc6", + "restored_state": "DB_MISSING" + } + ], + "restored_acknowledged": 33, + "unknown_http": 0 +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/final-snapshot.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/final-snapshot.json new file mode 100644 index 00000000..0da4ebfa --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/final-snapshot.json @@ -0,0 +1,803 @@ +{ + "consistency": { + "checked_documents": 34, + "duplicate_chunks": 0, + "duplicate_embeddings": 0, + "incomplete_jobs": 0, + "missing_embeddings": 0 + }, + "flyway_ran": true, + "records": [ + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792327460164362", + "ids": { + "documentId": 3, + "documentVersionId": 3, + "embeddingJobId": 1, + "fileObjectId": 1 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-965c69a074e94cacb37ed7cd2f1cfb2a", + "sha256": "1a8c22f7f7acce67d4544607b38e46ab4754d71cd4413e85533acfa1c30bb027", + "size": 1044, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792327461460810", + "ids": { + "documentId": 4, + "documentVersionId": 4, + "embeddingJobId": 2, + "fileObjectId": 2 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-3a043da834f14436b70a14733f2252b8", + "sha256": "1595721465e0cfb8d633de6b2a3674e7be455007aaf278d977f983be7ce5f609", + "size": 638, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792327462725332", + "ids": { + "documentId": 5, + "documentVersionId": 5, + "embeddingJobId": 3, + "fileObjectId": 3 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-aa478ffa2f2a43acb43c84ce2d8ac431", + "sha256": "6213e3cb37b48c59023d9bf356f7126f9f013358cb665acd23a3dc8613a50140", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792327464013839", + "ids": { + "documentId": 6, + "documentVersionId": 6, + "embeddingJobId": 4, + "fileObjectId": 4 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-9ca175c82653458b92bed4bcd9537a5f", + "sha256": "671f658102aa034a78d956f412eb24e80919a2d2a25df92ae5c58c393317500f", + "size": 638, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 2, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792327465201824", + "ids": { + "documentId": 7, + "documentVersionId": 7, + "embeddingJobId": 5, + "fileObjectId": 5 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-9e7df4353fed4e7aaa5a3c7f1408e4e5", + "sha256": "c5f4b1afadfdbc24aefed3d4ef5786b495b09f83e957583be16d49fd876f7921", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328086019431", + "ids": { + "documentId": 8, + "documentVersionId": 8, + "embeddingJobId": 6, + "fileObjectId": 6 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-c9166086f88c466d8a756e6f6d9622c5", + "sha256": "6403d52e2ebc1bf1adabfd78a27dad188d674a61a25d3125864a896f3a18e3b8", + "size": 1044, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 2, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328087449527", + "ids": { + "documentId": 9, + "documentVersionId": 9, + "embeddingJobId": 7, + "fileObjectId": 7 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-66ba7d8a8de148629607f6a26bbc8f90", + "sha256": "b5533ff9dc19e8ab13eceda011d203151c304adb6c15a76f9624c40e4f727229", + "size": 638, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328088900545", + "ids": { + "documentId": 10, + "documentVersionId": 10, + "embeddingJobId": 8, + "fileObjectId": 8 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-99d323fbaa964760a0b71ef6412e025d", + "sha256": "17ca77586adbaf5282fa290b8daa01e087a06aa263602a51f4ae6557813e6a96", + "size": 1044, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328090136180", + "ids": { + "documentId": 11, + "documentVersionId": 11, + "embeddingJobId": 9, + "fileObjectId": 9 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-b331678e5a7d42cf8a4c4db431cd19bf", + "sha256": "389fdb5425c931b59623ed2ba6888f9e120609317fe8963245d88aec7304d929", + "size": 638, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328091427646", + "ids": { + "documentId": 12, + "documentVersionId": 12, + "embeddingJobId": 10, + "fileObjectId": 10 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-0ba3a720b65e417f807def554042667e", + "sha256": "ea06cf64bc7c18fd6379ee9fbc9eade765483eadbeee54799f7add448ad4288c", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328092722116", + "ids": { + "documentId": 13, + "documentVersionId": 13, + "embeddingJobId": 11, + "fileObjectId": 11 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-49eb7ad416b24429afb537f16ae6333f", + "sha256": "2747f698cfa86d0f8f0873b9ec34139ab9a13df1a748a441538d2f764af69c4b", + "size": 639, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328094048871", + "ids": { + "documentId": 14, + "documentVersionId": 14, + "embeddingJobId": 12, + "fileObjectId": 12 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-c7f7b9de82414e0889a3e52223279673", + "sha256": "f052804612d30f76364e7c3e9103be400f0b1117eb36920e0c56de1eb80cc19d", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328095388030", + "ids": { + "documentId": 15, + "documentVersionId": 15, + "embeddingJobId": 13, + "fileObjectId": 13 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-3332040234024f5fab7f4545b4bb1f9f", + "sha256": "172c782cc9e625e2e54344a27890565cc8461a2aacc7c38e1b29d4197322a243", + "size": 639, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328096826330", + "ids": { + "documentId": 16, + "documentVersionId": 16, + "embeddingJobId": 14, + "fileObjectId": 14 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-0935757b0cd5475ab423f04174c1a17b", + "sha256": "f29454ce42ad1c2047303fb13400aba5043e7483c49ab66c6294ac7677c5b566", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328097997840", + "ids": { + "documentId": 17, + "documentVersionId": 17, + "embeddingJobId": 15, + "fileObjectId": 15 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-dd0d63abe92b4a9192a897b7af26e314", + "sha256": "c5f84d6ccff80425b9dd6a301379c50c6c0ba91185f968280bf6eff1d36f758a", + "size": 639, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328099653018", + "ids": { + "documentId": 18, + "documentVersionId": 18, + "embeddingJobId": 16, + "fileObjectId": 16 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-4ae3860380ef4bfcb24944535960303e", + "sha256": "e23b60a24d1b2f413ac0b5a9d712b9f9df7a7c256b58344588fb5dc8b8a3baf2", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328101064159", + "ids": { + "documentId": 19, + "documentVersionId": 19, + "embeddingJobId": 17, + "fileObjectId": 17 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-6192da40bdbd4619821046d81292d3c7", + "sha256": "7a4d5693fe950d8499727512a0c01f613c912b09ac020f4bfea558a347dc104f", + "size": 639, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328102247810", + "ids": { + "documentId": 20, + "documentVersionId": 20, + "embeddingJobId": 18, + "fileObjectId": 18 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-b29f691bda084cf783174e730f6e277f", + "sha256": "fdf9d7977772c42a6e4954dab9df0260a963f11c4475794d6d457279af48717a", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328103504958", + "ids": { + "documentId": 21, + "documentVersionId": 21, + "embeddingJobId": 19, + "fileObjectId": 19 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-abee4f99bb7a48bb9c763fdd49bd16f8", + "sha256": "409f56aa647a363708e7dfff63a795da589aba643c16578b7b8e8cf1c2b30010", + "size": 639, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328104848915", + "ids": { + "documentId": 22, + "documentVersionId": 22, + "embeddingJobId": 20, + "fileObjectId": 20 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-d73359977ccb452089a0bee66a0103ec", + "sha256": "5e5ef2b830e149e931c7284c1b4ff0daf9d491c4deb3464b8bf8bf95a69da16f", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328106308712", + "ids": { + "documentId": 23, + "documentVersionId": 23, + "embeddingJobId": 21, + "fileObjectId": 21 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-1cf0c03bbe594bb5827f2e273bc9cb14", + "sha256": "42701bbc7c882a301ccdc99ab8419a1ef05573f743a09d77cc4061e50d980643", + "size": 639, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328107545458", + "ids": { + "documentId": 24, + "documentVersionId": 24, + "embeddingJobId": 22, + "fileObjectId": 22 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-491bc1dea1d34d1fb175a4a78130b40d", + "sha256": "6d5d165797b911105c7313c58ad614efbc16e70ba568b99ea261d31929f807cd", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328108969233", + "ids": { + "documentId": 25, + "documentVersionId": 25, + "embeddingJobId": 23, + "fileObjectId": 23 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-1528b4c8ea56434286fd24803372273f", + "sha256": "6c5b22d9c0bc5e4d8b53e680fad8c1ffcdfdfed3f2077bfdaca176e81010409b", + "size": 639, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328110244934", + "ids": { + "documentId": 26, + "documentVersionId": 26, + "embeddingJobId": 24, + "fileObjectId": 24 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-5bbe80c1080d48dea97625043e1d6cb0", + "sha256": "4adfd5f972546b1dfa778665f20ffab54a1b862e8327f1b8a027b5c35ed41e90", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328111630812", + "ids": { + "documentId": 27, + "documentVersionId": 27, + "embeddingJobId": 25, + "fileObjectId": 25 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-134f03b858e8450b8b08c854a839ac90", + "sha256": "d80005d1c6c406a53d135ce7728671f5a50fa6497ffbbf969e7cb032629f2caa", + "size": 639, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328113382947", + "ids": { + "documentId": 28, + "documentVersionId": 28, + "embeddingJobId": 26, + "fileObjectId": 26 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-1289f0ebaa3f4751a0ad788c9b648ea3", + "sha256": "cf77c01b509b1129086a7b0aa93d64069b0bec875466228e24a5155c4677fa9b", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328114737996", + "ids": { + "documentId": 29, + "documentVersionId": 29, + "embeddingJobId": 27, + "fileObjectId": 27 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-2fb35853a420491c971f6c971b63896b", + "sha256": "7f923c7e4b2be2a18a9f4324f9c1d6269f2d01c74062645f5918b2e035b93076", + "size": 639, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328116002594", + "ids": { + "documentId": 30, + "documentVersionId": 30, + "embeddingJobId": 28, + "fileObjectId": 28 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-dd1a5e718ce24ce8b4508229da0951aa", + "sha256": "7630822014249f6f05b8fcc99ade977ac1ed1db491bd43a5198b8af046a8bcb5", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328117286515", + "ids": { + "documentId": 31, + "documentVersionId": 31, + "embeddingJobId": 29, + "fileObjectId": 29 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-c01c1678cb594cd993a9a0170bb18cd2", + "sha256": "0a05be12c9ee25da1b2bb3bc24fb51bed25970f9896f473067769bd4ac987680", + "size": 639, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328118488397", + "ids": { + "documentId": 32, + "documentVersionId": 32, + "embeddingJobId": 30, + "fileObjectId": 30 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-83040d6892d74b63ab768e9d90bb9d04", + "sha256": "d3d1f41587d7a417b5c9d6e1f7affd062395787a32b6e75043fb8f15f84e22ce", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328119765199", + "ids": { + "documentId": 33, + "documentVersionId": 33, + "embeddingJobId": 31, + "fileObjectId": 31 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-603a571ae0c147bea35927c304f9d772", + "sha256": "a871ca01a920e8762991610ff7e7c81ff7874d90214cc30ff20a95bdaba83899", + "size": 639, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792328121047737", + "ids": { + "documentId": 34, + "documentVersionId": 34, + "embeddingJobId": 32, + "fileObjectId": 32 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-1a65c1542f684feeb034c3b1a6b92c37", + "sha256": "4588a9b4f22011102e8ffcca351041c4ad02e784d45cb753f1f312f25d5c0d5b", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792328122347029", + "ids": { + "documentId": 35, + "documentVersionId": 35, + "embeddingJobId": 33, + "fileObjectId": 33 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-0e631fa0a6ea4ee1afac2ce5f99f4327", + "sha256": "38ea0ae6af7f1af5b8acc0bc844f34a8a8e05e7621a89725fc26c78f9ab6475e", + "size": 639, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "request_id": "upload-80ba1cb578334f68b4712ec46271edab", + "state": "DB_MISSING" + }, + { + "request_id": "upload-fbaf3249ead041aab8c8729f7cfe6bb5", + "state": "DB_MISSING" + }, + { + "request_id": "upload-f9cecc4bc6e645ebae7a7d8f0b60833f", + "state": "DB_MISSING" + }, + { + "request_id": "upload-266e9cd10bf44c4099c45351aff5a213", + "state": "DB_MISSING" + }, + { + "request_id": "upload-f385c020c7394b5993fc0d23dd0c4314", + "state": "DB_MISSING" + }, + { + "request_id": "upload-cd52989eefdc4d01993e077360b6e844", + "state": "DB_MISSING" + }, + { + "request_id": "upload-d008622649b04e9f871b3bd2296cc053", + "state": "DB_MISSING" + }, + { + "request_id": "upload-d3b7eec6c2394e2495f06d66006c6f00", + "state": "DB_MISSING" + }, + { + "request_id": "upload-665e96a9ada843efa78379b0e921ec48", + "state": "DB_MISSING" + }, + { + "request_id": "upload-046c77ba2e4a4b88ad0d1c6f7bf11bb9", + "state": "DB_MISSING" + }, + { + "request_id": "upload-607f1c2f64d64fb7b6632acc5f472d59", + "state": "DB_MISSING" + }, + { + "request_id": "upload-3ff63ce9c7b4402fbcbee28935a5d414", + "state": "DB_MISSING" + }, + { + "request_id": "upload-3a2ac04a9ac346d2b292f89ba0b096b3", + "state": "DB_MISSING" + }, + { + "request_id": "upload-24735b8cf98347f091b106994e210a7b", + "state": "DB_MISSING" + }, + { + "request_id": "upload-b0e8c7e2aea24562b5813170fa6a816d", + "state": "DB_MISSING" + }, + { + "request_id": "upload-04233cdad305433abee26af93d8356e6", + "state": "DB_MISSING" + }, + { + "request_id": "upload-3697e70376084b83a4129e6cc3d6d33f", + "state": "DB_MISSING" + }, + { + "request_id": "upload-23e0f579e82146679e7c3a58a1605aba", + "state": "DB_MISSING" + }, + { + "request_id": "upload-cf72758cb0624f18a5632f23c4419586", + "state": "DB_MISSING" + }, + { + "request_id": "upload-29c885e4f88d4e57aa18663fad3b240f", + "state": "DB_MISSING" + }, + { + "request_id": "upload-0ac416309fb24af199ca7e931bb45a7b", + "state": "DB_MISSING" + }, + { + "request_id": "upload-1daf93dff41c468f85bfdbbe70d6fcc0", + "state": "DB_MISSING" + }, + { + "request_id": "upload-4f5d46c32ba04d24957cc1780db113b4", + "state": "DB_MISSING" + }, + { + "request_id": "upload-e1cbe18e321045978d12d708efb76b72", + "state": "DB_MISSING" + }, + { + "request_id": "upload-e871accf911d4c9c9c1345f38362122b", + "state": "DB_MISSING" + }, + { + "request_id": "upload-04c4842d79894d72903e7b423e0d593a", + "state": "DB_MISSING" + }, + { + "request_id": "upload-09b3714d1ed542ecad7affd1258265e4", + "state": "DB_MISSING" + }, + { + "request_id": "upload-66a683aeca184c1a9fef9e2c5430947e", + "state": "DB_MISSING" + }, + { + "request_id": "upload-c278df614f444546967f1e1fa5ee1845", + "state": "DB_MISSING" + }, + { + "request_id": "upload-24f9a22d31904c4585030acce87e0473", + "state": "DB_MISSING" + }, + { + "request_id": "upload-28392863101e4285a08246980789e32f", + "state": "DB_MISSING" + }, + { + "request_id": "upload-3a06ef2536ca481fb0df95fd45d69bc6", + "state": "DB_MISSING" + } + ], + "repair_ran": true, + "source_access_used": false, + "worker_started": true +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/formal-spec.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/formal-spec.json new file mode 100644 index 00000000..8a319849 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/formal-spec.json @@ -0,0 +1,18 @@ +{ + "expected_search_top_k": 20, + "formal_workload": true, + "measurement_scope": "isolated document download/search/indexing; no public switchover, OpenProxy or Ollama generation", + "purpose": "업로드 성공 응답 원장과 실제 OpenSQL·GCS 최초 복원 상태 및 기능별 재개 경과 시간 대조", + "recovery_worker_max_concurrency": 2, + "revision": "3bff887a5f6abdf4155957a658599f8f0366a141", + "run_id": "DR-RESTORE-20261018-FORMAL1", + "started_at_utc": "2026-10-18T12:49:06.058119+00:00", + "success_criteria": "기능 코호트 원본·관계 일치, 최초 감사 후 Worker 활성화, 기능 6종 성공, Job 1개·Attempt 2, 최종 미완료/중복/누락 벡터 0, 원복·정리", + "workload": { + "functional_completed": 4, + "functional_interrupted": 1, + "loss_uploads": 60, + "nominal_request_interval_seconds": 11, + "offsite_interval_seconds": 300 + } +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/functional-observations.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/functional-observations.json new file mode 100644 index 00000000..f8111850 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/functional-observations.json @@ -0,0 +1,32 @@ +{ + "download": { + "at_utc": "2026-10-18T13:04:21.530770+00:00", + "passed": true, + "samples": 4 + }, + "job_resume": { + "at_utc": "2026-10-18T13:05:23.022354+00:00", + "passed": true, + "samples": 1 + }, + "new_download": { + "at_utc": "2026-10-18T13:10:57.724793+00:00", + "passed": true, + "samples": 1 + }, + "new_indexing": { + "at_utc": "2026-10-18T13:10:55.947040+00:00", + "passed": true, + "samples": 1 + }, + "new_upload": { + "at_utc": "2026-10-18T13:05:30.661209+00:00", + "passed": true, + "samples": 1 + }, + "search": { + "at_utc": "2026-10-18T13:04:37.379579+00:00", + "passed": true, + "samples": 1 + } +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/initial-audit.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/initial-audit.json new file mode 100644 index 00000000..6508b46c --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/initial-audit.json @@ -0,0 +1,465 @@ +{ + "acknowledged": 65, + "activation_allowed": true, + "http_error": 0, + "lost_or_incomplete_acknowledged": 32, + "requests": [ + { + "ack_at_utc": "2026-10-18T12:37:05.356+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-965c69a074e94cacb37ed7cd2f1cfb2a", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:37:50.492+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3a043da834f14436b70a14733f2252b8", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:38:29.983+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-aa478ffa2f2a43acb43c84ce2d8ac431", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:39:12.453+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-9ca175c82653458b92bed4bcd9537a5f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:41:01.038+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-9e7df4353fed4e7aaa5a3c7f1408e4e5", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:25.127+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c9166086f88c466d8a756e6f6d9622c5", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:34.772+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-66ba7d8a8de148629607f6a26bbc8f90", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:46.064+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-99d323fbaa964760a0b71ef6412e025d", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:57.093+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-b331678e5a7d42cf8a4c4db431cd19bf", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:08.644+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0ba3a720b65e417f807def554042667e", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:20.001+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-49eb7ad416b24429afb537f16ae6333f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:31.133+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c7f7b9de82414e0889a3e52223279673", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:41.632+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3332040234024f5fab7f4545b4bb1f9f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:52.682+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0935757b0cd5475ab423f04174c1a17b", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:03.643+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-dd0d63abe92b4a9192a897b7af26e314", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:16.097+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-4ae3860380ef4bfcb24944535960303e", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:25.209+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-6192da40bdbd4619821046d81292d3c7", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:37.046+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-b29f691bda084cf783174e730f6e277f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:47.680+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-abee4f99bb7a48bb9c763fdd49bd16f8", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:58.226+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-d73359977ccb452089a0bee66a0103ec", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:09.841+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1cf0c03bbe594bb5827f2e273bc9cb14", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:21.081+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-491bc1dea1d34d1fb175a4a78130b40d", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:31.532+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1528b4c8ea56434286fd24803372273f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:42.854+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-5bbe80c1080d48dea97625043e1d6cb0", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:53.221+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-134f03b858e8450b8b08c854a839ac90", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:04.581+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1289f0ebaa3f4751a0ad788c9b648ea3", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:15.508+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-2fb35853a420491c971f6c971b63896b", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:27.678+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-dd1a5e718ce24ce8b4508229da0951aa", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:37.267+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c01c1678cb594cd993a9a0170bb18cd2", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:48.798+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-83040d6892d74b63ab768e9d90bb9d04", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:59.902+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-603a571ae0c147bea35927c304f9d772", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:54:10.932+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1a65c1542f684feeb034c3b1a6b92c37", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:54:21.832+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0e631fa0a6ea4ee1afac2ce5f99f4327", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:55:43.909+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-80ba1cb578334f68b4712ec46271edab", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:55:50.968+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-fbaf3249ead041aab8c8729f7cfe6bb5", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:55:58.082+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-f9cecc4bc6e645ebae7a7d8f0b60833f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:05.900+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-266e9cd10bf44c4099c45351aff5a213", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:13.259+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-f385c020c7394b5993fc0d23dd0c4314", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:20.688+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-cd52989eefdc4d01993e077360b6e844", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:28.515+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-d008622649b04e9f871b3bd2296cc053", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:36.324+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-d3b7eec6c2394e2495f06d66006c6f00", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:44.441+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-665e96a9ada843efa78379b0e921ec48", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:51.804+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-046c77ba2e4a4b88ad0d1c6f7bf11bb9", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:59.481+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-607f1c2f64d64fb7b6632acc5f472d59", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:08.320+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3ff63ce9c7b4402fbcbee28935a5d414", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:16.165+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3a2ac04a9ac346d2b292f89ba0b096b3", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:24.035+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-24735b8cf98347f091b106994e210a7b", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:31.231+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-b0e8c7e2aea24562b5813170fa6a816d", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:38.977+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-04233cdad305433abee26af93d8356e6", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:47.528+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3697e70376084b83a4129e6cc3d6d33f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:56.017+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-23e0f579e82146679e7c3a58a1605aba", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:03.981+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-cf72758cb0624f18a5632f23c4419586", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:11.999+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-29c885e4f88d4e57aa18663fad3b240f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:19.128+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0ac416309fb24af199ca7e931bb45a7b", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:27.247+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1daf93dff41c468f85bfdbbe70d6fcc0", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:35.374+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-4f5d46c32ba04d24957cc1780db113b4", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:46.615+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-e1cbe18e321045978d12d708efb76b72", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:56.309+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-e871accf911d4c9c9c1345f38362122b", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:59:07.914+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-04c4842d79894d72903e7b423e0d593a", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:59:18.865+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-09b3714d1ed542ecad7affd1258265e4", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:30.969+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-66a683aeca184c1a9fef9e2c5430947e", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:38.813+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c278df614f444546967f1e1fa5ee1845", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:46.695+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-24f9a22d31904c4585030acce87e0473", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:54.537+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-28392863101e4285a08246980789e32f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:01:03.210+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3a06ef2536ca481fb0df95fd45d69bc6", + "restored_state": "DB_MISSING" + } + ], + "restored_acknowledged": 33, + "unknown_http": 0 +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/initial-snapshot.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/initial-snapshot.json new file mode 100644 index 00000000..9bdfc039 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/initial-snapshot.json @@ -0,0 +1,797 @@ +{ + "consistency": {}, + "flyway_ran": false, + "records": [ + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792327460164362", + "ids": { + "documentId": 3, + "documentVersionId": 3, + "embeddingJobId": 1, + "fileObjectId": 1 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-965c69a074e94cacb37ed7cd2f1cfb2a", + "sha256": "1a8c22f7f7acce67d4544607b38e46ab4754d71cd4413e85533acfa1c30bb027", + "size": 1044, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792327461460810", + "ids": { + "documentId": 4, + "documentVersionId": 4, + "embeddingJobId": 2, + "fileObjectId": 2 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-3a043da834f14436b70a14733f2252b8", + "sha256": "1595721465e0cfb8d633de6b2a3674e7be455007aaf278d977f983be7ce5f609", + "size": 638, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXED", + "embedding_count": 3, + "generation": "1792327462725332", + "ids": { + "documentId": 5, + "documentVersionId": 5, + "embeddingJobId": 3, + "fileObjectId": 3 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-aa478ffa2f2a43acb43c84ce2d8ac431", + "sha256": "6213e3cb37b48c59023d9bf356f7126f9f013358cb665acd23a3dc8613a50140", + "size": 1045, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 1, + "document_status": "INDEXED", + "embedding_count": 1, + "generation": "1792327464013839", + "ids": { + "documentId": 6, + "documentVersionId": 6, + "embeddingJobId": 4, + "fileObjectId": 4 + }, + "job_status": "INDEXED", + "relations_match": true, + "request_id": "upload-9ca175c82653458b92bed4bcd9537a5f", + "sha256": "671f658102aa034a78d956f412eb24e80919a2d2a25df92ae5c58c393317500f", + "size": 638, + "state": "RESTORED", + "version_status": "INDEXED" + }, + { + "attempt_count": 1, + "chunk_count": 3, + "document_status": "INDEXING", + "embedding_count": 0, + "generation": "1792327465201824", + "ids": { + "documentId": 7, + "documentVersionId": 7, + "embeddingJobId": 5, + "fileObjectId": 5 + }, + "job_status": "PROCESSING", + "relations_match": true, + "request_id": "upload-9e7df4353fed4e7aaa5a3c7f1408e4e5", + "sha256": "c5f4b1afadfdbc24aefed3d4ef5786b495b09f83e957583be16d49fd876f7921", + "size": 1045, + "state": "RESTORED", + "version_status": "EMBEDDING" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328086019431", + "ids": { + "documentId": 8, + "documentVersionId": 8, + "embeddingJobId": 6, + "fileObjectId": 6 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-c9166086f88c466d8a756e6f6d9622c5", + "sha256": "6403d52e2ebc1bf1adabfd78a27dad188d674a61a25d3125864a896f3a18e3b8", + "size": 1044, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328087449527", + "ids": { + "documentId": 9, + "documentVersionId": 9, + "embeddingJobId": 7, + "fileObjectId": 7 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-66ba7d8a8de148629607f6a26bbc8f90", + "sha256": "b5533ff9dc19e8ab13eceda011d203151c304adb6c15a76f9624c40e4f727229", + "size": 638, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328088900545", + "ids": { + "documentId": 10, + "documentVersionId": 10, + "embeddingJobId": 8, + "fileObjectId": 8 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-99d323fbaa964760a0b71ef6412e025d", + "sha256": "17ca77586adbaf5282fa290b8daa01e087a06aa263602a51f4ae6557813e6a96", + "size": 1044, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328090136180", + "ids": { + "documentId": 11, + "documentVersionId": 11, + "embeddingJobId": 9, + "fileObjectId": 9 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-b331678e5a7d42cf8a4c4db431cd19bf", + "sha256": "389fdb5425c931b59623ed2ba6888f9e120609317fe8963245d88aec7304d929", + "size": 638, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328091427646", + "ids": { + "documentId": 12, + "documentVersionId": 12, + "embeddingJobId": 10, + "fileObjectId": 10 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-0ba3a720b65e417f807def554042667e", + "sha256": "ea06cf64bc7c18fd6379ee9fbc9eade765483eadbeee54799f7add448ad4288c", + "size": 1045, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328092722116", + "ids": { + "documentId": 13, + "documentVersionId": 13, + "embeddingJobId": 11, + "fileObjectId": 11 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-49eb7ad416b24429afb537f16ae6333f", + "sha256": "2747f698cfa86d0f8f0873b9ec34139ab9a13df1a748a441538d2f764af69c4b", + "size": 639, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328094048871", + "ids": { + "documentId": 14, + "documentVersionId": 14, + "embeddingJobId": 12, + "fileObjectId": 12 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-c7f7b9de82414e0889a3e52223279673", + "sha256": "f052804612d30f76364e7c3e9103be400f0b1117eb36920e0c56de1eb80cc19d", + "size": 1045, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328095388030", + "ids": { + "documentId": 15, + "documentVersionId": 15, + "embeddingJobId": 13, + "fileObjectId": 13 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-3332040234024f5fab7f4545b4bb1f9f", + "sha256": "172c782cc9e625e2e54344a27890565cc8461a2aacc7c38e1b29d4197322a243", + "size": 639, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328096826330", + "ids": { + "documentId": 16, + "documentVersionId": 16, + "embeddingJobId": 14, + "fileObjectId": 14 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-0935757b0cd5475ab423f04174c1a17b", + "sha256": "f29454ce42ad1c2047303fb13400aba5043e7483c49ab66c6294ac7677c5b566", + "size": 1045, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328097997840", + "ids": { + "documentId": 17, + "documentVersionId": 17, + "embeddingJobId": 15, + "fileObjectId": 15 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-dd0d63abe92b4a9192a897b7af26e314", + "sha256": "c5f84d6ccff80425b9dd6a301379c50c6c0ba91185f968280bf6eff1d36f758a", + "size": 639, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328099653018", + "ids": { + "documentId": 18, + "documentVersionId": 18, + "embeddingJobId": 16, + "fileObjectId": 16 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-4ae3860380ef4bfcb24944535960303e", + "sha256": "e23b60a24d1b2f413ac0b5a9d712b9f9df7a7c256b58344588fb5dc8b8a3baf2", + "size": 1045, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328101064159", + "ids": { + "documentId": 19, + "documentVersionId": 19, + "embeddingJobId": 17, + "fileObjectId": 17 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-6192da40bdbd4619821046d81292d3c7", + "sha256": "7a4d5693fe950d8499727512a0c01f613c912b09ac020f4bfea558a347dc104f", + "size": 639, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328102247810", + "ids": { + "documentId": 20, + "documentVersionId": 20, + "embeddingJobId": 18, + "fileObjectId": 18 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-b29f691bda084cf783174e730f6e277f", + "sha256": "fdf9d7977772c42a6e4954dab9df0260a963f11c4475794d6d457279af48717a", + "size": 1045, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328103504958", + "ids": { + "documentId": 21, + "documentVersionId": 21, + "embeddingJobId": 19, + "fileObjectId": 19 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-abee4f99bb7a48bb9c763fdd49bd16f8", + "sha256": "409f56aa647a363708e7dfff63a795da589aba643c16578b7b8e8cf1c2b30010", + "size": 639, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328104848915", + "ids": { + "documentId": 22, + "documentVersionId": 22, + "embeddingJobId": 20, + "fileObjectId": 20 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-d73359977ccb452089a0bee66a0103ec", + "sha256": "5e5ef2b830e149e931c7284c1b4ff0daf9d491c4deb3464b8bf8bf95a69da16f", + "size": 1045, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328106308712", + "ids": { + "documentId": 23, + "documentVersionId": 23, + "embeddingJobId": 21, + "fileObjectId": 21 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-1cf0c03bbe594bb5827f2e273bc9cb14", + "sha256": "42701bbc7c882a301ccdc99ab8419a1ef05573f743a09d77cc4061e50d980643", + "size": 639, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328107545458", + "ids": { + "documentId": 24, + "documentVersionId": 24, + "embeddingJobId": 22, + "fileObjectId": 22 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-491bc1dea1d34d1fb175a4a78130b40d", + "sha256": "6d5d165797b911105c7313c58ad614efbc16e70ba568b99ea261d31929f807cd", + "size": 1045, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328108969233", + "ids": { + "documentId": 25, + "documentVersionId": 25, + "embeddingJobId": 23, + "fileObjectId": 23 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-1528b4c8ea56434286fd24803372273f", + "sha256": "6c5b22d9c0bc5e4d8b53e680fad8c1ffcdfdfed3f2077bfdaca176e81010409b", + "size": 639, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328110244934", + "ids": { + "documentId": 26, + "documentVersionId": 26, + "embeddingJobId": 24, + "fileObjectId": 24 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-5bbe80c1080d48dea97625043e1d6cb0", + "sha256": "4adfd5f972546b1dfa778665f20ffab54a1b862e8327f1b8a027b5c35ed41e90", + "size": 1045, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328111630812", + "ids": { + "documentId": 27, + "documentVersionId": 27, + "embeddingJobId": 25, + "fileObjectId": 25 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-134f03b858e8450b8b08c854a839ac90", + "sha256": "d80005d1c6c406a53d135ce7728671f5a50fa6497ffbbf969e7cb032629f2caa", + "size": 639, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328113382947", + "ids": { + "documentId": 28, + "documentVersionId": 28, + "embeddingJobId": 26, + "fileObjectId": 26 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-1289f0ebaa3f4751a0ad788c9b648ea3", + "sha256": "cf77c01b509b1129086a7b0aa93d64069b0bec875466228e24a5155c4677fa9b", + "size": 1045, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328114737996", + "ids": { + "documentId": 29, + "documentVersionId": 29, + "embeddingJobId": 27, + "fileObjectId": 27 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-2fb35853a420491c971f6c971b63896b", + "sha256": "7f923c7e4b2be2a18a9f4324f9c1d6269f2d01c74062645f5918b2e035b93076", + "size": 639, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328116002594", + "ids": { + "documentId": 30, + "documentVersionId": 30, + "embeddingJobId": 28, + "fileObjectId": 28 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-dd1a5e718ce24ce8b4508229da0951aa", + "sha256": "7630822014249f6f05b8fcc99ade977ac1ed1db491bd43a5198b8af046a8bcb5", + "size": 1045, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328117286515", + "ids": { + "documentId": 31, + "documentVersionId": 31, + "embeddingJobId": 29, + "fileObjectId": 29 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-c01c1678cb594cd993a9a0170bb18cd2", + "sha256": "0a05be12c9ee25da1b2bb3bc24fb51bed25970f9896f473067769bd4ac987680", + "size": 639, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328118488397", + "ids": { + "documentId": 32, + "documentVersionId": 32, + "embeddingJobId": 30, + "fileObjectId": 30 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-83040d6892d74b63ab768e9d90bb9d04", + "sha256": "d3d1f41587d7a417b5c9d6e1f7affd062395787a32b6e75043fb8f15f84e22ce", + "size": 1045, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328119765199", + "ids": { + "documentId": 33, + "documentVersionId": 33, + "embeddingJobId": 31, + "fileObjectId": 31 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-603a571ae0c147bea35927c304f9d772", + "sha256": "a871ca01a920e8762991610ff7e7c81ff7874d90214cc30ff20a95bdaba83899", + "size": 639, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328121047737", + "ids": { + "documentId": 34, + "documentVersionId": 34, + "embeddingJobId": 32, + "fileObjectId": 32 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-1a65c1542f684feeb034c3b1a6b92c37", + "sha256": "4588a9b4f22011102e8ffcca351041c4ad02e784d45cb753f1f312f25d5c0d5b", + "size": 1045, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "attempt_count": 0, + "chunk_count": 0, + "document_status": "UPLOADED", + "embedding_count": 0, + "generation": "1792328122347029", + "ids": { + "documentId": 35, + "documentVersionId": 35, + "embeddingJobId": 33, + "fileObjectId": 33 + }, + "job_status": "PENDING", + "relations_match": true, + "request_id": "upload-0e631fa0a6ea4ee1afac2ce5f99f4327", + "sha256": "38ea0ae6af7f1af5b8acc0bc844f34a8a8e05e7621a89725fc26c78f9ab6475e", + "size": 639, + "state": "RESTORED", + "version_status": "UPLOADED" + }, + { + "request_id": "upload-80ba1cb578334f68b4712ec46271edab", + "state": "DB_MISSING" + }, + { + "request_id": "upload-fbaf3249ead041aab8c8729f7cfe6bb5", + "state": "DB_MISSING" + }, + { + "request_id": "upload-f9cecc4bc6e645ebae7a7d8f0b60833f", + "state": "DB_MISSING" + }, + { + "request_id": "upload-266e9cd10bf44c4099c45351aff5a213", + "state": "DB_MISSING" + }, + { + "request_id": "upload-f385c020c7394b5993fc0d23dd0c4314", + "state": "DB_MISSING" + }, + { + "request_id": "upload-cd52989eefdc4d01993e077360b6e844", + "state": "DB_MISSING" + }, + { + "request_id": "upload-d008622649b04e9f871b3bd2296cc053", + "state": "DB_MISSING" + }, + { + "request_id": "upload-d3b7eec6c2394e2495f06d66006c6f00", + "state": "DB_MISSING" + }, + { + "request_id": "upload-665e96a9ada843efa78379b0e921ec48", + "state": "DB_MISSING" + }, + { + "request_id": "upload-046c77ba2e4a4b88ad0d1c6f7bf11bb9", + "state": "DB_MISSING" + }, + { + "request_id": "upload-607f1c2f64d64fb7b6632acc5f472d59", + "state": "DB_MISSING" + }, + { + "request_id": "upload-3ff63ce9c7b4402fbcbee28935a5d414", + "state": "DB_MISSING" + }, + { + "request_id": "upload-3a2ac04a9ac346d2b292f89ba0b096b3", + "state": "DB_MISSING" + }, + { + "request_id": "upload-24735b8cf98347f091b106994e210a7b", + "state": "DB_MISSING" + }, + { + "request_id": "upload-b0e8c7e2aea24562b5813170fa6a816d", + "state": "DB_MISSING" + }, + { + "request_id": "upload-04233cdad305433abee26af93d8356e6", + "state": "DB_MISSING" + }, + { + "request_id": "upload-3697e70376084b83a4129e6cc3d6d33f", + "state": "DB_MISSING" + }, + { + "request_id": "upload-23e0f579e82146679e7c3a58a1605aba", + "state": "DB_MISSING" + }, + { + "request_id": "upload-cf72758cb0624f18a5632f23c4419586", + "state": "DB_MISSING" + }, + { + "request_id": "upload-29c885e4f88d4e57aa18663fad3b240f", + "state": "DB_MISSING" + }, + { + "request_id": "upload-0ac416309fb24af199ca7e931bb45a7b", + "state": "DB_MISSING" + }, + { + "request_id": "upload-1daf93dff41c468f85bfdbbe70d6fcc0", + "state": "DB_MISSING" + }, + { + "request_id": "upload-4f5d46c32ba04d24957cc1780db113b4", + "state": "DB_MISSING" + }, + { + "request_id": "upload-e1cbe18e321045978d12d708efb76b72", + "state": "DB_MISSING" + }, + { + "request_id": "upload-e871accf911d4c9c9c1345f38362122b", + "state": "DB_MISSING" + }, + { + "request_id": "upload-04c4842d79894d72903e7b423e0d593a", + "state": "DB_MISSING" + }, + { + "request_id": "upload-09b3714d1ed542ecad7affd1258265e4", + "state": "DB_MISSING" + }, + { + "request_id": "upload-66a683aeca184c1a9fef9e2c5430947e", + "state": "DB_MISSING" + }, + { + "request_id": "upload-c278df614f444546967f1e1fa5ee1845", + "state": "DB_MISSING" + }, + { + "request_id": "upload-24f9a22d31904c4585030acce87e0473", + "state": "DB_MISSING" + }, + { + "request_id": "upload-28392863101e4285a08246980789e32f", + "state": "DB_MISSING" + }, + { + "request_id": "upload-3a06ef2536ca481fb0df95fd45d69bc6", + "state": "DB_MISSING" + } + ], + "repair_ran": false, + "source_access_used": false, + "worker_started": false +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/inputs.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/inputs.json new file mode 100644 index 00000000..40f57454 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/inputs.json @@ -0,0 +1,432 @@ +{ + "backup": { + "generation": "1792327475187159", + "id": "20261018T124224", + "sha256": "a2b2b179a02299978c2c67b5e8c3b5ba3c2e7216623c88f4231c9c636a259151", + "size": 142940160 + }, + "captured_at_utc": "2026-10-18T13:01:24.524257+00:00", + "checks": { + "backup_verified": true, + "identity_available_offsite": true, + "public_config_preserved": true, + "receiver_healthy": true, + "restore_target_isolated": true, + "rollback_ready": true, + "wal_chain_verified": true + }, + "identity_sha256": "4427d99d5a46a822deafc4effe1f5a901bd0b8e4d9a32e7d58fb851a5e25aaee", + "originals": [ + { + "fileObjectId": 1, + "generation": "1792327460164362", + "sha256": "1a8c22f7f7acce67d4544607b38e46ab4754d71cd4413e85533acfa1c30bb027", + "size": 1044 + }, + { + "fileObjectId": 2, + "generation": "1792327461460810", + "sha256": "1595721465e0cfb8d633de6b2a3674e7be455007aaf278d977f983be7ce5f609", + "size": 638 + }, + { + "fileObjectId": 3, + "generation": "1792327462725332", + "sha256": "6213e3cb37b48c59023d9bf356f7126f9f013358cb665acd23a3dc8613a50140", + "size": 1045 + }, + { + "fileObjectId": 4, + "generation": "1792327464013839", + "sha256": "671f658102aa034a78d956f412eb24e80919a2d2a25df92ae5c58c393317500f", + "size": 638 + }, + { + "fileObjectId": 5, + "generation": "1792327465201824", + "sha256": "c5f4b1afadfdbc24aefed3d4ef5786b495b09f83e957583be16d49fd876f7921", + "size": 1045 + }, + { + "fileObjectId": 6, + "generation": "1792328086019431", + "sha256": "6403d52e2ebc1bf1adabfd78a27dad188d674a61a25d3125864a896f3a18e3b8", + "size": 1044 + }, + { + "fileObjectId": 7, + "generation": "1792328087449527", + "sha256": "b5533ff9dc19e8ab13eceda011d203151c304adb6c15a76f9624c40e4f727229", + "size": 638 + }, + { + "fileObjectId": 8, + "generation": "1792328088900545", + "sha256": "17ca77586adbaf5282fa290b8daa01e087a06aa263602a51f4ae6557813e6a96", + "size": 1044 + }, + { + "fileObjectId": 9, + "generation": "1792328090136180", + "sha256": "389fdb5425c931b59623ed2ba6888f9e120609317fe8963245d88aec7304d929", + "size": 638 + }, + { + "fileObjectId": 10, + "generation": "1792328091427646", + "sha256": "ea06cf64bc7c18fd6379ee9fbc9eade765483eadbeee54799f7add448ad4288c", + "size": 1045 + }, + { + "fileObjectId": 11, + "generation": "1792328092722116", + "sha256": "2747f698cfa86d0f8f0873b9ec34139ab9a13df1a748a441538d2f764af69c4b", + "size": 639 + }, + { + "fileObjectId": 12, + "generation": "1792328094048871", + "sha256": "f052804612d30f76364e7c3e9103be400f0b1117eb36920e0c56de1eb80cc19d", + "size": 1045 + }, + { + "fileObjectId": 13, + "generation": "1792328095388030", + "sha256": "172c782cc9e625e2e54344a27890565cc8461a2aacc7c38e1b29d4197322a243", + "size": 639 + }, + { + "fileObjectId": 14, + "generation": "1792328096826330", + "sha256": "f29454ce42ad1c2047303fb13400aba5043e7483c49ab66c6294ac7677c5b566", + "size": 1045 + }, + { + "fileObjectId": 15, + "generation": "1792328097997840", + "sha256": "c5f84d6ccff80425b9dd6a301379c50c6c0ba91185f968280bf6eff1d36f758a", + "size": 639 + }, + { + "fileObjectId": 16, + "generation": "1792328099653018", + "sha256": "e23b60a24d1b2f413ac0b5a9d712b9f9df7a7c256b58344588fb5dc8b8a3baf2", + "size": 1045 + }, + { + "fileObjectId": 17, + "generation": "1792328101064159", + "sha256": "7a4d5693fe950d8499727512a0c01f613c912b09ac020f4bfea558a347dc104f", + "size": 639 + }, + { + "fileObjectId": 18, + "generation": "1792328102247810", + "sha256": "fdf9d7977772c42a6e4954dab9df0260a963f11c4475794d6d457279af48717a", + "size": 1045 + }, + { + "fileObjectId": 19, + "generation": "1792328103504958", + "sha256": "409f56aa647a363708e7dfff63a795da589aba643c16578b7b8e8cf1c2b30010", + "size": 639 + }, + { + "fileObjectId": 20, + "generation": "1792328104848915", + "sha256": "5e5ef2b830e149e931c7284c1b4ff0daf9d491c4deb3464b8bf8bf95a69da16f", + "size": 1045 + }, + { + "fileObjectId": 21, + "generation": "1792328106308712", + "sha256": "42701bbc7c882a301ccdc99ab8419a1ef05573f743a09d77cc4061e50d980643", + "size": 639 + }, + { + "fileObjectId": 22, + "generation": "1792328107545458", + "sha256": "6d5d165797b911105c7313c58ad614efbc16e70ba568b99ea261d31929f807cd", + "size": 1045 + }, + { + "fileObjectId": 23, + "generation": "1792328108969233", + "sha256": "6c5b22d9c0bc5e4d8b53e680fad8c1ffcdfdfed3f2077bfdaca176e81010409b", + "size": 639 + }, + { + "fileObjectId": 24, + "generation": "1792328110244934", + "sha256": "4adfd5f972546b1dfa778665f20ffab54a1b862e8327f1b8a027b5c35ed41e90", + "size": 1045 + }, + { + "fileObjectId": 25, + "generation": "1792328111630812", + "sha256": "d80005d1c6c406a53d135ce7728671f5a50fa6497ffbbf969e7cb032629f2caa", + "size": 639 + }, + { + "fileObjectId": 26, + "generation": "1792328113382947", + "sha256": "cf77c01b509b1129086a7b0aa93d64069b0bec875466228e24a5155c4677fa9b", + "size": 1045 + }, + { + "fileObjectId": 27, + "generation": "1792328114737996", + "sha256": "7f923c7e4b2be2a18a9f4324f9c1d6269f2d01c74062645f5918b2e035b93076", + "size": 639 + }, + { + "fileObjectId": 28, + "generation": "1792328116002594", + "sha256": "7630822014249f6f05b8fcc99ade977ac1ed1db491bd43a5198b8af046a8bcb5", + "size": 1045 + }, + { + "fileObjectId": 29, + "generation": "1792328117286515", + "sha256": "0a05be12c9ee25da1b2bb3bc24fb51bed25970f9896f473067769bd4ac987680", + "size": 639 + }, + { + "fileObjectId": 30, + "generation": "1792328118488397", + "sha256": "d3d1f41587d7a417b5c9d6e1f7affd062395787a32b6e75043fb8f15f84e22ce", + "size": 1045 + }, + { + "fileObjectId": 31, + "generation": "1792328119765199", + "sha256": "a871ca01a920e8762991610ff7e7c81ff7874d90214cc30ff20a95bdaba83899", + "size": 639 + }, + { + "fileObjectId": 32, + "generation": "1792328121047737", + "sha256": "4588a9b4f22011102e8ffcca351041c4ad02e784d45cb753f1f312f25d5c0d5b", + "size": 1045 + }, + { + "fileObjectId": 33, + "generation": "1792328122347029", + "sha256": "38ea0ae6af7f1af5b8acc0bc844f34a8a8e05e7621a89725fc26c78f9ab6475e", + "size": 639 + }, + { + "fileObjectId": 34, + "generation": "1792328384031780", + "sha256": "2b2330ff3f21d26f156091669add4e90615d0364196fa79bb1e6b43ad9d7b621", + "size": 1045 + }, + { + "fileObjectId": 35, + "generation": "1792328385296471", + "sha256": "33cff912087abdd6949f053df3efd3ee4cc3dc9f4410a9d610186511bf56a8da", + "size": 639 + }, + { + "fileObjectId": 36, + "generation": "1792328386910770", + "sha256": "bb8222fa45c741a91a26739414b68b4cab720cc2d938b0075016bc1627de5962", + "size": 1045 + }, + { + "fileObjectId": 37, + "generation": "1792328388171503", + "sha256": "09e6f22bc526b38bf65db51e22bdcd4c9c6ab2679644ab17a21b07cb6237e6dc", + "size": 639 + }, + { + "fileObjectId": 38, + "generation": "1792328389485824", + "sha256": "c6d0c699ab27b05a52f159dcd884648f6b7a504b1fe203139e47570379b634f6", + "size": 1045 + }, + { + "fileObjectId": 39, + "generation": "1792328390799793", + "sha256": "848b879592449d6780759eb458312a4118a971e0896bbbddb44d84fa7cf4cf42", + "size": 639 + }, + { + "fileObjectId": 40, + "generation": "1792328392073698", + "sha256": "df0f390b9cdaa5d6b0c69e1b438912bcc94ada13e9b0083fdde37bc743b69e09", + "size": 1045 + }, + { + "fileObjectId": 41, + "generation": "1792328393237369", + "sha256": "cfed5717eb7e18c65982136a31b8663eef977b9967baf0c3407d3f13239035b3", + "size": 639 + }, + { + "fileObjectId": 42, + "generation": "1792328394464378", + "sha256": "32cf92e6f18e615ea091b1beb7694cb399ed4b136f08846395c60f8575867572", + "size": 1045 + }, + { + "fileObjectId": 43, + "generation": "1792328395738784", + "sha256": "71936f694c856a446c6d291fee8592f6d00e0ec41782c14c9b2c43c78c99911f", + "size": 639 + }, + { + "fileObjectId": 44, + "generation": "1792328396953352", + "sha256": "602c72f4ccbba0a7902589bdb314791cdda40ab0bc2b955a3cc08de13b634fda", + "size": 1045 + }, + { + "fileObjectId": 45, + "generation": "1792328398174154", + "sha256": "b056a7db567d451a920fdbf7ccb00d57ed6e663ff4ab33a723354823ffdf0602", + "size": 639 + }, + { + "fileObjectId": 46, + "generation": "1792328399451108", + "sha256": "3978a9d63e633c2b1cfa2514a6e4ea14839c757483acf2fa8017babdd62b431b", + "size": 1045 + }, + { + "fileObjectId": 47, + "generation": "1792328400811689", + "sha256": "aa206070315b0964462c62379da5396667d4b96260c3e89d9f73a7dff03f59a1", + "size": 639 + }, + { + "fileObjectId": 48, + "generation": "1792328402070421", + "sha256": "a6dcbc76355bcc39db7c2d9d126c7545b1916931348cb7275c6233aa8489a8a9", + "size": 1045 + }, + { + "fileObjectId": 49, + "generation": "1792328403368873", + "sha256": "2adfe330ad106f99f0a774e0582caa3d56731285a6202b2bb3bc24724f830938", + "size": 639 + }, + { + "fileObjectId": 50, + "generation": "1792328404687457", + "sha256": "c60d6cf690c9e6ca2455ee2124f5b96bb59320da7a327d49b4fb5ddc1664f5fe", + "size": 1045 + }, + { + "fileObjectId": 51, + "generation": "1792328405929490", + "sha256": "20fbdeab3b3b91276cb3300ce4b5b414c8f75b4d2d218bd92f446476188cc24c", + "size": 639 + }, + { + "fileObjectId": 52, + "generation": "1792328407203767", + "sha256": "077ca2cd195eca9abb971513cb7cae2129b4ae233ac03adf4cac21fb6330c34e", + "size": 1045 + }, + { + "fileObjectId": 53, + "generation": "1792328408450431", + "sha256": "0b20c2871e345692b751a8e04705ddbced533266963b8f9a7b89fa678cfc1776", + "size": 639 + }, + { + "fileObjectId": 54, + "generation": "1792328409828439", + "sha256": "394d49de7b07c54a9b04801b1f57dbe1549ec0ec93f6048a831a1d3bf6daa923", + "size": 1046 + }, + { + "fileObjectId": 55, + "generation": "1792328411052970", + "sha256": "202241ea917fa4c556f9a12f7f4828bdb13243c693ecc7602955653ee1ae0afc", + "size": 639 + }, + { + "fileObjectId": 56, + "generation": "1792328412581686", + "sha256": "d4a894bed85f5339e721740e2163666e88d52743d15b4026953c242ad520c3ca", + "size": 1045 + }, + { + "fileObjectId": 57, + "generation": "1792328413860674", + "sha256": "636cabda006f3854054325b933df56c549af856d1b7287ddbfd23f02d7558463", + "size": 639 + }, + { + "fileObjectId": 58, + "generation": "1792328415146526", + "sha256": "5428bf31e08c974068911906aa9eb2bedcaacf95e032e1cfbe7244fbbf95514e", + "size": 1045 + }, + { + "fileObjectId": 59, + "generation": "1792328416443892", + "sha256": "3a39dcb938ccaa58f17bb27ea56d5c6e905607193b2ad872b2b3bd5005e3a26f", + "size": 639 + }, + { + "fileObjectId": 60, + "generation": "1792328417847807", + "sha256": "b2dd6bb46d8bc4eaa85c4b896744b8cb5203cef0a46d978a49b346aec7a21d3d", + "size": 1045 + } + ], + "schema_version": 1, + "wals": [ + { + "generation": "1792327483514667", + "name": "00000006.history", + "sha256": "7dbe25c372b72e1132dca125c614a696011f1aeb3c1dfdac9cc8e19055a6d3c6", + "size": 209 + }, + { + "generation": "1792327486114407", + "name": "00000006000000000000001E", + "sha256": "f6b03616fc67281e7772e125edf5e18e65fe999409844f564158f8581456b2b5", + "size": 16777216 + }, + { + "generation": "1792327490310809", + "name": "00000006000000000000001F", + "sha256": "99e9c17abdeaff81a84125179fafd7f43f702d7f5a3251b9f8ca9987380c135c", + "size": 16777216 + }, + { + "generation": "1792327494374579", + "name": "000000060000000000000020", + "sha256": "cca03fedad9e7d5e7f9e9c585c27abb96d42cb6205c1b13d2c62e5ab7e9fb3d7", + "size": 16777216 + }, + { + "generation": "1792328125375427", + "name": "000000060000000000000021", + "sha256": "7b20f95786aa9238fbb93dfd399216c034d93392d22559fea2fda917551d9e8b", + "size": 16777216 + }, + { + "generation": "1792328129379945", + "name": "000000060000000000000022", + "sha256": "d3172ba08c364acda7721e4cbab32bdff67735550cf47002648dce40aac6d50f", + "size": 16777216 + }, + { + "generation": "1792328133749894", + "name": "000000060000000000000023", + "sha256": "852b7974ecf0ba081e258df1c44f6419469f97184617452a38b9e4e7f5840015", + "size": 16777216 + }, + { + "generation": "1792328420319071", + "name": "000000060000000000000024", + "sha256": "e204570a27cdb84ec75831c7626e4e2874c0051683ed1180368ca7f3c40f8db7", + "size": 16777216 + } + ] +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/manifest.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/manifest.json new file mode 100644 index 00000000..5884f47f --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/manifest.json @@ -0,0 +1,6 @@ +{ + "created_at_utc": "2026-10-18T12:31:37.286+00:00", + "revision": "3bff887a5f6abdf4155957a658599f8f0366a141", + "run_id": "DR-RESTORE-20261018-FORMAL1", + "schema_version": 1 +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/preparation-result.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/preparation-result.json new file mode 100644 index 00000000..4c1f5ee9 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/preparation-result.json @@ -0,0 +1,6 @@ +{ + "finished_at_utc": "2026-10-18T12:44:55.932528+00:00", + "formal_recovery_executed": false, + "preparation": "PASS", + "run_id": "DR-RESTORE-20261018-FORMAL1" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/runtime-cleanup-observations.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/runtime-cleanup-observations.json new file mode 100644 index 00000000..14e9be8f --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/runtime-cleanup-observations.json @@ -0,0 +1,34 @@ +{ + "containers_removed": true, + "failures": [], + "finished_at_utc": "2026-10-18T13:13:05.675975+00:00", + "gcs": { + "active_objects_remaining": 0, + "deleted_active_objects": { + "database": 10, + "preserved": 61, + "source": 65 + }, + "immediate_physical_erasure_claimed": false, + "soft_delete_policy_may_retain_deleted_objects": true + }, + "private_runtime_data_removed": true, + "processes_removed": true, + "public_config_unchanged": true, + "shared_trial_buckets_retained_for_next_serial_run": true, + "source_objects_removed": true, + "ssh_keys_removed": true, + "started_at_utc": "2026-10-18T13:11:30.221813+00:00", + "target": { + "guard": { + "completed": true, + "patroni_resumed": 2, + "scratch_stopped": true + }, + "scratch_removed": true, + "scratch_stopped": true, + "source_cluster_restored": true, + "target_never_borrowed": false, + "temporary_key_removed": true + } +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/summary.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/summary.json new file mode 100644 index 00000000..1c76364d --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-FORMAL1/summary.json @@ -0,0 +1,975 @@ +{ + "execution": "PASS", + "final_audit": { + "acknowledged": 65, + "activation_allowed": true, + "http_error": 0, + "lost_or_incomplete_acknowledged": 32, + "requests": [ + { + "ack_at_utc": "2026-10-18T12:37:05.356+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-965c69a074e94cacb37ed7cd2f1cfb2a", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:37:50.492+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3a043da834f14436b70a14733f2252b8", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:38:29.983+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-aa478ffa2f2a43acb43c84ce2d8ac431", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:39:12.453+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-9ca175c82653458b92bed4bcd9537a5f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:41:01.038+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-9e7df4353fed4e7aaa5a3c7f1408e4e5", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:25.127+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c9166086f88c466d8a756e6f6d9622c5", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:34.772+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-66ba7d8a8de148629607f6a26bbc8f90", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:46.064+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-99d323fbaa964760a0b71ef6412e025d", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:57.093+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-b331678e5a7d42cf8a4c4db431cd19bf", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:08.644+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0ba3a720b65e417f807def554042667e", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:20.001+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-49eb7ad416b24429afb537f16ae6333f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:31.133+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c7f7b9de82414e0889a3e52223279673", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:41.632+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3332040234024f5fab7f4545b4bb1f9f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:52.682+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0935757b0cd5475ab423f04174c1a17b", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:03.643+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-dd0d63abe92b4a9192a897b7af26e314", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:16.097+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-4ae3860380ef4bfcb24944535960303e", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:25.209+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-6192da40bdbd4619821046d81292d3c7", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:37.046+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-b29f691bda084cf783174e730f6e277f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:47.680+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-abee4f99bb7a48bb9c763fdd49bd16f8", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:58.226+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-d73359977ccb452089a0bee66a0103ec", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:09.841+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1cf0c03bbe594bb5827f2e273bc9cb14", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:21.081+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-491bc1dea1d34d1fb175a4a78130b40d", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:31.532+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1528b4c8ea56434286fd24803372273f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:42.854+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-5bbe80c1080d48dea97625043e1d6cb0", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:53.221+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-134f03b858e8450b8b08c854a839ac90", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:04.581+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1289f0ebaa3f4751a0ad788c9b648ea3", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:15.508+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-2fb35853a420491c971f6c971b63896b", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:27.678+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-dd1a5e718ce24ce8b4508229da0951aa", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:37.267+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c01c1678cb594cd993a9a0170bb18cd2", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:48.798+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-83040d6892d74b63ab768e9d90bb9d04", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:59.902+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-603a571ae0c147bea35927c304f9d772", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:54:10.932+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1a65c1542f684feeb034c3b1a6b92c37", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:54:21.832+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0e631fa0a6ea4ee1afac2ce5f99f4327", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:55:43.909+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-80ba1cb578334f68b4712ec46271edab", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:55:50.968+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-fbaf3249ead041aab8c8729f7cfe6bb5", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:55:58.082+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-f9cecc4bc6e645ebae7a7d8f0b60833f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:05.900+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-266e9cd10bf44c4099c45351aff5a213", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:13.259+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-f385c020c7394b5993fc0d23dd0c4314", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:20.688+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-cd52989eefdc4d01993e077360b6e844", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:28.515+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-d008622649b04e9f871b3bd2296cc053", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:36.324+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-d3b7eec6c2394e2495f06d66006c6f00", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:44.441+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-665e96a9ada843efa78379b0e921ec48", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:51.804+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-046c77ba2e4a4b88ad0d1c6f7bf11bb9", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:59.481+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-607f1c2f64d64fb7b6632acc5f472d59", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:08.320+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3ff63ce9c7b4402fbcbee28935a5d414", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:16.165+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3a2ac04a9ac346d2b292f89ba0b096b3", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:24.035+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-24735b8cf98347f091b106994e210a7b", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:31.231+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-b0e8c7e2aea24562b5813170fa6a816d", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:38.977+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-04233cdad305433abee26af93d8356e6", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:47.528+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3697e70376084b83a4129e6cc3d6d33f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:56.017+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-23e0f579e82146679e7c3a58a1605aba", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:03.981+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-cf72758cb0624f18a5632f23c4419586", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:11.999+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-29c885e4f88d4e57aa18663fad3b240f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:19.128+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0ac416309fb24af199ca7e931bb45a7b", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:27.247+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1daf93dff41c468f85bfdbbe70d6fcc0", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:35.374+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-4f5d46c32ba04d24957cc1780db113b4", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:46.615+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-e1cbe18e321045978d12d708efb76b72", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:56.309+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-e871accf911d4c9c9c1345f38362122b", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:59:07.914+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-04c4842d79894d72903e7b423e0d593a", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:59:18.865+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-09b3714d1ed542ecad7affd1258265e4", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:30.969+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-66a683aeca184c1a9fef9e2c5430947e", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:38.813+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c278df614f444546967f1e1fa5ee1845", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:46.695+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-24f9a22d31904c4585030acce87e0473", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:54.537+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-28392863101e4285a08246980789e32f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:01:03.210+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3a06ef2536ca481fb0df95fd45d69bc6", + "restored_state": "DB_MISSING" + } + ], + "restored_acknowledged": 33, + "unknown_http": 0 + }, + "functional_checks": { + "download": { + "at_utc": "2026-10-18T13:04:21.530770+00:00", + "from_cutoff_seconds": 176.992, + "passed": true, + "samples": 4 + }, + "job_resume": { + "at_utc": "2026-10-18T13:05:23.022354+00:00", + "from_cutoff_seconds": 238.483, + "passed": true, + "samples": 1 + }, + "new_download": { + "at_utc": "2026-10-18T13:10:57.724793+00:00", + "from_cutoff_seconds": 573.186, + "passed": true, + "samples": 1 + }, + "new_indexing": { + "at_utc": "2026-10-18T13:10:55.947040+00:00", + "from_cutoff_seconds": 571.408, + "passed": true, + "samples": 1 + }, + "new_upload": { + "at_utc": "2026-10-18T13:05:30.661209+00:00", + "from_cutoff_seconds": 246.122, + "passed": true, + "samples": 1 + }, + "search": { + "at_utc": "2026-10-18T13:04:37.379579+00:00", + "from_cutoff_seconds": 192.841, + "passed": true, + "samples": 1 + } + }, + "initial_audit": { + "acknowledged": 65, + "activation_allowed": true, + "http_error": 0, + "lost_or_incomplete_acknowledged": 32, + "requests": [ + { + "ack_at_utc": "2026-10-18T12:37:05.356+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-965c69a074e94cacb37ed7cd2f1cfb2a", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:37:50.492+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3a043da834f14436b70a14733f2252b8", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:38:29.983+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-aa478ffa2f2a43acb43c84ce2d8ac431", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:39:12.453+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-9ca175c82653458b92bed4bcd9537a5f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:41:01.038+00:00", + "cohort": "functional", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-9e7df4353fed4e7aaa5a3c7f1408e4e5", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:25.127+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c9166086f88c466d8a756e6f6d9622c5", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:34.772+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-66ba7d8a8de148629607f6a26bbc8f90", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:46.064+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-99d323fbaa964760a0b71ef6412e025d", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:49:57.093+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-b331678e5a7d42cf8a4c4db431cd19bf", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:08.644+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0ba3a720b65e417f807def554042667e", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:20.001+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-49eb7ad416b24429afb537f16ae6333f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:31.133+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c7f7b9de82414e0889a3e52223279673", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:41.632+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3332040234024f5fab7f4545b4bb1f9f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:50:52.682+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0935757b0cd5475ab423f04174c1a17b", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:03.643+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-dd0d63abe92b4a9192a897b7af26e314", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:16.097+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-4ae3860380ef4bfcb24944535960303e", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:25.209+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-6192da40bdbd4619821046d81292d3c7", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:37.046+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-b29f691bda084cf783174e730f6e277f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:47.680+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-abee4f99bb7a48bb9c763fdd49bd16f8", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:51:58.226+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-d73359977ccb452089a0bee66a0103ec", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:09.841+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1cf0c03bbe594bb5827f2e273bc9cb14", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:21.081+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-491bc1dea1d34d1fb175a4a78130b40d", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:31.532+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1528b4c8ea56434286fd24803372273f", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:42.854+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-5bbe80c1080d48dea97625043e1d6cb0", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:52:53.221+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-134f03b858e8450b8b08c854a839ac90", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:04.581+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1289f0ebaa3f4751a0ad788c9b648ea3", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:15.508+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-2fb35853a420491c971f6c971b63896b", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:27.678+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-dd1a5e718ce24ce8b4508229da0951aa", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:37.267+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c01c1678cb594cd993a9a0170bb18cd2", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:48.798+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-83040d6892d74b63ab768e9d90bb9d04", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:53:59.902+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-603a571ae0c147bea35927c304f9d772", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:54:10.932+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1a65c1542f684feeb034c3b1a6b92c37", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:54:21.832+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0e631fa0a6ea4ee1afac2ce5f99f4327", + "restored_state": "RESTORED" + }, + { + "ack_at_utc": "2026-10-18T12:55:43.909+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-80ba1cb578334f68b4712ec46271edab", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:55:50.968+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-fbaf3249ead041aab8c8729f7cfe6bb5", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:55:58.082+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-f9cecc4bc6e645ebae7a7d8f0b60833f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:05.900+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-266e9cd10bf44c4099c45351aff5a213", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:13.259+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-f385c020c7394b5993fc0d23dd0c4314", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:20.688+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-cd52989eefdc4d01993e077360b6e844", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:28.515+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-d008622649b04e9f871b3bd2296cc053", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:36.324+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-d3b7eec6c2394e2495f06d66006c6f00", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:44.441+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-665e96a9ada843efa78379b0e921ec48", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:51.804+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-046c77ba2e4a4b88ad0d1c6f7bf11bb9", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:56:59.481+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-607f1c2f64d64fb7b6632acc5f472d59", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:08.320+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3ff63ce9c7b4402fbcbee28935a5d414", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:16.165+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3a2ac04a9ac346d2b292f89ba0b096b3", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:24.035+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-24735b8cf98347f091b106994e210a7b", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:31.231+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-b0e8c7e2aea24562b5813170fa6a816d", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:38.977+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-04233cdad305433abee26af93d8356e6", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:47.528+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3697e70376084b83a4129e6cc3d6d33f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:57:56.017+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-23e0f579e82146679e7c3a58a1605aba", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:03.981+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-cf72758cb0624f18a5632f23c4419586", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:11.999+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-29c885e4f88d4e57aa18663fad3b240f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:19.128+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-0ac416309fb24af199ca7e931bb45a7b", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:27.247+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-1daf93dff41c468f85bfdbbe70d6fcc0", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:35.374+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-4f5d46c32ba04d24957cc1780db113b4", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:46.615+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-e1cbe18e321045978d12d708efb76b72", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:58:56.309+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-e871accf911d4c9c9c1345f38362122b", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:59:07.914+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-04c4842d79894d72903e7b423e0d593a", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T12:59:18.865+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-09b3714d1ed542ecad7affd1258265e4", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:30.969+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-66a683aeca184c1a9fef9e2c5430947e", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:38.813+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-c278df614f444546967f1e1fa5ee1845", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:46.695+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-24f9a22d31904c4585030acce87e0473", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:00:54.537+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-28392863101e4285a08246980789e32f", + "restored_state": "DB_MISSING" + }, + { + "ack_at_utc": "2026-10-18T13:01:03.210+00:00", + "cohort": "loss", + "http_outcome": "ACKNOWLEDGED", + "request_id": "upload-3a06ef2536ca481fb0df95fd45d69bc6", + "restored_state": "DB_MISSING" + } + ], + "restored_acknowledged": 33, + "unknown_http": 0 + }, + "reason": null, + "rpo_guarantee": "NOT_EVALUATED", + "run_id": "DR-RESTORE-20261018-FORMAL1", + "scope": "isolated_document_download_search_indexing" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-REHEARSAL1/operator-restore-error.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-REHEARSAL1/operator-restore-error.json new file mode 100644 index 00000000..d643bb96 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-REHEARSAL1/operator-restore-error.json @@ -0,0 +1,6 @@ +{ + "at_utc": "2026-10-18T11:53:22.026246+00:00", + "error_type": "RuntimeError", + "stage": "LICENSED_TARGET_START_FAILED", + "started_at_utc": "2026-10-18T11:51:39.537554+00:00" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-REHEARSAL1/ssh-cleanup-reconciliation-r2.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-REHEARSAL1/ssh-cleanup-reconciliation-r2.json new file mode 100644 index 00000000..d8087ab5 --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-REHEARSAL1/ssh-cleanup-reconciliation-r2.json @@ -0,0 +1,10 @@ +{ + "at_utc": "2026-10-18T11:57:58.120273+00:00", + "existing_unrelated_keys_changed": false, + "key_values_retained_or_printed": false, + "matching_keys_removed": 0, + "matching_owned_expiry_window_keys": 0, + "no_matching_key_remaining": true, + "purpose": "만료된 시험 전용 터널 키 제거 결과의 신선한 프로필 재확인", + "run_id": "DR-SSH-CLEANUP-20261018-R2" +} diff --git a/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-REHEARSAL1/summary.json b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-REHEARSAL1/summary.json new file mode 100644 index 00000000..76fa93cb --- /dev/null +++ b/docs/test-results/dr-functional-recovery-20261018/runs/DR-RESTORE-20261018-REHEARSAL1/summary.json @@ -0,0 +1,10 @@ +{ + "execution": "INCOMPLETE", + "final_audit": null, + "functional_checks": {}, + "initial_audit": null, + "reason": "OPERATOR_RESTORE_FAILED", + "rpo_guarantee": "NOT_EVALUATED", + "run_id": "DR-RESTORE-20261018-REHEARSAL1", + "scope": "isolated_document_download_search_indexing" +} diff --git a/scripts/opensql/dr_recovery.py b/scripts/opensql/dr_recovery.py index 0ed264c6..9d2dacdf 100755 --- a/scripts/opensql/dr_recovery.py +++ b/scripts/opensql/dr_recovery.py @@ -28,6 +28,13 @@ "identity_available_offsite", "restore_target_isolated", "rollback_ready", "public_config_preserved") FUNCTIONS = ("download", "search", "job_resume", "new_upload", "new_indexing", "new_download") +# Match each Java enum independently: a Job status is not a DocumentVersion status. +SNAPSHOT_STATUSES = { + "document_status": {"DRAFT", "UPLOADED", "INDEXING", "INDEXED", "FAILED", "ARCHIVED", "DELETED"}, + "version_status": {"UPLOADED", "PARSING", "CHUNKED", "EMBEDDING", "INDEXED", "FAILED"}, + "job_status": {"PENDING", "PROCESSING", "INDEXED", "FAILED", "CANCELED"}, +} + class ContractError(ValueError): @@ -275,8 +282,8 @@ def audit_records(records, snapshot, *, pristine=True, originals=None): for key in ("attempt_count", "chunk_count", "embedding_count"): if key in row and (type(row[key]) is not int or row[key] < 0): raise ContractError("INVALID_SNAPSHOT_COUNTER") - for key in ("document_status", "version_status", "job_status"): - if key in row and row[key] not in ("UPLOADED", "PENDING", "PROCESSING", "INDEXED", "FAILED", "DELETED"): + for key, statuses in SNAPSHOT_STATUSES.items(): + if key in row and row[key] not in statuses: raise ContractError("INVALID_SNAPSHOT_STATUS") key = row.get("request_id") if key in indexed or key not in {r["request_id"] for r in records}: diff --git a/scripts/opensql/test_dr_recovery.py b/scripts/opensql/test_dr_recovery.py index 909f9b95..2314aee5 100644 --- a/scripts/opensql/test_dr_recovery.py +++ b/scripts/opensql/test_dr_recovery.py @@ -12,7 +12,7 @@ class RecoveryContractTest(unittest.TestCase): - """Keep first-restoration evidence immutable before any simulated Worker activation.""" + """Validate first-restoration ordering and status compatibility with actual Java enums.""" def setUp(self): self.tmp = tempfile.TemporaryDirectory() @@ -221,6 +221,39 @@ def test_racing_response_after_audit_is_rejected_when_replayed(self): with self.assertRaisesRegex(dr.ContractError, 'RESPONSE_AFTER_AUDIT'): dr.requests(self.run) + def assert_wire_statuses(self, field, enum_path): + # Java enum values are an independent input contract, not the Python allow-list. + import re + text = (Path(__file__).resolve().parents[2] / enum_path).read_text() + body = text.split('public enum ', 1)[1].split('{', 1)[1].split('}', 1)[0] + values = re.findall(r'^\s*([A-Z][A-Z_]+)\s*[,;]?\s*$', body, re.MULTILINE) + self.assertTrue(values) + self.sent() + for value in values: + with self.subTest(field=field, value=value): + snapshot = self.snapshot(); snapshot['records'][0][field] = value + self.assertTrue(dr.audit_records(dr.requests(self.run), snapshot)['activation_allowed']) + + def test_document_status_wire_contract(self): + self.assert_wire_statuses('document_status', + 'backend/src/main/java/com/opensource/docgrid/domain/document/enums/DocumentStatus.java') + + def test_version_status_wire_contract(self): + self.assert_wire_statuses('version_status', + 'backend/src/main/java/com/opensource/docgrid/domain/document/enums/DocumentVersionStatus.java') + + def test_job_status_wire_contract(self): + self.assert_wire_statuses('job_status', + 'backend/src/main/java/com/opensource/docgrid/domain/embedding/enums/EmbeddingJobStatus.java') + + def test_status_from_another_entity_is_rejected(self): + self.sent() + for field, value in [('document_status', 'PENDING'), ('version_status', 'PROCESSING'), ('job_status', 'UPLOADED')]: + with self.subTest(field=field): + snapshot = self.snapshot(); snapshot['records'][0][field] = value + with self.assertRaisesRegex(dr.ContractError, 'INVALID_SNAPSHOT_STATUS'): + dr.audit_records(dr.requests(self.run), snapshot) + def test_object_missing_blocks_worker(self): self.sent(); result = dr.audit_records(dr.requests(self.run), self.snapshot({"r1": "OBJECT_MISSING"})) self.assertFalse(result["activation_allowed"])