From 223d7eaec9b5ebe5c6e161bff5a0bf81fcc32473 Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Wed, 28 Oct 2026 18:41:58 +0530 Subject: [PATCH 01/12] feat: add a full wipe command also, `context_chat:reindex` now also schedules the initial import of content providers if they have not been initialised Signed-off-by: kyteinsky Assisted-by: Github Copilot:claude-opus-5-5 --- .github/workflows/integration-test-cron.yml | 39 ++++ appinfo/info.xml | 1 + lib/Command/Reindex.php | 18 +- lib/Command/Wipe.php | 79 ++++++++ lib/Service/LangRopeService.php | 10 + lib/Service/ProviderConfigService.php | 14 ++ lib/Service/ReindexService.php | 107 +++++++++++ tests/integration/ReindexServiceTest.php | 191 ++++++++++++++++++++ 8 files changed, 449 insertions(+), 10 deletions(-) create mode 100644 lib/Command/Wipe.php create mode 100644 lib/Service/ReindexService.php create mode 100644 tests/integration/ReindexServiceTest.php diff --git a/.github/workflows/integration-test-cron.yml b/.github/workflows/integration-test-cron.yml index d457e561..b62ebb03 100644 --- a/.github/workflows/integration-test-cron.yml +++ b/.github/workflows/integration-test-cron.yml @@ -405,6 +405,45 @@ jobs: kill -9 $WORKER1_PID kill -9 $WORKER2_PID + - name: Wipe all the indexed data and check that the queues and the vector DB are empty + run: | + ./occ context_chat:wipe --force + + stats=$(./occ context_chat:stats --json) + echo "$stats" + indexed=$(echo "$stats" | jq '[.vectordb_document_counts | to_entries[].value] | add // 0') + queued=$(echo "$stats" | jq '[.queued_documents_counts | to_entries[].value] | add // 0') + actions=$(echo "$stats" | jq '.queued_actions_count') + fs_events=$(echo "$stats" | jq '.queued_fs_events_count') + # the queued counts leave out locked items, check those separately + queued_locked=$(echo "$stats" | jq '[.queued_documents_locked_counts | to_entries[].value] | add // 0') + actions_locked=$(echo "$stats" | jq '.queued_actions_locked_count') + backend_available=$(echo "$stats" | jq '.backend_available') + echo "indexed=$indexed queued=$queued queued_locked=$queued_locked actions=$actions actions_locked=$actions_locked fs_events=$fs_events backend_available=$backend_available" + if [ "$backend_available" != "true" ] || [ "$indexed" != "0" ] || [ "$queued" != "0" ] || [ "$queued_locked" != "0" ] || [ "$actions" != "0" ] || [ "$actions_locked" != "0" ] || [ "$fs_events" != "0" ]; then + echo "Error: expected an empty vector DB and empty queues after the wipe" + exit 1 + fi + + - name: Reindex and check that documents get indexed again + run: | + ./occ context_chat:reindex + echo "::group::Waiting up to 10 minutes for documents to be indexed again" + for i in {1..60}; do + stats=$(timeout 30 ./occ context_chat:stats --json || true) + indexed=$(echo "$stats" | jq '.vectordb_document_counts.files__default // 0' 2>/dev/null || echo 0) + echo "Attempt $i: indexed=$indexed" + if [ "$indexed" -ge 10 ]; then + echo "Reindex works, $indexed documents indexed again" + echo "::endgroup::" + exit 0 + fi + sleep 10 + done + echo "::endgroup::" + echo "Error: documents were not indexed again after the reindex" + exit 1 + - name: Show nextcloud logs if: always() run: | diff --git a/appinfo/info.xml b/appinfo/info.xml index 2b8cda78..7075cb78 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -47,6 +47,7 @@ Refer to the [Context Chat Backend's readme](https://github.com/nextcloud/contex OCA\ContextChat\Command\Search OCA\ContextChat\Command\Statistics OCA\ContextChat\Command\Reindex + OCA\ContextChat\Command\Wipe diff --git a/lib/Command/Reindex.php b/lib/Command/Reindex.php index ca3de909..73b322b0 100644 --- a/lib/Command/Reindex.php +++ b/lib/Command/Reindex.php @@ -9,8 +9,7 @@ namespace OCA\ContextChat\Command; -use OCA\ContextChat\BackgroundJobs\SchedulerJob; -use OCP\BackgroundJob\IJobList; +use OCA\ContextChat\Service\ReindexService; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -20,31 +19,30 @@ * * SchedulerJob -> StorageCrawlJob -> IndexerJob is seeded only by the repair step and * removes itself once the initial crawl finishes. There is otherwise no way to re-enumerate - * mounts (e.g. after installing on an instance whose files predate the app, or to recover a crawl - * that did not complete) short of reinstalling the app. This command re-adds SchedulerJob so the - * full enumeration runs again; it is a no-op if one is already scheduled, and already-indexed - * files are skipped. + * mounts (e.g. to recover a crawl that did not complete, or after context_chat:wipe) short of + * reinstalling the app. This command re-adds SchedulerJob so the full enumeration runs again if one + * doesn't exist already. + * Content providers' initial import also gets scheduled (e.g. after context_chat:wipe) if it has not run yet. */ class Reindex extends Command { public function __construct( - private IJobList $jobList, + private ReindexService $reindexService, ) { parent::__construct(); } protected function configure() { $this->setName('context_chat:reindex') - ->setDescription('Schedule a full re-crawl of all the files in all the mounts. Indexed files are not re-indexed when compared against context_chat_backend\'s vector DB. Content providers are not re-indexed.'); + ->setDescription('Schedule a full re-crawl of all the files in all the mounts. Indexed files are not re-indexed when compared against context_chat_backend\'s vector DB. Content providers are only re-imported after context_chat:wipe.'); } protected function execute(InputInterface $input, OutputInterface $output): int { - if ($this->jobList->has(SchedulerJob::class, null)) { + if (!$this->reindexService->scheduleReindex()) { $output->writeln('A full re-crawl is already scheduled; nothing to do.'); return 0; } - $this->jobList->add(SchedulerJob::class); $output->writeln('Scheduled a full re-crawl. SchedulerJob will enumerate all mounts on its next run.'); return 0; } diff --git a/lib/Command/Wipe.php b/lib/Command/Wipe.php new file mode 100644 index 00000000..c5960d9b --- /dev/null +++ b/lib/Command/Wipe.php @@ -0,0 +1,79 @@ +setName('context_chat:wipe') + ->setDescription('Delete all the indexed documents from context_chat_backend\'s vector DB, clear the indexing and actions queues and reset the indexing statistics. Nothing gets indexed again until context_chat:reindex is run or the --reindex option is used.') + ->addOption( + 'reindex', + null, + InputOption::VALUE_NONE, + 'Schedule a full re-crawl of all the files and a re-import of all the content providers after the wipe', + ) + ->addOption( + 'force', + 'f', + InputOption::VALUE_NONE, + 'Do not ask for confirmation', + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int { + /** @var QuestionHelper $helper */ + $helper = $this->getHelper('question'); + if (!$input->getOption('force')) { + $question = new ConfirmationQuestion('All the indexed documents in Context Chat will be deleted. The original documents are not touched. Continue? [y/N] ', false); + if (!$helper->ask($input, $output, $question)) { + $output->writeln('Aborted, nothing was deleted. Use --force to skip the confirmation.'); + return 1; + } + } + + try { + $this->reindexService->wipe(); + } catch (\Throwable $e) { + $output->writeln('Wipe failed: ' . $e->getMessage() . ''); + $output->writeln('The command can be safely run again once the problem is fixed.'); + return 1; + } + $output->writeln('Deleted all the indexed documents and cleared the queues.'); + + if ($input->getOption('reindex')) { + $this->reindexService->scheduleReindex(); + $output->writeln('Scheduled a full re-crawl of files and re-import of the content providers.'); + } else { + $question = new ConfirmationQuestion('Re-index the documents now? [Y/n] ', default: true); + if ($helper->ask($input, $output, $question)) { + $this->reindexService->scheduleReindex(); + $output->writeln('Scheduled a full re-crawl of files and re-import of the content providers.'); + } else { + $output->writeln('Run context_chat:reindex to index the documents again later.'); + } + } + return 0; + } +} diff --git a/lib/Service/LangRopeService.php b/lib/Service/LangRopeService.php index 96150bcc..26339916 100644 --- a/lib/Service/LangRopeService.php +++ b/lib/Service/LangRopeService.php @@ -224,6 +224,16 @@ public function deleteProvider(string $providerKey): void { $this->requestToExApp('/deleteProvider', 'POST', $params); } + /** + * Delete all the indexed documents from the backend's vector DB + * + * @return void + * @throws \RuntimeException + */ + public function deleteAll(): void { + $this->requestToExApp('/deleteAll', 'POST'); + } + /** * @param string $userId * @return void diff --git a/lib/Service/ProviderConfigService.php b/lib/Service/ProviderConfigService.php index 344c98e0..a5d8ba2b 100644 --- a/lib/Service/ProviderConfigService.php +++ b/lib/Service/ProviderConfigService.php @@ -112,6 +112,20 @@ public function updateProvider( $this->appConfig->setAppValueString('providers', json_encode($providers), lazy: true); } + /** + * Mark all the content providers, except the files provider, as not initiated + * so their initial import runs again + */ + public function resetProvidersInitiation(): void { + $providers = $this->getProviders(); + foreach ($providers as $key => $provider) { + if ($key !== self::getDefaultProviderKey()) { + $providers[$key]['isInitiated'] = false; + } + } + $this->appConfig->setAppValueString('providers', json_encode($providers), lazy: true); + } + /** * @param string $appId * @param ?string $providerId diff --git a/lib/Service/ReindexService.php b/lib/Service/ReindexService.php new file mode 100644 index 00000000..598dc5de --- /dev/null +++ b/lib/Service/ReindexService.php @@ -0,0 +1,107 @@ +providerConfig->getProviders() as $provider) { + if ($provider['isInitiated'] || $provider['classString'] === '' + || $this->jobList->has(InitialContentImportJob::class, $provider['classString']) + ) { + continue; + } + $this->jobList->add(InitialContentImportJob::class, $provider['classString']); + } + + if ($this->jobList->has(SchedulerJob::class, null)) { + return false; + } + $this->jobList->add(SchedulerJob::class); + return true; + } + + /** + * Delete everything that was queued or indexed, here and in context_chat_backend's vector DB. + * The local state is cleared first so nothing new is handed to the backend while it is wiped. + * It is safe to run this again when the backend request fails. + * + * @throws \OCP\DB\Exception + * @throws \RuntimeException when the vector DB could not be wiped + */ + public function wipe(): void { + $this->logger->info('Wiping all the Context Chat data'); + + // stop the crawl and the content provider imports from queueing more items + foreach ([SchedulerJob::class, StorageCrawlJob::class, InitialContentImportJob::class] as $jobClass) { + $this->jobList->remove($jobClass); + } + + $this->db->beginTransaction(); + try { + foreach (self::QUEUE_TABLES as $table) { + $this->db->getQueryBuilder()->delete($table)->executeStatement(); + } + $this->db->commit(); + } catch (\Throwable $e) { + $this->db->rollBack(); + throw $e; + } + + foreach (self::STATS_CONFIG_KEYS as $key) { + $this->appConfig->deleteAppValue($key); + } + // the time taken for the next complete index is measured from here + $this->appConfig->setAppValueInt('installed_time', $this->timeFactory->getTime(), lazy: true); + $this->providerConfig->resetProvidersInitiation(); + + $this->langRopeService->deleteAll(); + $this->logger->info('Wiped all the Context Chat data'); + } +} diff --git a/tests/integration/ReindexServiceTest.php b/tests/integration/ReindexServiceTest.php new file mode 100644 index 00000000..f79c53e0 --- /dev/null +++ b/tests/integration/ReindexServiceTest.php @@ -0,0 +1,191 @@ +db = Server::get(IDBConnection::class); + $this->jobList = Server::get(IJobList::class); + $this->appConfig = Server::get(IAppConfig::class); + $this->providerConfig = Server::get(ProviderConfigService::class); + $this->langRope = $this->createMock(LangRopeService::class); + $time = $this->createMock(ITimeFactory::class); + $time->method('getTime')->willReturn(self::NOW); + + $this->service = new ReindexService( + $this->db, + $this->jobList, + $this->appConfig, + $this->providerConfig, + $this->langRope, + $time, + Server::get(Logger::class), + ); + + $this->savedProviders = $this->appConfig->getAppValueString('providers', '', lazy: true); + $this->clearState(); + } + + protected function tearDown(): void { + $this->clearState(); + $this->appConfig->setAppValueString('providers', $this->savedProviders, lazy: true); + parent::tearDown(); + } + + private function clearState(): void { + foreach (ReindexService::QUEUE_TABLES as $table) { + $this->db->getQueryBuilder()->delete($table)->executeStatement(); + } + foreach ([SchedulerJob::class, StorageCrawlJob::class, InitialContentImportJob::class] as $jobClass) { + $this->jobList->remove($jobClass); + } + } + + private function seedIndexedState(): void { + $file = new QueueFile(); + $file->setFileId(42); + $file->setStorageId(1); + $file->setRootId(1); + $file->setUpdate(false); + Server::get(QueueMapper::class)->insertIntoQueue($file); + + $item = new QueueContentItem(); + $item->setItemId('item1'); + $item->setAppId('someapp'); + $item->setProviderId('prov'); + $item->setTitle('title'); + $item->setContent('content'); + $item->setDocumentType('text'); + $item->setLastModified(new \DateTime()); + $item->setUsers('admin'); + Server::get(QueueContentItemMapper::class)->insert($item); + + $action = new QueueAction(); + $action->setType(ActionType::DELETE_SOURCE_IDS); + $action->setPayload('{"sourceIds":["files__default: 42"]}'); + Server::get(QueueActionMapper::class)->insertIntoQueue($action); + + Server::get(FsEventMapper::class)->insertRow(FsEventType::CREATE->value, 'admin', 42); + + $this->jobList->add(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0]); + $this->jobList->add(InitialContentImportJob::class, self::PROVIDER_CLASS); + + $this->appConfig->setAppValueInt('last_indexed_time', 123, lazy: true); + $this->appConfig->setAppValueInt('last_enqueued_db_id', 7, lazy: true); + $this->appConfig->setAppValueInt('eligible_files_count', 99, lazy: true); + $this->appConfig->setAppValueInt('eligible_files_count_time', 456, lazy: true); + + $this->providerConfig->updateProvider('files', 'default', '', true); + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true); + } + + private function rowCount(string $table): int { + $qb = $this->db->getQueryBuilder(); + return (int)$qb->select($qb->func()->count('*'))->from($table)->executeQuery()->fetchOne(); + } + + public function testWipeClearsAllLocalStateAndTheBackend(): void { + $this->seedIndexedState(); + $this->langRope->expects($this->once())->method('deleteAll'); + + $this->service->wipe(); + + foreach (ReindexService::QUEUE_TABLES as $table) { + $this->assertSame(0, $this->rowCount($table), $table . ' should be empty'); + } + $this->assertFalse($this->jobList->has(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0])); + $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + $this->assertSame(0, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true)); + $this->assertSame(-1, $this->appConfig->getAppValueInt('last_enqueued_db_id', -1, lazy: true)); + $this->assertSame(0, $this->appConfig->getAppValueInt('eligible_files_count_time', 0, lazy: true)); + $this->assertSame(self::NOW, $this->appConfig->getAppValueInt('installed_time', 0, lazy: true)); + + $providers = $this->providerConfig->getProviders(); + $this->assertFalse($providers['someapp__prov']['isInitiated'], 'content providers must be imported again'); + $this->assertTrue($providers[ProviderConfigService::getDefaultProviderKey()]['isInitiated'], 'files are re-crawled by SchedulerJob instead'); + } + + public function testWipeClearsLocalStateBeforeTheBackendFails(): void { + $this->seedIndexedState(); + $this->langRope->method('deleteAll')->willThrowException(new \RuntimeException('backend down')); + + try { + $this->service->wipe(); + $this->fail('wipe() must report the backend failure'); + } catch (\RuntimeException $e) { + $this->assertSame('backend down', $e->getMessage()); + } + + // nothing may be indexed or pruned against a half-wiped vector DB, and a re-run must work + foreach (ReindexService::QUEUE_TABLES as $table) { + $this->assertSame(0, $this->rowCount($table), $table . ' should be empty'); + } + } + + public function testScheduleReindexAfterWipeQueuesCrawlAndProviderImportsOnce(): void { + $this->seedIndexedState(); + $this->service->wipe(); + + $this->assertTrue($this->service->scheduleReindex()); + $this->assertFalse($this->service->scheduleReindex(), 'an already scheduled crawl is not scheduled again'); + + $this->assertTrue($this->jobList->has(SchedulerJob::class, null)); + $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + $importJobs = iterator_to_array($this->jobList->getJobsIterator(InitialContentImportJob::class, null, 0), false); + $this->assertCount(1, $importJobs); + } + + public function testScheduleReindexSkipsInitiatedProviders(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true); + + $this->service->scheduleReindex(); + + $this->assertTrue($this->jobList->has(SchedulerJob::class, null)); + $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + } +} From 90e51b46fc2e0660fedcc27f3e5c8a2ad1a7166c Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Thu, 29 Oct 2026 06:21:02 +0530 Subject: [PATCH 02/12] feat: add option to only wipe one provider and reindex it when the option --provider is set, the queues or indexing stats are not touched. Signed-off-by: kyteinsky --- .github/workflows/integration-test-cron.yml | 28 ++++ lib/Command/Reindex.php | 33 +++- lib/Command/Wipe.php | 81 ++++++++-- lib/Db/QueueContentItemMapper.php | 11 ++ lib/Service/LangRopeService.php | 13 +- lib/Service/ProviderConfigService.php | 33 ++++ lib/Service/ReindexService.php | 160 +++++++++++++++++--- lib/Service/StatisticsService.php | 4 +- tests/integration/ReindexServiceTest.php | 128 ++++++++++++++-- 9 files changed, 427 insertions(+), 64 deletions(-) diff --git a/.github/workflows/integration-test-cron.yml b/.github/workflows/integration-test-cron.yml index b62ebb03..a7d22da1 100644 --- a/.github/workflows/integration-test-cron.yml +++ b/.github/workflows/integration-test-cron.yml @@ -405,6 +405,34 @@ jobs: kill -9 $WORKER1_PID kill -9 $WORKER2_PID + - name: Wipe and reindex only the files provider + run: | + ./occ context_chat:wipe --force --provider files__default --reindex + + stats=$(./occ context_chat:stats --json) + echo "$stats" + indexed=$(echo "$stats" | jq '.vectordb_document_counts.files__default // 0') + if [ "$indexed" != "0" ]; then + echo "Error: expected no indexed files right after the files wipe, got $indexed" + exit 1 + fi + + echo "::group::Waiting up to 10 minutes for the files to be indexed again" + for i in {1..60}; do + stats=$(timeout 30 ./occ context_chat:stats --json || true) + indexed=$(echo "$stats" | jq '.vectordb_document_counts.files__default // 0' 2>/dev/null || echo 0) + echo "Attempt $i: indexed=$indexed" + if [ "$indexed" -ge 10 ]; then + echo "::endgroup::" + echo "Files reindexed, $indexed documents" + exit 0 + fi + sleep 10 + done + echo "::endgroup::" + echo "Error: files were not indexed again after the files wipe" + exit 1 + - name: Wipe all the indexed data and check that the queues and the vector DB are empty run: | ./occ context_chat:wipe --force diff --git a/lib/Command/Reindex.php b/lib/Command/Reindex.php index 73b322b0..e3b2dc6f 100644 --- a/lib/Command/Reindex.php +++ b/lib/Command/Reindex.php @@ -12,6 +12,7 @@ use OCA\ContextChat\Service\ReindexService; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; /** @@ -34,16 +35,40 @@ public function __construct( protected function configure() { $this->setName('context_chat:reindex') - ->setDescription('Schedule a full re-crawl of all the files in all the mounts. Indexed files are not re-indexed when compared against context_chat_backend\'s vector DB. Content providers are only re-imported after context_chat:wipe.'); + ->setDescription( + 'Schedule a full re-crawl of all the files in all the mounts and the import of the content providers' + . ' that were not imported yet. Indexed files are not re-indexed when compared against context_chat_backend\'s vector DB.' + ) + ->addOption( + 'provider', + 'p', + InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, + 'Only reindex this provider, e.g. "files__default" (appId__providerId) for the files.' + . ' A named content provider is imported again even if it was imported before. Multiple values can be specified by repeating the option.', + ); } protected function execute(InputInterface $input, OutputInterface $output): int { - if (!$this->reindexService->scheduleReindex()) { - $output->writeln('A full re-crawl is already scheduled; nothing to do.'); + /** @var list $providerKeys */ + $providerKeys = $input->getOption('provider'); + $scope = $providerKeys === [] ? null : $providerKeys; + + if ($scope !== null) { + $known = $this->reindexService->getRegisteredProviderKeys(); + $unknown = array_diff($scope, $known); + if ($unknown !== []) { + $output->writeln('Unknown provider(s): ' . implode(', ', $unknown) . ''); + $output->writeln('Registered providers: ' . implode(', ', $known)); + return 1; + } + } + + if (!$this->reindexService->scheduleReindex($scope)) { + $output->writeln('The re-index is already scheduled; nothing to do.'); return 0; } - $output->writeln('Scheduled a full re-crawl. SchedulerJob will enumerate all mounts on its next run.'); + $output->writeln('Scheduled the re-index. The background jobs will queue the documents on their next run.'); return 0; } } diff --git a/lib/Command/Wipe.php b/lib/Command/Wipe.php index c5960d9b..ae2caa06 100644 --- a/lib/Command/Wipe.php +++ b/lib/Command/Wipe.php @@ -27,12 +27,22 @@ public function __construct( protected function configure() { $this->setName('context_chat:wipe') - ->setDescription('Delete all the indexed documents from context_chat_backend\'s vector DB, clear the indexing and actions queues and reset the indexing statistics. Nothing gets indexed again until context_chat:reindex is run or the --reindex option is used.') + ->setDescription( + 'Delete the indexed documents from context_chat_backend\'s vector DB and clear the queues.' + . ' With --provider option, only the provider\'s data is deleted, the actions queue, and the indexing statistics are left intact.' + . ' Nothing gets indexed again until context_chat:reindex is run or the --reindex option is used.' + ) + ->addOption( + 'provider', + 'p', + InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, + 'Only wipe this provider, e.g. "files__default" (appId__providerId) for the files. Multiple values can be specified by repeating the option.', + ) ->addOption( 'reindex', null, InputOption::VALUE_NONE, - 'Schedule a full re-crawl of all the files and a re-import of all the content providers after the wipe', + 'Schedule a re-crawl of the files and a re-import of the content providers that were wiped', ) ->addOption( 'force', @@ -43,10 +53,36 @@ protected function configure() { } protected function execute(InputInterface $input, OutputInterface $output): int { + /** @var list $providerKeys */ + $providerKeys = $input->getOption('provider'); + $scope = $providerKeys === [] ? null : $providerKeys; + + if ($scope !== null) { + try { + $known = $this->reindexService->getWipeableProviderKeys(); + } catch (\RuntimeException $e) { + $output->writeln('Could not list the indexed providers: ' . $e->getMessage() . ''); + return 1; + } + $unknown = array_diff($scope, $known); + if ($unknown !== []) { + $output->writeln('Unknown provider(s): ' . implode(', ', $unknown) . ''); + $output->writeln('Known providers: ' . implode(', ', $known)); + return 1; + } + } + + $scopeText = $scope === null ? '' : ' of ' . implode(', ', $scope); + /** @var QuestionHelper $helper */ $helper = $this->getHelper('question'); if (!$input->getOption('force')) { - $question = new ConfirmationQuestion('All the indexed documents in Context Chat will be deleted. The original documents are not touched. Continue? [y/N] ', false); + $question = new ConfirmationQuestion( + 'All the indexed documents in Context Chat' + . $scopeText + . ' will be deleted. The original documents are not touched. Continue? [y/N] ', + false, + ); if (!$helper->ask($input, $output, $question)) { $output->writeln('Aborted, nothing was deleted. Use --force to skip the confirmation.'); return 1; @@ -54,26 +90,39 @@ protected function execute(InputInterface $input, OutputInterface $output): int } try { - $this->reindexService->wipe(); + $this->reindexService->wipe($scope); } catch (\Throwable $e) { - $output->writeln('Wipe failed: ' . $e->getMessage() . ''); - $output->writeln('The command can be safely run again once the problem is fixed.'); + $output->writeln('Wipe failed. The command can be safely run again once the problem is fixed.'); + $output->writeln('Error: ' . $e->getMessage() . ''); + $output->writeln('' . $e->getTraceAsString() . ''); return 1; } - $output->writeln('Deleted all the indexed documents and cleared the queues.'); + $output->writeln('Deleted all the indexed documents' . $scopeText . ' and cleared the queues.'); + + // providers that are not registered anymore cannot be indexed again + $reindexScope = $scope === null + ? null + : array_values(array_intersect($scope, $this->reindexService->getRegisteredProviderKeys())); + if ($reindexScope === []) { + return 0; + } if ($input->getOption('reindex')) { - $this->reindexService->scheduleReindex(); - $output->writeln('Scheduled a full re-crawl of files and re-import of the content providers.'); - } else { - $question = new ConfirmationQuestion('Re-index the documents now? [Y/n] ', default: true); - if ($helper->ask($input, $output, $question)) { - $this->reindexService->scheduleReindex(); - $output->writeln('Scheduled a full re-crawl of files and re-import of the content providers.'); - } else { - $output->writeln('Run context_chat:reindex to index the documents again later.'); + try { + $this->reindexService->scheduleReindex($reindexScope); + } catch (\Throwable $e) { + $output->writeln('Scheduling re-index failed. It can be tried again with context_chat:reindex.'); + $output->writeln('Error: ' . $e->getMessage() . ''); + $output->writeln('' . $e->getTraceAsString() . ''); + return 1; } + $output->writeln('Scheduled the re-index.'); + } elseif ($reindexScope === null) { + $output->writeln('Run context_chat:reindex to index the documents again.'); + } else { + $output->writeln('Run context_chat:reindex --provider ' . implode(' --provider ', $reindexScope) . ' to index the documents again.'); } + return 0; } } diff --git a/lib/Db/QueueContentItemMapper.php b/lib/Db/QueueContentItemMapper.php index 2cc79cfc..b61f1cd6 100644 --- a/lib/Db/QueueContentItemMapper.php +++ b/lib/Db/QueueContentItemMapper.php @@ -71,6 +71,17 @@ public function removeFromQueue(array $ids): void { } } + /** + * @throws \OCP\DB\Exception + */ + public function removeProvider(string $appId, string $providerId): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getTableName()) + ->where($qb->expr()->eq('app_id', $qb->createNamedParameter($appId))) + ->andWhere($qb->expr()->eq('provider_id', $qb->createNamedParameter($providerId))) + ->executeStatement(); + } + /** * @throws \OCP\DB\Exception * @return array diff --git a/lib/Service/LangRopeService.php b/lib/Service/LangRopeService.php index 26339916..84e8cf93 100644 --- a/lib/Service/LangRopeService.php +++ b/lib/Service/LangRopeService.php @@ -189,14 +189,13 @@ private function requestToExApp( */ public function getIndexedDocumentsCounts(): array { $response = $this->requestToExApp('/countIndexedDocuments', 'POST', extraOptions: ['timeout' => 10]); - if ($response === []) { - // No documents indexed yet - return []; - } - if (!isset($response[ProviderConfigService::getDefaultProviderKey()])) { - throw new \RuntimeException("Malformed indexed documents count response from Context Chat Backend (ExApp): '" - . ProviderConfigService::getDefaultProviderKey() . "' key is missing, response: " . strval(json_encode($response))); + foreach ($response as $count) { + if (!is_int($count)) { + throw new \RuntimeException('Malformed indexed documents count response from Context Chat Backend (ExApp): ' + . strval(json_encode($response))); + } } + // only the providers with indexed documents are listed return $response; } diff --git a/lib/Service/ProviderConfigService.php b/lib/Service/ProviderConfigService.php index a5d8ba2b..f513596d 100644 --- a/lib/Service/ProviderConfigService.php +++ b/lib/Service/ProviderConfigService.php @@ -126,6 +126,39 @@ public function resetProvidersInitiation(): void { $this->appConfig->setAppValueString('providers', json_encode($providers), lazy: true); } + /** + * Get the parts of a provider key + * + * @param string $providerKey The provider key in the format "appId__providerId" + * @return array{0: string, 1: string} The appId and providerId + * @throws \RuntimeException when the provider key format is invalid + */ + public function getProviderParts(string $providerKey): array { + $identifierValues = explode('__', $providerKey, 2); + if (count($identifierValues) !== 2) { + throw new \RuntimeException('Invalid provider key format: ' . $providerKey); + } + return $identifierValues; + } + + /** + * Mark a specific provider as initiated or not initiated + * + * @param string $providerKey The provider key in the format "appId__providerId" + * @param bool $isInitiated + * @return void + * @throws \RuntimeException when the provider key format is invalid + */ + public function setProviderInitiated(string $providerKey, bool $isInitiated): void { + $provider = $this->getProvider($providerKey); + if ($provider === null) { + return; + } + [$appId, $providerId] = $this->getProviderParts($providerKey); + /** @psalm-suppress ArgumentTypeCoercion */ + $this->updateProvider($appId, $providerId, $provider['classString'], $isInitiated); + } + /** * @param string $appId * @param ?string $providerId diff --git a/lib/Service/ReindexService.php b/lib/Service/ReindexService.php index 598dc5de..4e9d6f89 100644 --- a/lib/Service/ReindexService.php +++ b/lib/Service/ReindexService.php @@ -12,6 +12,7 @@ use OCA\ContextChat\BackgroundJobs\InitialContentImportJob; use OCA\ContextChat\BackgroundJobs\SchedulerJob; use OCA\ContextChat\BackgroundJobs\StorageCrawlJob; +use OCA\ContextChat\Db\QueueContentItemMapper; use OCA\ContextChat\Logger; use OCP\AppFramework\Services\IAppConfig; use OCP\AppFramework\Utility\ITimeFactory; @@ -19,11 +20,16 @@ use OCP\IDBConnection; class ReindexService { - public const QUEUE_TABLES = [ + // queues that only hold files + private const FILES_QUEUE_TABLES = [ 'context_chat_queue', + 'context_chat_fs_events', + ]; + + public const QUEUE_TABLES = [ + ...self::FILES_QUEUE_TABLES, 'context_chat_content_queue', 'context_chat_action_queue', - 'context_chat_fs_events', ]; private const STATS_CONFIG_KEYS = [ @@ -39,32 +45,89 @@ public function __construct( private IAppConfig $appConfig, private ProviderConfigService $providerConfig, private LangRopeService $langRopeService, + private QueueContentItemMapper $contentQueue, private ITimeFactory $timeFactory, private Logger $logger, ) { } /** - * Schedule the full filesystem crawl and the initial import of every content provider that has not - * been imported yet. Already scheduled jobs are not scheduled again. + * Keys of the providers that can be reindexed: the files and the registered content providers. + * + * @return list + */ + public function getRegisteredProviderKeys(): array { + return $this->uniqueSorted(array_merge( + [ProviderConfigService::getDefaultProviderKey()], + array_keys($this->providerConfig->getProviders()), + )); + } + + /** + * Keys of the providers that can be wiped: the registered ones and the ones that still have + * documents in the vector DB, e.g. of an app that has been removed since. + * + * @return list + * @throws \RuntimeException when the backend could not be reached + */ + public function getWipeableProviderKeys(): array { + return $this->uniqueSorted(array_merge( + $this->getRegisteredProviderKeys(), + array_keys($this->langRopeService->getIndexedDocumentsCounts()), + )); + } + + /** + * @param array $keys + * @return list + */ + private function uniqueSorted(array $keys): array { + $keys = array_values(array_unique(array_map(strval(...), $keys))); + sort($keys); + return $keys; + } + + /** + * Schedule the full filesystem crawl and the initial import of the content providers. + * Jobs that are already scheduled are not scheduled again. * - * @return bool whether the filesystem crawl was scheduled, false if it was already scheduled + * @param list|null $providerKeys null means everything. Without a scope only the content + * providers that have not been imported yet are imported (which is all of them after a full wipe), + * content providers explicitly specified are set as not initiated and scheduled for import. + * @return bool whether anything new was scheduled */ - public function scheduleReindex(): bool { - foreach ($this->providerConfig->getProviders() as $provider) { - if ($provider['isInitiated'] || $provider['classString'] === '' - || $this->jobList->has(InitialContentImportJob::class, $provider['classString']) - ) { + public function scheduleReindex(?array $providerKeys = null): bool { + $scheduled = false; + $filesKey = ProviderConfigService::getDefaultProviderKey(); + + foreach ($this->providerConfig->getProviders() as $key => $provider) { + if ($key === $filesKey || $provider['classString'] === '') { + continue; + } + if ($providerKeys === null) { + // a full wipe turns all providers as not initiated so they will be scheduled for import and not skipped + if ($provider['isInitiated']) { + continue; + } + } elseif (in_array($key, $providerKeys, true)) { + $this->providerConfig->setProviderInitiated($key, false); + } else { + continue; + } + if ($this->jobList->has(InitialContentImportJob::class, $provider['classString'])) { continue; } $this->jobList->add(InitialContentImportJob::class, $provider['classString']); + $scheduled = true; } - if ($this->jobList->has(SchedulerJob::class, null)) { - return false; + if (($providerKeys === null || in_array($filesKey, $providerKeys, true)) + && !$this->jobList->has(SchedulerJob::class, null) + ) { + $this->jobList->add(SchedulerJob::class); + $scheduled = true; } - $this->jobList->add(SchedulerJob::class); - return true; + return $scheduled; } /** @@ -72,36 +135,91 @@ public function scheduleReindex(): bool { * The local state is cleared first so nothing new is handed to the backend while it is wiped. * It is safe to run this again when the backend request fails. * + * @param list|null $providerKeys null means everything including the actions queue, otherwise only the given providers * @throws \OCP\DB\Exception * @throws \RuntimeException when the vector DB could not be wiped */ - public function wipe(): void { + public function wipe(?array $providerKeys = null): void { + if ($providerKeys === null) { + $this->wipeAll(); + return; + } + + foreach (array_unique($providerKeys) as $providerKey) { + $this->logger->info('Wiping the Context Chat data of provider ' . $providerKey); + if ($providerKey === ProviderConfigService::getDefaultProviderKey()) { + $this->clearFilesState(); + } else { + $this->clearContentProviderState($providerKey); + } + $this->langRopeService->deleteProvider($providerKey); + } + } + + private function wipeAll(): void { $this->logger->info('Wiping all the Context Chat data'); // stop the crawl and the content provider imports from queueing more items foreach ([SchedulerJob::class, StorageCrawlJob::class, InitialContentImportJob::class] as $jobClass) { $this->jobList->remove($jobClass); } + $this->clearTables(self::QUEUE_TABLES); + $this->resetStats(); + $this->providerConfig->resetProvidersInitiation(); + + $this->langRopeService->deleteAll(); + $this->logger->info('Wiped all the Context Chat data'); + } + + /** + * The actions queue is left alone, it is shared by all the providers and the users. + * Leftover actions for the wiped documents are skipped by the backend. + */ + private function clearFilesState(): void { + foreach ([SchedulerJob::class, StorageCrawlJob::class] as $jobClass) { + $this->jobList->remove($jobClass); + } + $this->clearTables(self::FILES_QUEUE_TABLES); + // the stats only track the files + $this->resetStats(); + } + private function clearContentProviderState(string $providerKey): void { + $provider = $this->providerConfig->getProvider($providerKey); + if ($provider !== null && $provider['classString'] !== '') { + $this->jobList->remove(InitialContentImportJob::class, $provider['classString']); + } + + [$appId, $providerId] = $this->providerConfig->getProviderParts($providerKey); + $this->contentQueue->removeProvider($appId, $providerId); + + if ($provider !== null) { + $this->providerConfig->setProviderInitiated($providerKey, false); + } + } + + /** + * @param list $tables + * @throws \RuntimeException + */ + private function clearTables(array $tables): void { $this->db->beginTransaction(); try { - foreach (self::QUEUE_TABLES as $table) { + foreach ($tables as $table) { $this->db->getQueryBuilder()->delete($table)->executeStatement(); } $this->db->commit(); } catch (\Throwable $e) { $this->db->rollBack(); - throw $e; + throw new \RuntimeException('Failed to clear tables ' . implode(', ', $tables) . ': ' . $e->getMessage(), 0, $e); } + } + private function resetStats(): void { foreach (self::STATS_CONFIG_KEYS as $key) { $this->appConfig->deleteAppValue($key); } // the time taken for the next complete index is measured from here $this->appConfig->setAppValueInt('installed_time', $this->timeFactory->getTime(), lazy: true); - $this->providerConfig->resetProvidersInitiation(); - - $this->langRopeService->deleteAll(); - $this->logger->info('Wiped all the Context Chat data'); } } diff --git a/lib/Service/StatisticsService.php b/lib/Service/StatisticsService.php index 7d214ef0..c3086b52 100644 --- a/lib/Service/StatisticsService.php +++ b/lib/Service/StatisticsService.php @@ -69,9 +69,7 @@ public function getStatistics(): array { $this->logger->error($e->getMessage(), ['exception' => $e]); $stats['backend_available'] = false; } - if (empty($stats['vectordb_document_counts'] ?? [])) { - $stats['vectordb_document_counts'] = [ ProviderConfigService::getDefaultProviderKey() => 0 ]; - } + $stats['vectordb_document_counts'] = ($stats['vectordb_document_counts'] ?? []) + [ ProviderConfigService::getDefaultProviderKey() => 0 ]; try { $queued_files_count = $this->queueService->count(); diff --git a/tests/integration/ReindexServiceTest.php b/tests/integration/ReindexServiceTest.php index f79c53e0..ec092516 100644 --- a/tests/integration/ReindexServiceTest.php +++ b/tests/integration/ReindexServiceTest.php @@ -9,6 +9,7 @@ namespace OCA\ContextChat\Tests; +use OCA\ContextChat\AppInfo\Application; use OCA\ContextChat\BackgroundJobs\InitialContentImportJob; use OCA\ContextChat\BackgroundJobs\SchedulerJob; use OCA\ContextChat\BackgroundJobs\StorageCrawlJob; @@ -37,6 +38,7 @@ #[Group('DB')] class ReindexServiceTest extends TestCase { private const PROVIDER_CLASS = 'OCA\\SomeApp\\ContentProvider'; + private const OTHER_PROVIDER_CLASS = 'OCA\\OtherApp\\ContentProvider'; private const NOW = 1_800_000_000; private IDBConnection $db; @@ -51,8 +53,10 @@ protected function setUp(): void { parent::setUp(); $this->db = Server::get(IDBConnection::class); $this->jobList = Server::get(IJobList::class); - $this->appConfig = Server::get(IAppConfig::class); - $this->providerConfig = Server::get(ProviderConfigService::class); + // the app scoped config is only available from the app's container + $container = (new Application())->getContainer(); + $this->appConfig = $container->get(IAppConfig::class); + $this->providerConfig = $container->get(ProviderConfigService::class); $this->langRope = $this->createMock(LangRopeService::class); $time = $this->createMock(ITimeFactory::class); $time->method('getTime')->willReturn(self::NOW); @@ -63,12 +67,14 @@ protected function setUp(): void { $this->appConfig, $this->providerConfig, $this->langRope, + Server::get(QueueContentItemMapper::class), $time, Server::get(Logger::class), ); $this->savedProviders = $this->appConfig->getAppValueString('providers', '', lazy: true); $this->clearState(); + $this->appConfig->setAppValueString('providers', '', lazy: true); } protected function tearDown(): void { @@ -94,16 +100,18 @@ private function seedIndexedState(): void { $file->setUpdate(false); Server::get(QueueMapper::class)->insertIntoQueue($file); - $item = new QueueContentItem(); - $item->setItemId('item1'); - $item->setAppId('someapp'); - $item->setProviderId('prov'); - $item->setTitle('title'); - $item->setContent('content'); - $item->setDocumentType('text'); - $item->setLastModified(new \DateTime()); - $item->setUsers('admin'); - Server::get(QueueContentItemMapper::class)->insert($item); + foreach (['someapp', 'otherapp'] as $appId) { + $item = new QueueContentItem(); + $item->setItemId('item1'); + $item->setAppId($appId); + $item->setProviderId('prov'); + $item->setTitle('title'); + $item->setContent('content'); + $item->setDocumentType('text'); + $item->setLastModified(new \DateTime()); + $item->setUsers('admin'); + Server::get(QueueContentItemMapper::class)->insert($item); + } $action = new QueueAction(); $action->setType(ActionType::DELETE_SOURCE_IDS); @@ -114,6 +122,7 @@ private function seedIndexedState(): void { $this->jobList->add(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0]); $this->jobList->add(InitialContentImportJob::class, self::PROVIDER_CLASS); + $this->jobList->add(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS); $this->appConfig->setAppValueInt('last_indexed_time', 123, lazy: true); $this->appConfig->setAppValueInt('last_enqueued_db_id', 7, lazy: true); @@ -122,6 +131,19 @@ private function seedIndexedState(): void { $this->providerConfig->updateProvider('files', 'default', '', true); $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true); + $this->providerConfig->updateProvider('otherapp', 'prov', self::OTHER_PROVIDER_CLASS, true); + } + + /** @return list */ + private function contentQueueAppIds(): array { + $qb = $this->db->getQueryBuilder(); + $result = $qb->selectDistinct('app_id')->from('context_chat_content_queue')->executeQuery(); + $appIds = []; + while (($appId = $result->fetchOne()) !== false) { + $appIds[] = $appId; + } + $result->closeCursor(); + return $appIds; } private function rowCount(string $table): int { @@ -132,6 +154,7 @@ private function rowCount(string $table): int { public function testWipeClearsAllLocalStateAndTheBackend(): void { $this->seedIndexedState(); $this->langRope->expects($this->once())->method('deleteAll'); + $this->langRope->expects($this->never())->method('deleteProvider'); $this->service->wipe(); @@ -140,6 +163,7 @@ public function testWipeClearsAllLocalStateAndTheBackend(): void { } $this->assertFalse($this->jobList->has(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0])); $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS)); $this->assertSame(0, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true)); $this->assertSame(-1, $this->appConfig->getAppValueInt('last_enqueued_db_id', -1, lazy: true)); $this->assertSame(0, $this->appConfig->getAppValueInt('eligible_files_count_time', 0, lazy: true)); @@ -177,7 +201,7 @@ public function testScheduleReindexAfterWipeQueuesCrawlAndProviderImportsOnce(): $this->assertTrue($this->jobList->has(SchedulerJob::class, null)); $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); $importJobs = iterator_to_array($this->jobList->getJobsIterator(InitialContentImportJob::class, null, 0), false); - $this->assertCount(1, $importJobs); + $this->assertCount(2, $importJobs); } public function testScheduleReindexSkipsInitiatedProviders(): void { @@ -188,4 +212,82 @@ public function testScheduleReindexSkipsInitiatedProviders(): void { $this->assertTrue($this->jobList->has(SchedulerJob::class, null)); $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); } + + public function testWipeFilesOnlyLeavesContentProvidersAndActions(): void { + $this->seedIndexedState(); + $this->langRope->expects($this->never())->method('deleteAll'); + $this->langRope->expects($this->once())->method('deleteProvider')->with('files__default'); + + $this->service->wipe(['files__default']); + + $this->assertSame(0, $this->rowCount('context_chat_queue')); + $this->assertSame(0, $this->rowCount('context_chat_fs_events')); + $this->assertSame(2, $this->rowCount('context_chat_content_queue'), 'content providers must be kept'); + $this->assertSame(1, $this->rowCount('context_chat_action_queue'), 'actions of other providers and users must be kept'); + $this->assertFalse($this->jobList->has(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0])); + $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + $this->assertSame(0, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true)); + $this->assertTrue($this->providerConfig->getProviders()['someapp__prov']['isInitiated']); + } + + public function testWipeOneContentProviderLeavesTheRest(): void { + $this->seedIndexedState(); + $this->langRope->expects($this->never())->method('deleteAll'); + $this->langRope->expects($this->once())->method('deleteProvider')->with('someapp__prov'); + + $this->service->wipe(['someapp__prov']); + + $this->assertSame(['otherapp'], $this->contentQueueAppIds()); + $this->assertSame(1, $this->rowCount('context_chat_queue'), 'files must be kept'); + $this->assertSame(1, $this->rowCount('context_chat_fs_events'), 'files must be kept'); + $this->assertSame(1, $this->rowCount('context_chat_action_queue')); + $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS)); + $this->assertSame(123, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true), 'file stats must be kept'); + + $providers = $this->providerConfig->getProviders(); + $this->assertFalse($providers['someapp__prov']['isInitiated']); + $this->assertTrue($providers['otherapp__prov']['isInitiated']); + } + + public function testWipeOfAnUnregisteredProviderOnlyDeletesInTheBackend(): void { + $this->seedIndexedState(); + $this->langRope->expects($this->once())->method('deleteProvider')->with('removedapp__prov'); + + $this->service->wipe(['removedapp__prov']); + + $this->assertSame(2, $this->rowCount('context_chat_content_queue')); + $this->assertArrayNotHasKey('removedapp__prov', $this->providerConfig->getProviders()); + } + + public function testScheduleReindexOfANamedProviderReimportsItEvenIfImported(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true); + $this->providerConfig->updateProvider('otherapp', 'prov', self::OTHER_PROVIDER_CLASS, true); + + $this->assertTrue($this->service->scheduleReindex(['someapp__prov'])); + + $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS)); + $this->assertFalse($this->jobList->has(SchedulerJob::class, null), 'files are not in the scope'); + // InitialContentImportJob skips initiated providers + $this->assertFalse($this->providerConfig->getProviders()['someapp__prov']['isInitiated']); + } + + public function testScheduleReindexOfFilesOnly(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, false); + + $this->assertTrue($this->service->scheduleReindex(['files__default'])); + $this->assertFalse($this->service->scheduleReindex(['files__default'])); + + $this->assertTrue($this->jobList->has(SchedulerJob::class, null)); + $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); + } + + public function testWipeableProviderKeysIncludeProvidersOnlyKnownToTheBackend(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true); + $this->langRope->method('getIndexedDocumentsCounts')->willReturn(['files__default' => 3, 'removedapp__prov' => 1]); + + $this->assertSame(['files__default', 'someapp__prov'], $this->service->getRegisteredProviderKeys()); + $this->assertSame(['files__default', 'removedapp__prov', 'someapp__prov'], $this->service->getWipeableProviderKeys()); + } } From 2e80617be041af23dcd569de22765978b65ca354 Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Thu, 29 Oct 2026 08:05:07 +0530 Subject: [PATCH 03/12] feat: return empty responses in fetch endpoints while a wipe happens hand out no queue items to the backend during a wipe and wait for the ones it is still indexing, so wiped documents cannot come back, also when the backend runs on Kubernetes. The wait can be changed with `occ config:app:set context_chat wipe_drain_timeout --value= --type=integer`, default 600, and skipped with `--ignore-handed-out` when the backend was stopped while indexing or some queue items are locked for 24h due to failures. Signed-off-by: kyteinsky --- lib/Command/Wipe.php | 18 +- lib/Controller/QueueController.php | 13 ++ lib/Service/FetchGuard.php | 126 ++++++++++++++ lib/Service/ReindexService.php | 61 +++++-- tests/integration/ReindexServiceTest.php | 213 ++++++++++++++++++++++- 5 files changed, 412 insertions(+), 19 deletions(-) create mode 100644 lib/Service/FetchGuard.php diff --git a/lib/Command/Wipe.php b/lib/Command/Wipe.php index ae2caa06..df5639be 100644 --- a/lib/Command/Wipe.php +++ b/lib/Command/Wipe.php @@ -44,6 +44,12 @@ protected function configure() { InputOption::VALUE_NONE, 'Schedule a re-crawl of the files and a re-import of the content providers that were wiped', ) + ->addOption( + 'ignore-handed-out', + null, + InputOption::VALUE_NONE, + 'Do not wait for the documents context_chat_backend is still indexing. Only use this when the backend was stopped while indexing them, otherwise they can be written to the vector DB after the wipe.', + ) ->addOption( 'force', 'f', @@ -74,9 +80,9 @@ protected function execute(InputInterface $input, OutputInterface $output): int $scopeText = $scope === null ? '' : ' of ' . implode(', ', $scope); - /** @var QuestionHelper $helper */ - $helper = $this->getHelper('question'); if (!$input->getOption('force')) { + /** @var QuestionHelper $helper */ + $helper = $this->getHelper('question'); $question = new ConfirmationQuestion( 'All the indexed documents in Context Chat' . $scopeText @@ -90,7 +96,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int } try { - $this->reindexService->wipe($scope); + $this->reindexService->wipe( + $scope, + (bool)$input->getOption('ignore-handed-out'), + static function (string $message) use ($output): void { + $output->writeln('' . $message . '...'); + }, + ); } catch (\Throwable $e) { $output->writeln('Wipe failed. The command can be safely run again once the problem is fixed.'); $output->writeln('Error: ' . $e->getMessage() . ''); diff --git a/lib/Controller/QueueController.php b/lib/Controller/QueueController.php index f35eaa14..d260afd5 100644 --- a/lib/Controller/QueueController.php +++ b/lib/Controller/QueueController.php @@ -17,6 +17,7 @@ use OCA\ContextChat\Db\QueueContentItemMapper; use OCA\ContextChat\Db\QueueFile; use OCA\ContextChat\Db\QueueMapper; +use OCA\ContextChat\Service\FetchGuard; use OCA\ContextChat\Service\ProviderConfigService; use OCA\ContextChat\Type\Source; use OCP\AppFramework\Http; @@ -47,6 +48,7 @@ public function __construct( private IJobList $jobList, private ITimeFactory $timeFactory, private QueueMapper $queueMapper, + private FetchGuard $fetchGuard, string $corsMethods = 'PUT, POST, GET, DELETE, PATCH', string $corsAllowedHeaders = 'Authorization, Content-Type, Accept, OCS-APIRequest', int $corsMaxAge = 1728000, @@ -106,6 +108,13 @@ public function getDocumentsQueueItems( if ($n > $maxN) { $n = $maxN; } + // hand out nothing, the backend waits and asks again + if ($this->fetchGuard->isPaused()) { + return new DataResponse([ + 'files' => new \stdClass(), + 'content_providers' => new \stdClass(), + ]); + } try { $files = []; while (count($files) < $n) { @@ -216,6 +225,10 @@ public function countDocumentsQueueItems(QueueMapper $queueMapper, QueueContentI #[ExAppRequired] #[ApiRoute(verb: 'GET', url: '/queues/actions/')] public function getActionsQueueItems(QueueActionMapper $queueActionMapper, int $n = 512) : DataResponse { + if ($this->fetchGuard->isPaused()) { + // stdClass is used to represent an empty object in the response + return new DataResponse(['actions' => new \stdClass()]); + } try { $actions = []; while (count($actions) < $n) { diff --git a/lib/Service/FetchGuard.php b/lib/Service/FetchGuard.php new file mode 100644 index 00000000..4ede4c99 --- /dev/null +++ b/lib/Service/FetchGuard.php @@ -0,0 +1,126 @@ +timeFactory->getTime() + $this->getDrainTimeout() + $this->getRequestTimeout() + self::MARGIN; + $this->appConfig->setAppValueInt(self::CONFIG_KEY, $until, lazy: true); + return $until; + } + + /** + * Lifts the pause, unless another one was started since. + */ + public function resume(int $until): void { + if ($this->getPausedUntil() === $until) { + $this->appConfig->deleteAppValue(self::CONFIG_KEY); + } + } + + public function isPaused(): bool { + return $this->getPausedUntil() > $this->timeFactory->getTime(); + } + + /** + * Waits for the documents the backend took before the pause to be reported back. A document that + * is still being indexed would be written to the vector DB after it was wiped. + * + * @param ?callable(int): void $onWaiting called with the number of documents still being indexed, + * once before waiting and again whenever that number changes + * @throws \RuntimeException when they are not reported back in time + * @throws \OCP\DB\Exception + */ + public function waitForHandedOutDocuments(?callable $onWaiting = null): void { + $deadline = $this->timeFactory->getTime() + $this->getDrainTimeout(); + $reported = null; + while (($locked = $this->countHandedOutDocuments()) > 0) { + if ($onWaiting !== null && $locked !== $reported) { + $onWaiting($locked); + $reported = $locked; + } + if ($this->timeFactory->getTime() >= $deadline) { + throw new \RuntimeException( + $locked . ' documents are still being indexed by context_chat_backend after waiting ' + . $this->getDrainTimeout() . ' seconds. Try again later. If the backend was stopped while' + . ' indexing them, they would only be released after a day, use --ignore-handed-out then.' + ); + } + sleep(1); + } + } + + /** + * @throws \OCP\DB\Exception + */ + private function countHandedOutDocuments(): int { + return $this->queueMapper->countLocked() + intval(array_sum($this->contentQueue->countLocked())); + } + + private function getDrainTimeout(): int { + return $this->appConfig->getAppValueInt('wipe_drain_timeout', self::DEFAULT_DRAIN_TIMEOUT, lazy: true); + } + + private function getRequestTimeout(): int { + return (int)$this->appConfig->getAppValueString( + 'request_timeout', + strval(Application::CC_DEFAULT_REQUEST_TIMEOUT), + lazy: true, + ); + } + + /** + * Read from the database instead of IAppConfig. IAppConfig values are cached by each web server + * for a few seconds, and all of them must stop handing out items before the wipe starts. + * This ensures that we get the most up-to-date value for the pause state. + */ + private function getPausedUntil(): int { + $qb = $this->db->getQueryBuilder(); + $result = $qb->select('configvalue') + ->from('appconfig') + ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID))) + ->andWhere($qb->expr()->eq('configkey', $qb->createNamedParameter(self::CONFIG_KEY))) + ->executeQuery(); + $value = $result->fetchOne(); + $result->closeCursor(); + return $value === false ? 0 : (int)$value; + } +} diff --git a/lib/Service/ReindexService.php b/lib/Service/ReindexService.php index 4e9d6f89..409c9369 100644 --- a/lib/Service/ReindexService.php +++ b/lib/Service/ReindexService.php @@ -46,6 +46,7 @@ public function __construct( private ProviderConfigService $providerConfig, private LangRopeService $langRopeService, private QueueContentItemMapper $contentQueue, + private FetchGuard $fetchGuard, private ITimeFactory $timeFactory, private Logger $logger, ) { @@ -132,41 +133,75 @@ public function scheduleReindex(?array $providerKeys = null): bool { /** * Delete everything that was queued or indexed, here and in context_chat_backend's vector DB. - * The local state is cleared first so nothing new is handed to the backend while it is wiped. + * No queue items are handed to the backend until the wipe is over, and the documents it is still + * indexing are waited for, so that its workers, possibly running elsewhere on Kubernetes, cannot + * write to the vector DB while it is wiped. * It is safe to run this again when the backend request fails. * * @param list|null $providerKeys null means everything including the actions queue, otherwise only the given providers + * @param bool $ignoreHandedOut do not wait for the documents the backend is indexing, for when some documents are stuck + * in the queue for 24h due to error or if the backend stopped while indexing them/having a lock + * @param ?callable(string): void $onStep called with a short description before each step, for progress output * @throws \OCP\DB\Exception * @throws \RuntimeException when the vector DB could not be wiped */ - public function wipe(?array $providerKeys = null): void { - if ($providerKeys === null) { - $this->wipeAll(); - return; - } + public function wipe(?array $providerKeys = null, bool $ignoreHandedOut = false, ?callable $onStep = null): void { + $onStep = $onStep ?? static function (string $_): void { + }; - foreach (array_unique($providerKeys) as $providerKey) { - $this->logger->info('Wiping the Context Chat data of provider ' . $providerKey); - if ($providerKey === ProviderConfigService::getDefaultProviderKey()) { - $this->clearFilesState(); + $onStep('Pausing the hand-out of queued documents and actions to context_chat_backend'); + $pausedUntil = $this->fetchGuard->pause(); + try { + if ($ignoreHandedOut) { + $onStep('Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)'); } else { - $this->clearContentProviderState($providerKey); + $onStep('Checking for documents context_chat_backend is still indexing'); + $this->fetchGuard->waitForHandedOutDocuments(static function (int $locked) use ($onStep): void { + $onStep('Waiting for context_chat_backend to finish indexing ' . $locked . ' documents'); + }); + } + + if ($providerKeys === null) { + $this->wipeAll($onStep); + return; } - $this->langRopeService->deleteProvider($providerKey); + + foreach (array_unique($providerKeys) as $providerKey) { + $this->logger->info('Wiping the Context Chat data of provider ' . $providerKey); + if ($providerKey === ProviderConfigService::getDefaultProviderKey()) { + $onStep('Removing the crawl jobs, clearing the files indexing queue and the file system events, resetting the indexing statistics'); + $this->clearFilesState(); + } else { + $onStep('Removing the import job and clearing the indexing queue of ' . $providerKey); + $this->clearContentProviderState($providerKey); + } + $onStep('Deleting the documents of ' . $providerKey . ' from the vector DB'); + $this->langRopeService->deleteProvider($providerKey); + } + } finally { + $onStep('Resuming the hand-out of queued documents and actions'); + $this->fetchGuard->resume($pausedUntil); } } - private function wipeAll(): void { + /** + * @param callable(string): void $onStep + */ + private function wipeAll(callable $onStep): void { $this->logger->info('Wiping all the Context Chat data'); + $onStep('Removing the scheduled crawl and content import jobs'); // stop the crawl and the content provider imports from queueing more items foreach ([SchedulerJob::class, StorageCrawlJob::class, InitialContentImportJob::class] as $jobClass) { $this->jobList->remove($jobClass); } + $onStep('Clearing the indexing, actions and file system events queues'); $this->clearTables(self::QUEUE_TABLES); + $onStep('Resetting the indexing statistics and the content providers\' import state'); $this->resetStats(); $this->providerConfig->resetProvidersInitiation(); + $onStep('Deleting all the documents from the vector DB'); $this->langRopeService->deleteAll(); $this->logger->info('Wiped all the Context Chat data'); } diff --git a/tests/integration/ReindexServiceTest.php b/tests/integration/ReindexServiceTest.php index ec092516..cda6319d 100644 --- a/tests/integration/ReindexServiceTest.php +++ b/tests/integration/ReindexServiceTest.php @@ -21,6 +21,7 @@ use OCA\ContextChat\Db\QueueFile; use OCA\ContextChat\Db\QueueMapper; use OCA\ContextChat\Logger; +use OCA\ContextChat\Service\FetchGuard; use OCA\ContextChat\Service\LangRopeService; use OCA\ContextChat\Service\ProviderConfigService; use OCA\ContextChat\Service\ReindexService; @@ -47,7 +48,9 @@ class ReindexServiceTest extends TestCase { private ProviderConfigService $providerConfig; private LangRopeService&MockObject $langRope; private ReindexService $service; - private string $savedProviders; + private FetchGuard $fetchGuard; + /** @var list> */ + private array $savedConfig; protected function setUp(): void { parent::setUp(); @@ -58,6 +61,7 @@ protected function setUp(): void { $this->appConfig = $container->get(IAppConfig::class); $this->providerConfig = $container->get(ProviderConfigService::class); $this->langRope = $this->createMock(LangRopeService::class); + $this->fetchGuard = $this->newGuard(Server::get(ITimeFactory::class)); $time = $this->createMock(ITimeFactory::class); $time->method('getTime')->willReturn(self::NOW); @@ -68,21 +72,46 @@ protected function setUp(): void { $this->providerConfig, $this->langRope, Server::get(QueueContentItemMapper::class), + $this->fetchGuard, $time, Server::get(Logger::class), ); - $this->savedProviders = $this->appConfig->getAppValueString('providers', '', lazy: true); + $this->savedConfig = $this->readConfig(); $this->clearState(); $this->appConfig->setAppValueString('providers', '', lazy: true); } protected function tearDown(): void { $this->clearState(); - $this->appConfig->setAppValueString('providers', $this->savedProviders, lazy: true); + // the wipe resets the stats and the providers, put back what the instance had before + $qb = $this->db->getQueryBuilder(); + $qb->delete('appconfig') + ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID))) + ->executeStatement(); + foreach ($this->savedConfig as $row) { + $qb = $this->db->getQueryBuilder(); + $qb->insert('appconfig'); + foreach ($row as $column => $value) { + $qb->setValue($column, $qb->createNamedParameter($value)); + } + $qb->executeStatement(); + } + Server::get(\OCP\IAppConfig::class)->clearCache(true); parent::tearDown(); } + /** @return list> */ + private function readConfig(): array { + $qb = $this->db->getQueryBuilder(); + $result = $qb->select('*')->from('appconfig') + ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID))) + ->executeQuery(); + $rows = $result->fetchAll(); + $result->closeCursor(); + return $rows; + } + private function clearState(): void { foreach (ReindexService::QUEUE_TABLES as $table) { $this->db->getQueryBuilder()->delete($table)->executeStatement(); @@ -290,4 +319,182 @@ public function testWipeableProviderKeysIncludeProvidersOnlyKnownToTheBackend(): $this->assertSame(['files__default', 'someapp__prov'], $this->service->getRegisteredProviderKeys()); $this->assertSame(['files__default', 'removedapp__prov', 'someapp__prov'], $this->service->getWipeableProviderKeys()); } + + private function newGuard(ITimeFactory $time): FetchGuard { + return new FetchGuard($this->db, $this->appConfig, $time, Server::get(QueueMapper::class), Server::get(QueueContentItemMapper::class)); + } + + /** @return ITimeFactory&MockObject a clock that advances one second per sleep(1) of the guard */ + private function clock(int &$now): ITimeFactory { + $time = $this->createMock(ITimeFactory::class); + $time->method('getTime')->willReturnCallback(function () use (&$now) { + return $now; + }); + return $time; + } + + public function testFetchingIsPausedOnlyWhileTheBackendIsWiped(): void { + $this->seedIndexedState(); + $seen = null; + $this->langRope->method('deleteAll')->willReturnCallback(function () use (&$seen) { + $seen = $this->fetchGuard->isPaused(); + }); + + $this->service->wipe(); + + $this->assertTrue($seen, 'nothing may be handed out while the vector DB is wiped'); + $this->assertFalse($this->fetchGuard->isPaused()); + } + + public function testFetchingResumesWhenTheBackendFails(): void { + $this->seedIndexedState(); + $this->langRope->method('deleteAll')->willThrowException(new \RuntimeException('backend down')); + + try { + $this->service->wipe(); + } catch (\RuntimeException) { + } + + $this->assertFalse($this->fetchGuard->isPaused()); + } + + public function testWipeIsRefusedWhileDocumentsAreStillBeingIndexed(): void { + $this->seedIndexedState(); + $queueMapper = Server::get(QueueMapper::class); + $queueMapper->lock($queueMapper->getMaxId()); + $this->appConfig->setAppValueInt('wipe_drain_timeout', 0, lazy: true); + $this->langRope->expects($this->never())->method('deleteAll'); + + try { + $this->service->wipe(); + $this->fail('the wipe must not run while a document is being indexed'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('1 documents are still being indexed', $e->getMessage()); + } finally { + $this->appConfig->deleteAppValue('wipe_drain_timeout'); + } + + $this->assertSame(1, $queueMapper->countLocked(), 'nothing may be cleared when the wipe is refused'); + $this->assertFalse($this->fetchGuard->isPaused()); + } + + public function testResumeDoesNotLiftANewerPause(): void { + $now = self::NOW; + $guard = $this->newGuard($this->clock($now)); + + $first = $guard->pause(); + $now += 10; + $second = $guard->pause(); + $guard->resume($first); + + $this->assertTrue($guard->isPaused(), 'the second wipe is still running'); + $guard->resume($second); + $this->assertFalse($guard->isPaused()); + } + + public function testPauseExpiresOnItsOwn(): void { + $now = self::NOW; + $guard = $this->newGuard($this->clock($now)); + + $until = $guard->pause(); + $this->assertTrue($guard->isPaused()); + $now = $until; + + $this->assertFalse($guard->isPaused(), 'a wipe that died must not stop the indexing forever'); + $guard->resume($until); + } + + public function testWaitEndsOnceTheHandedOutBatchIsReportedBack(): void { + $queueMapper = Server::get(QueueMapper::class); + $file = new QueueFile(); + $file->setFileId(43); + $file->setStorageId(1); + $file->setRootId(1); + $file->setUpdate(false); + $id = $queueMapper->insertIntoQueue($file)->getId(); + $queueMapper->lock($id); + + $checks = 0; + $time = $this->createMock(ITimeFactory::class); + $time->method('getTime')->willReturnCallback(function () use (&$checks, $queueMapper, $id) { + // the backend reports the batch back while the wipe is waiting + if (++$checks === 3) { + $queueMapper->removeFromQueue([$id]); + } + return self::NOW; + }); + + $reported = []; + $this->newGuard($time)->waitForHandedOutDocuments(function (int $locked) use (&$reported) { + $reported[] = $locked; + }); + + $this->assertSame(0, $queueMapper->countLocked()); + $this->assertSame([1], $reported, 'the count is reported once, not on every poll'); + } + + /** + * The steps are reported in order, each one before its work: what the admin reads is what runs next. + */ + public function testWipeReportsEachStepBeforeDoingIt(): void { + $this->seedIndexedState(); + $events = []; + $this->langRope->method('deleteAll')->willReturnCallback(function () use (&$events) { + $events[] = 'backend:deleteAll'; + }); + + $this->service->wipe(null, false, function (string $message) use (&$events) { + // mark the moment the queues are emptied, to check the order against the work + $events[] = $message . ($this->rowCount('context_chat_queue') === 0 ? ' [queues empty]' : ''); + }); + + $this->assertSame([ + 'Pausing the hand-out of queued documents and actions to context_chat_backend', + 'Checking for documents context_chat_backend is still indexing', + 'Removing the scheduled crawl and content import jobs', + 'Clearing the indexing, actions and file system events queues', + 'Resetting the indexing statistics and the content providers\' import state [queues empty]', + 'Deleting all the documents from the vector DB [queues empty]', + 'backend:deleteAll', + 'Resuming the hand-out of queued documents and actions [queues empty]', + ], $events); + } + + public function testWipeOfOneProviderReportsItsSteps(): void { + $this->seedIndexedState(); + $events = []; + $this->langRope->method('deleteProvider')->willReturnCallback(function (string $key) use (&$events) { + $events[] = 'backend:deleteProvider ' . $key; + }); + + $this->service->wipe(['someapp__prov'], true, function (string $message) use (&$events) { + $events[] = $message; + }); + + $this->assertSame([ + 'Pausing the hand-out of queued documents and actions to context_chat_backend', + 'Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)', + 'Removing the import job and clearing the indexing queue of someapp__prov', + 'Deleting the documents of someapp__prov from the vector DB', + 'backend:deleteProvider someapp__prov', + 'Resuming the hand-out of queued documents and actions', + ], $events); + } + + public function testResumeIsReportedWhenTheBackendFails(): void { + $this->seedIndexedState(); + $this->langRope->method('deleteAll')->willThrowException(new \RuntimeException('backend down')); + $events = []; + + try { + $this->service->wipe(null, false, function (string $message) use (&$events) { + $events[] = $message; + }); + $this->fail('the backend failure must be reported'); + } catch (\RuntimeException) { + } + + $this->assertSame('Deleting all the documents from the vector DB', $events[count($events) - 2]); + $this->assertSame('Resuming the hand-out of queued documents and actions', end($events)); + } } From 4484b411fc625380a4c6f6428ca5fae8e9cdd657 Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Thu, 29 Oct 2026 18:24:33 +0530 Subject: [PATCH 04/12] removeme: use the backend's branch for testing ci Signed-off-by: kyteinsky --- .github/workflows/integration-test-cron.yml | 2 ++ .github/workflows/integration-test-file-listener.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/integration-test-cron.yml b/.github/workflows/integration-test-cron.yml index a7d22da1..6e15e81e 100644 --- a/.github/workflows/integration-test-cron.yml +++ b/.github/workflows/integration-test-cron.yml @@ -127,6 +127,8 @@ jobs: with: repository: nextcloud/context_chat_backend path: context_chat_backend/ + # todo: remove this + ref: feat/vector-db-wipe persist-credentials: false - name: Checkout viewer diff --git a/.github/workflows/integration-test-file-listener.yml b/.github/workflows/integration-test-file-listener.yml index 2b445242..dcb0caaa 100644 --- a/.github/workflows/integration-test-file-listener.yml +++ b/.github/workflows/integration-test-file-listener.yml @@ -115,6 +115,8 @@ jobs: with: repository: nextcloud/context_chat_backend path: context_chat_backend/ + # todo: remove this + ref: feat/vector-db-wipe persist-credentials: false - name: Checkout viewer From 416c20c215786c5683e8a22d5d041299573b9788 Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Thu, 29 Oct 2026 18:30:23 +0530 Subject: [PATCH 05/12] fix: cs fix Signed-off-by: kyteinsky --- lib/Service/ReindexService.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/Service/ReindexService.php b/lib/Service/ReindexService.php index 409c9369..2fe51cec 100644 --- a/lib/Service/ReindexService.php +++ b/lib/Service/ReindexService.php @@ -93,8 +93,8 @@ private function uniqueSorted(array $keys): array { * Jobs that are already scheduled are not scheduled again. * * @param list|null $providerKeys null means everything. Without a scope only the content - * providers that have not been imported yet are imported (which is all of them after a full wipe), - * content providers explicitly specified are set as not initiated and scheduled for import. + * providers that have not been imported yet are imported (which is all of them after a full wipe), + * content providers explicitly specified are set as not initiated and scheduled for import. * @return bool whether anything new was scheduled */ public function scheduleReindex(?array $providerKeys = null): bool { @@ -140,7 +140,7 @@ public function scheduleReindex(?array $providerKeys = null): bool { * * @param list|null $providerKeys null means everything including the actions queue, otherwise only the given providers * @param bool $ignoreHandedOut do not wait for the documents the backend is indexing, for when some documents are stuck - * in the queue for 24h due to error or if the backend stopped while indexing them/having a lock + * in the queue for 24h due to error or if the backend stopped while indexing them/having a lock * @param ?callable(string): void $onStep called with a short description before each step, for progress output * @throws \OCP\DB\Exception * @throws \RuntimeException when the vector DB could not be wiped From f4cdbe3b9c7de5970403fef39c3798020bc4d0c8 Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Wed, 4 Nov 2026 13:35:37 +0530 Subject: [PATCH 06/12] fix: pause for max 24h while wipe completes or crashes Signed-off-by: kyteinsky Assisted-by: Github Copilot:claude-opus-5-5 --- lib/Command/Wipe.php | 2 ++ lib/Service/FetchGuard.php | 17 +++++------------ tests/integration/ReindexServiceTest.php | 12 ++++++++++++ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/lib/Command/Wipe.php b/lib/Command/Wipe.php index df5639be..07334816 100644 --- a/lib/Command/Wipe.php +++ b/lib/Command/Wipe.php @@ -31,6 +31,8 @@ protected function configure() { 'Delete the indexed documents from context_chat_backend\'s vector DB and clear the queues.' . ' With --provider option, only the provider\'s data is deleted, the actions queue, and the indexing statistics are left intact.' . ' Nothing gets indexed again until context_chat:reindex is run or the --reindex option is used.' + . ' While the wipe runs, no documents or actions are handed to context_chat_backend. If the command is killed' + . ' before it ends, this pause stays for up to a day, run the command again to lift it.' ) ->addOption( 'provider', diff --git a/lib/Service/FetchGuard.php b/lib/Service/FetchGuard.php index 4ede4c99..989aac9e 100644 --- a/lib/Service/FetchGuard.php +++ b/lib/Service/FetchGuard.php @@ -21,12 +21,13 @@ * On Kubernetes the backend's indexing and updates pods are not the one serving the wipe request, so * they can only be held back from here. * - * A pause expires on its own, so a wipe that dies half way does not stop the indexing forever. + * A pause expires on its own after a day, so a wipe that dies half way does not stop the indexing forever. + * Running the wipe again lifts it right away: it starts a new pause and lifts that one when it ends. */ class FetchGuard { private const CONFIG_KEY = 'queue_fetch_paused_until'; - // on top of the longest a wipe request to the backend can take - private const MARGIN = 5 * 60; + // pause indexing while wipe happens, max 1 day so a crashed process does not stop it indefinitely + private const PAUSE_DURATION = 24 * 60 * 60; public const DEFAULT_DRAIN_TIMEOUT = 30 * 60; // 30 mins for the currently indexing documents to be done public function __construct( @@ -42,7 +43,7 @@ public function __construct( * @return int the expiry of the pause, to be passed to resume() */ public function pause(): int { - $until = $this->timeFactory->getTime() + $this->getDrainTimeout() + $this->getRequestTimeout() + self::MARGIN; + $until = $this->timeFactory->getTime() + self::PAUSE_DURATION; $this->appConfig->setAppValueInt(self::CONFIG_KEY, $until, lazy: true); return $until; } @@ -99,14 +100,6 @@ private function getDrainTimeout(): int { return $this->appConfig->getAppValueInt('wipe_drain_timeout', self::DEFAULT_DRAIN_TIMEOUT, lazy: true); } - private function getRequestTimeout(): int { - return (int)$this->appConfig->getAppValueString( - 'request_timeout', - strval(Application::CC_DEFAULT_REQUEST_TIMEOUT), - lazy: true, - ); - } - /** * Read from the database instead of IAppConfig. IAppConfig values are cached by each web server * for a few seconds, and all of them must stop handing out items before the wipe starts. diff --git a/tests/integration/ReindexServiceTest.php b/tests/integration/ReindexServiceTest.php index cda6319d..883a9bf6 100644 --- a/tests/integration/ReindexServiceTest.php +++ b/tests/integration/ReindexServiceTest.php @@ -404,6 +404,18 @@ public function testPauseExpiresOnItsOwn(): void { $guard->resume($until); } + public function testPauseOutlastsAScopedWipeOfManySlowProviders(): void { + $now = self::NOW; + $guard = $this->newGuard($this->clock($now)); + + $until = $guard->pause(); + // the drain timeout, then 20 deleteProvider requests that each run into the request timeout + $now += FetchGuard::DEFAULT_DRAIN_TIMEOUT + 20 * Application::CC_DEFAULT_REQUEST_TIMEOUT; + + $this->assertTrue($guard->isPaused(), 'indexing must not restart while providers are still being wiped'); + $guard->resume($until); + } + public function testWaitEndsOnceTheHandedOutBatchIsReportedBack(): void { $queueMapper = Server::get(QueueMapper::class); $file = new QueueFile(); From 62f512faddfc5e18cf246cd42655b0c017a7e7fd Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Thu, 5 Nov 2026 14:46:55 +0530 Subject: [PATCH 07/12] fix: do not pause for a content provider's wipe Signed-off-by: kyteinsky --- lib/Controller/QueueController.php | 8 +- lib/Db/QueueContentItemMapper.php | 17 +- lib/Service/FetchGuard.php | 66 ++++++-- lib/Service/ReindexService.php | 55 ++++-- .../QueueControllerFetchGuardTest.php | 104 ++++++++++++ .../integration/QueueControllerPauseTest.php | 132 +++++++++++++++ tests/integration/ReindexServiceTest.php | 156 +++++++++++++++--- 7 files changed, 483 insertions(+), 55 deletions(-) create mode 100644 tests/integration/QueueControllerFetchGuardTest.php create mode 100644 tests/integration/QueueControllerPauseTest.php diff --git a/lib/Controller/QueueController.php b/lib/Controller/QueueController.php index d260afd5..41f85044 100644 --- a/lib/Controller/QueueController.php +++ b/lib/Controller/QueueController.php @@ -108,16 +108,20 @@ public function getDocumentsQueueItems( if ($n > $maxN) { $n = $maxN; } + // hand out nothing, the backend waits and asks again + // wipe everything case, only files wipe or a content provider wipe does not pause here if ($this->fetchGuard->isPaused()) { return new DataResponse([ 'files' => new \stdClass(), 'content_providers' => new \stdClass(), ]); } + try { $files = []; - while (count($files) < $n) { + // a wipe of the files holds back the files only + while (count($files) < $n && !$this->fetchGuard->areFilesPaused()) { $limit = $n - count($files); $documents = $queueMapper->getFromQueue($limit); if (empty($documents)) { @@ -134,6 +138,7 @@ public function getDocumentsQueueItems( } } } + $contentItems = []; while (count($contentItems) < $n) { $limit = $n - count($contentItems); @@ -147,6 +152,7 @@ public function getDocumentsQueueItems( } } } + return new DataResponse([ 'files' => (object)$files, 'content_providers' => (object)$contentItems, diff --git a/lib/Db/QueueContentItemMapper.php b/lib/Db/QueueContentItemMapper.php index b61f1cd6..135e7ba8 100644 --- a/lib/Db/QueueContentItemMapper.php +++ b/lib/Db/QueueContentItemMapper.php @@ -72,14 +72,25 @@ public function removeFromQueue(array $ids): void { } /** + * @param bool $onlyUnlocked keep the items being indexed, i.e. locked within the lock timeout * @throws \OCP\DB\Exception */ - public function removeProvider(string $appId, string $providerId): void { + public function removeProvider(string $appId, string $providerId, bool $onlyUnlocked = false): void { $qb = $this->db->getQueryBuilder(); $qb->delete($this->getTableName()) ->where($qb->expr()->eq('app_id', $qb->createNamedParameter($appId))) - ->andWhere($qb->expr()->eq('provider_id', $qb->createNamedParameter($providerId))) - ->executeStatement(); + ->andWhere($qb->expr()->eq('provider_id', $qb->createNamedParameter($providerId))); + if ($onlyUnlocked) { + // the same items getFromQueue() can hand out + $qb->andWhere($qb->expr()->orX( + $qb->expr()->isNull('locked_at'), + $qb->expr()->lte('locked_at', $qb->createNamedParameter( + (new \DateTime())->sub(new \DateInterval('PT' . self::LOCK_TIMEOUT . 'S')), + IQueryBuilder::PARAM_DATETIME_MUTABLE + )), + )); + } + $qb->executeStatement(); } /** diff --git a/lib/Service/FetchGuard.php b/lib/Service/FetchGuard.php index 989aac9e..707c6bf0 100644 --- a/lib/Service/FetchGuard.php +++ b/lib/Service/FetchGuard.php @@ -21,11 +21,15 @@ * On Kubernetes the backend's indexing and updates pods are not the one serving the wipe request, so * they can only be held back from here. * + * A full wipe holds back everything while a wipe of the files provider only the files. A wipe of any + * content provider does not pause, it clears the provider's queue before it waits for the handed out documents instead. + * * A pause expires on its own after a day, so a wipe that dies half way does not stop the indexing forever. * Running the wipe again lifts it right away: it starts a new pause and lifts that one when it ends. */ class FetchGuard { - private const CONFIG_KEY = 'queue_fetch_paused_until'; + private const ALL_KEY = 'queue_fetch_paused_until'; + private const FILES_KEY = 'files_fetch_paused_until'; // pause indexing while wipe happens, max 1 day so a crashed process does not stop it indefinitely private const PAUSE_DURATION = 24 * 60 * 60; public const DEFAULT_DRAIN_TIMEOUT = 30 * 60; // 30 mins for the currently indexing documents to be done @@ -40,40 +44,56 @@ public function __construct( } /** - * @return int the expiry of the pause, to be passed to resume() + * @param bool $onlyFiles hold back the files only, for a wipe of the files + * @return array{0: string, 1: int} the pause, to be passed to resume() */ - public function pause(): int { + public function pause(bool $onlyFiles = false): array { + $key = $onlyFiles ? self::FILES_KEY : self::ALL_KEY; $until = $this->timeFactory->getTime() + self::PAUSE_DURATION; - $this->appConfig->setAppValueInt(self::CONFIG_KEY, $until, lazy: true); - return $until; + $this->appConfig->setAppValueInt($key, $until, lazy: true); + return [$key, $until]; } /** - * Lifts the pause, unless another one was started since. + * Lifts the pause, unless another one of the same kind was started since. + * + * @param array{0: string, 1: int} $pause */ - public function resume(int $until): void { - if ($this->getPausedUntil() === $until) { - $this->appConfig->deleteAppValue(self::CONFIG_KEY); + public function resume(array $pause): void { + [$key, $until] = $pause; + if ($this->readUntil($key) === $until) { + $this->appConfig->deleteAppValue($key); } } + /** + * Whether a full wipe is running: nothing is handed out, not even actions. + */ public function isPaused(): bool { - return $this->getPausedUntil() > $this->timeFactory->getTime(); + return $this->readUntil(self::ALL_KEY) > $this->timeFactory->getTime(); } /** - * Waits for the documents the backend took before the pause to be reported back. A document that - * is still being indexed would be written to the vector DB after it was wiped. + * Whether the files are held back, by a full wipe or a wipe of the files. + */ + public function areFilesPaused(): bool { + return $this->isPaused() || $this->readUntil(self::FILES_KEY) > $this->timeFactory->getTime(); + } + + /** + * Waits for the documents the backend took to be reported back. A document that is still being indexed + * would be written to the vector DB after it was wiped. * + * @param list|null $providerKeys only wait for the documents of these providers, null for all of them * @param ?callable(int): void $onWaiting called with the number of documents still being indexed, * once before waiting and again whenever that number changes * @throws \RuntimeException when they are not reported back in time * @throws \OCP\DB\Exception */ - public function waitForHandedOutDocuments(?callable $onWaiting = null): void { + public function waitForHandedOutDocuments(?array $providerKeys = null, ?callable $onWaiting = null): void { $deadline = $this->timeFactory->getTime() + $this->getDrainTimeout(); $reported = null; - while (($locked = $this->countHandedOutDocuments()) > 0) { + while (($locked = $this->countHandedOutDocuments($providerKeys)) > 0) { if ($onWaiting !== null && $locked !== $reported) { $onWaiting($locked); $reported = $locked; @@ -90,10 +110,20 @@ public function waitForHandedOutDocuments(?callable $onWaiting = null): void { } /** + * @param list|null $providerKeys * @throws \OCP\DB\Exception */ - private function countHandedOutDocuments(): int { - return $this->queueMapper->countLocked() + intval(array_sum($this->contentQueue->countLocked())); + private function countHandedOutDocuments(?array $providerKeys): int { + $count = 0; + if ($providerKeys === null || in_array(ProviderConfigService::getDefaultProviderKey(), $providerKeys, true)) { + $count += $this->queueMapper->countLocked(); + } + foreach ($this->contentQueue->countLocked() as $providerKey => $locked) { + if ($providerKeys === null || in_array($providerKey, $providerKeys, true)) { + $count += (int)$locked; + } + } + return $count; } private function getDrainTimeout(): int { @@ -105,12 +135,12 @@ private function getDrainTimeout(): int { * for a few seconds, and all of them must stop handing out items before the wipe starts. * This ensures that we get the most up-to-date value for the pause state. */ - private function getPausedUntil(): int { + private function readUntil(string $key): int { $qb = $this->db->getQueryBuilder(); $result = $qb->select('configvalue') ->from('appconfig') ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID))) - ->andWhere($qb->expr()->eq('configkey', $qb->createNamedParameter(self::CONFIG_KEY))) + ->andWhere($qb->expr()->eq('configkey', $qb->createNamedParameter($key))) ->executeQuery(); $value = $result->fetchOne(); $result->closeCursor(); diff --git a/lib/Service/ReindexService.php b/lib/Service/ReindexService.php index 2fe51cec..c5bb2730 100644 --- a/lib/Service/ReindexService.php +++ b/lib/Service/ReindexService.php @@ -133,9 +133,13 @@ public function scheduleReindex(?array $providerKeys = null): bool { /** * Delete everything that was queued or indexed, here and in context_chat_backend's vector DB. - * No queue items are handed to the backend until the wipe is over, and the documents it is still - * indexing are waited for, so that its workers, possibly running elsewhere on Kubernetes, cannot - * write to the vector DB while it is wiped. + * The documents the backend is still indexing are waited for here, so that its workers, possibly running + * on Kubernetes, cannot write them to the vector DB after it was wiped. + * + * A full wipe holds back all the queues until it is over, a wipe of the files only holds the files queue. + * A content provider is not held back since its unlocked/untouched queued items are deleted before the wait, + * so once the wait is over, none of what was queued before the wipe can still be indexed. Items the provider + * submits during the wipe may be indexed before or after the delete in the backend. * It is safe to run this again when the backend request fails. * * @param list|null $providerKeys null means everything including the actions queue, otherwise only the given providers @@ -149,14 +153,29 @@ public function wipe(?array $providerKeys = null, bool $ignoreHandedOut = false, $onStep = $onStep ?? static function (string $_): void { }; - $onStep('Pausing the hand-out of queued documents and actions to context_chat_backend'); - $pausedUntil = $this->fetchGuard->pause(); + $filesKey = ProviderConfigService::getDefaultProviderKey(); + $providerKeys = $providerKeys === null ? null : array_values(array_unique($providerKeys)); + $contentProviderKeys = $providerKeys === null ? [] : array_values(array_diff($providerKeys, [$filesKey])); + + $pause = null; + if ($providerKeys === null) { + $onStep('Pausing the hand-out of queued documents and actions to context_chat_backend'); + $pause = $this->fetchGuard->pause(); + } elseif (in_array($filesKey, $providerKeys, true)) { + $onStep('Pausing the hand-out of queued files to context_chat_backend'); + $pause = $this->fetchGuard->pause(onlyFiles: true); + } try { + foreach ($contentProviderKeys as $providerKey) { + $onStep('Removing the import job and clearing the indexing queue of ' . $providerKey); + $this->clearContentProviderState($providerKey); + } + if ($ignoreHandedOut) { $onStep('Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)'); } else { $onStep('Checking for documents context_chat_backend is still indexing'); - $this->fetchGuard->waitForHandedOutDocuments(static function (int $locked) use ($onStep): void { + $this->fetchGuard->waitForHandedOutDocuments($providerKeys, static function (int $locked) use ($onStep): void { $onStep('Waiting for context_chat_backend to finish indexing ' . $locked . ' documents'); }); } @@ -166,21 +185,27 @@ public function wipe(?array $providerKeys = null, bool $ignoreHandedOut = false, return; } - foreach (array_unique($providerKeys) as $providerKey) { + foreach ($providerKeys as $providerKey) { $this->logger->info('Wiping the Context Chat data of provider ' . $providerKey); - if ($providerKey === ProviderConfigService::getDefaultProviderKey()) { + if ($providerKey === $filesKey) { $onStep('Removing the crawl jobs, clearing the files indexing queue and the file system events, resetting the indexing statistics'); $this->clearFilesState(); } else { - $onStep('Removing the import job and clearing the indexing queue of ' . $providerKey); - $this->clearContentProviderState($providerKey); + // `clearContentProviderState()` already ran so the content queue may be empty + // but clean it up again since with --ignore-handed-out locked rows are not waited for to be cleared + [$appId, $providerId] = $this->providerConfig->getProviderParts($providerKey); + $this->contentQueue->removeProvider($appId, $providerId); } $onStep('Deleting the documents of ' . $providerKey . ' from the vector DB'); $this->langRopeService->deleteProvider($providerKey); } } finally { - $onStep('Resuming the hand-out of queued documents and actions'); - $this->fetchGuard->resume($pausedUntil); + if ($pause !== null) { + $onStep($providerKeys === null + ? 'Resuming the hand-out of queued documents and actions' + : 'Resuming the hand-out of queued files'); + $this->fetchGuard->resume($pause); + } } } @@ -219,6 +244,10 @@ private function clearFilesState(): void { $this->resetStats(); } + /** + * Leaves the items the backend is indexing in the queue, so the wipe can wait for them, and deletes the rest. + * Deleted items can no longer be locked, so nothing more of the provider's queue can be handed out. + */ private function clearContentProviderState(string $providerKey): void { $provider = $this->providerConfig->getProvider($providerKey); if ($provider !== null && $provider['classString'] !== '') { @@ -226,7 +255,7 @@ private function clearContentProviderState(string $providerKey): void { } [$appId, $providerId] = $this->providerConfig->getProviderParts($providerKey); - $this->contentQueue->removeProvider($appId, $providerId); + $this->contentQueue->removeProvider($appId, $providerId, onlyUnlocked: true); if ($provider !== null) { $this->providerConfig->setProviderInitiated($providerKey, false); diff --git a/tests/integration/QueueControllerFetchGuardTest.php b/tests/integration/QueueControllerFetchGuardTest.php new file mode 100644 index 00000000..ef5dc8fb --- /dev/null +++ b/tests/integration/QueueControllerFetchGuardTest.php @@ -0,0 +1,104 @@ +controller = Server::get(QueueController::class); + $this->queueMapper = Server::get(QueueMapper::class); + $this->actionMapper = Server::get(QueueActionMapper::class); + $this->guard = (new Application())->getContainer()->get(FetchGuard::class); + $this->queueMapper->clearQueue(); + $this->actionMapper->removeFromQueue($this->allActionIds()); + + $file = new QueueFile(); + $file->setFileId(42); + $file->setStorageId(1); + $file->setRootId(1); + $file->setUpdate(false); + $this->queueMapper->insertIntoQueue($file); + + $action = new QueueAction(); + $action->setType(ActionType::DELETE_SOURCE_IDS); + $action->setPayload('{"sourceIds":["files__default: 42"]}'); + $this->actionMapper->insertIntoQueue($action); + } + + protected function tearDown(): void { + if ($this->pausedUntil !== null) { + $this->guard->resume($this->pausedUntil); + } + $this->queueMapper->clearQueue(); + $this->actionMapper->removeFromQueue($this->allActionIds()); + parent::tearDown(); + } + + /** @return list */ + private function allActionIds(): array { + return array_map(static fn (QueueAction $a) => $a->getId(), $this->actionMapper->getFromQueue(1000)); + } + + public function testNothingIsHandedOutOrLockedWhilePaused(): void { + $this->pausedUntil = $this->guard->pause(); + + $documents = $this->controller->getDocumentsQueueItems( + Server::get(IRootFolder::class), + $this->queueMapper, + Server::get(QueueContentItemMapper::class), + Server::get(\OCP\Files\Config\IUserMountCache::class), + ); + $actions = $this->controller->getActionsQueueItems($this->actionMapper); + + $this->assertSame(Http::STATUS_OK, $documents->getStatus()); + $this->assertEquals(['files' => new \stdClass(), 'content_providers' => new \stdClass()], $documents->getData()); + $this->assertEquals(['actions' => new \stdClass()], $actions->getData()); + $this->assertSame(0, $this->queueMapper->countLocked(), 'a paused fetch must not lock queue items'); + $this->assertSame(1, $this->queueMapper->count()); + $this->assertSame(0, $this->actionMapper->countLocked()); + $this->assertSame(1, $this->actionMapper->count()); + } + + public function testActionsAreHandedOutAgainAfterResume(): void { + $this->pausedUntil = $this->guard->pause(); + $this->guard->resume($this->pausedUntil); + $this->pausedUntil = null; + + $actions = $this->controller->getActionsQueueItems($this->actionMapper); + + $this->assertCount(1, (array)$actions->getData()['actions']); + } +} diff --git a/tests/integration/QueueControllerPauseTest.php b/tests/integration/QueueControllerPauseTest.php new file mode 100644 index 00000000..6bb15d76 --- /dev/null +++ b/tests/integration/QueueControllerPauseTest.php @@ -0,0 +1,132 @@ + */ + private array $pauses = []; + + protected function setUp(): void { + parent::setUp(); + $this->db = Server::get(IDBConnection::class); + $this->fetchGuard = Server::get(FetchGuard::class); + $this->clearQueues(); + + $item = new QueueContentItem(); + $item->setItemId('item1'); + $item->setAppId('someapp'); + $item->setProviderId('prov'); + $item->setTitle('title'); + $item->setContent('content'); + $item->setDocumentType('text'); + $item->setLastModified(new \DateTime()); + $item->setUsers('admin'); + Server::get(QueueContentItemMapper::class)->insert($item); + + // a file that cannot be resolved: when it is handed out it is dropped from the queue + $file = new QueueFile(); + $file->setFileId(PHP_INT_MAX >> 16); + $file->setStorageId(1); + $file->setRootId(1); + $file->setUpdate(false); + Server::get(QueueMapper::class)->insertIntoQueue($file); + + $action = new QueueAction(); + $action->setType(ActionType::DELETE_SOURCE_IDS); + $action->setPayload('{"sourceIds":["someapp__prov: item1"]}'); + Server::get(QueueActionMapper::class)->insertIntoQueue($action); + } + + protected function tearDown(): void { + foreach ($this->pauses as $pause) { + $this->fetchGuard->resume($pause); + } + $this->clearQueues(); + parent::tearDown(); + } + + private function clearQueues(): void { + foreach (['context_chat_queue', 'context_chat_content_queue', 'context_chat_action_queue'] as $table) { + $this->db->getQueryBuilder()->delete($table)->executeStatement(); + } + } + + /** + * Arguments are resolved by type from the DI container, like the AppFramework dispatcher does. + * + * @return array + */ + private function call(string $endpoint): array { + $controller = Server::get(QueueController::class); + $method = new \ReflectionMethod($controller, $endpoint); + $args = []; + foreach ($method->getParameters() as $parameter) { + $type = $parameter->getType(); + $args[] = ($type instanceof \ReflectionNamedType && !$type->isBuiltin()) + ? Server::get($type->getName()) + : 64; + } + $response = $method->invokeArgs($controller, $args); + $this->assertSame(Http::STATUS_OK, $response->getStatus()); + return $response->getData(); + } + + private function filesInQueue(): int { + return Server::get(QueueMapper::class)->count(); + } + + public function testFilesWipeHoldsBackOnlyTheFiles(): void { + $this->pauses[] = $this->fetchGuard->pause(onlyFiles: true); + + $response = $this->call('getDocumentsQueueItems'); + + $this->assertCount(1, (array)$response['content_providers']); + $this->assertSame(1, $this->filesInQueue(), 'the file must not be touched'); + $this->assertCount(1, (array)$this->call('getActionsQueueItems')['actions']); + } + + public function testFullWipeHoldsBackEverything(): void { + $this->pauses[] = $this->fetchGuard->pause(); + + $response = $this->call('getDocumentsQueueItems'); + + $this->assertSame([], (array)$response['content_providers']); + $this->assertSame(1, $this->filesInQueue()); + $this->assertSame([], (array)$this->call('getActionsQueueItems')['actions']); + } + + public function testNothingIsHeldBackWithoutAWipe(): void { + $response = $this->call('getDocumentsQueueItems'); + + $this->assertCount(1, (array)$response['content_providers']); + $this->assertSame(0, $this->filesInQueue(), 'the file was handed out and dropped'); + } +} diff --git a/tests/integration/ReindexServiceTest.php b/tests/integration/ReindexServiceTest.php index 883a9bf6..5d7254d1 100644 --- a/tests/integration/ReindexServiceTest.php +++ b/tests/integration/ReindexServiceTest.php @@ -130,16 +130,7 @@ private function seedIndexedState(): void { Server::get(QueueMapper::class)->insertIntoQueue($file); foreach (['someapp', 'otherapp'] as $appId) { - $item = new QueueContentItem(); - $item->setItemId('item1'); - $item->setAppId($appId); - $item->setProviderId('prov'); - $item->setTitle('title'); - $item->setContent('content'); - $item->setDocumentType('text'); - $item->setLastModified(new \DateTime()); - $item->setUsers('admin'); - Server::get(QueueContentItemMapper::class)->insert($item); + $this->queueContentItem($appId, 'item1'); } $action = new QueueAction(); @@ -163,6 +154,32 @@ private function seedIndexedState(): void { $this->providerConfig->updateProvider('otherapp', 'prov', self::OTHER_PROVIDER_CLASS, true); } + private function queueContentItem(string $appId, string $itemId): void { + $item = new QueueContentItem(); + $item->setItemId($itemId); + $item->setAppId($appId); + $item->setProviderId('prov'); + $item->setTitle('title'); + $item->setContent('content'); + $item->setDocumentType('text'); + $item->setLastModified(new \DateTime()); + $item->setUsers('admin'); + Server::get(QueueContentItemMapper::class)->insert($item); + } + + /** @return list appId/itemId of the content items in the queue */ + private function contentQueueItems(): array { + $qb = $this->db->getQueryBuilder(); + $result = $qb->select('app_id', 'item_id')->from('context_chat_content_queue')->executeQuery(); + $items = []; + while (($row = $result->fetch()) !== false) { + $items[] = $row['app_id'] . '/' . $row['item_id']; + } + $result->closeCursor(); + sort($items); + return $items; + } + /** @return list */ private function contentQueueAppIds(): array { $qb = $this->db->getQueryBuilder(); @@ -396,24 +413,104 @@ public function testPauseExpiresOnItsOwn(): void { $now = self::NOW; $guard = $this->newGuard($this->clock($now)); - $until = $guard->pause(); + $pause = $guard->pause(); $this->assertTrue($guard->isPaused()); - $now = $until; + $now += FetchGuard::PAUSE_DURATION; $this->assertFalse($guard->isPaused(), 'a wipe that died must not stop the indexing forever'); - $guard->resume($until); + $guard->resume($pause); } public function testPauseOutlastsAScopedWipeOfManySlowProviders(): void { $now = self::NOW; $guard = $this->newGuard($this->clock($now)); - $until = $guard->pause(); + $pause = $guard->pause(onlyFiles: true); // the drain timeout, then 20 deleteProvider requests that each run into the request timeout $now += FetchGuard::DEFAULT_DRAIN_TIMEOUT + 20 * Application::CC_DEFAULT_REQUEST_TIMEOUT; - $this->assertTrue($guard->isPaused(), 'indexing must not restart while providers are still being wiped'); - $guard->resume($until); + $this->assertTrue($guard->areFilesPaused(), 'indexing must not restart while providers are still being wiped'); + $guard->resume($pause); + } + + public function testFullAndFilesPausesAreLiftedSeparately(): void { + $files = $this->fetchGuard->pause(onlyFiles: true); + $all = $this->fetchGuard->pause(); + $this->fetchGuard->resume($all); + + $this->assertTrue($this->fetchGuard->areFilesPaused(), 'the files wipe is still running'); + $this->fetchGuard->resume($files); + $this->assertFalse($this->fetchGuard->areFilesPaused()); + } + + public function testFilesWipePausesOnlyTheFiles(): void { + $this->seedIndexedState(); + $seen = null; + $this->langRope->method('deleteProvider')->willReturnCallback(function () use (&$seen) { + $seen = [$this->fetchGuard->areFilesPaused(), $this->fetchGuard->isPaused()]; + }); + + $this->service->wipe(['files__default']); + + $this->assertSame([true, false], $seen); + $this->assertFalse($this->fetchGuard->areFilesPaused()); + } + + public function testContentProviderWipeOnlyWaitsForItsOwnDocuments(): void { + $this->seedIndexedState(); + // a file is being indexed, unrelated to the provider being wiped + $queueMapper = Server::get(QueueMapper::class); + $queueMapper->lock($queueMapper->getMaxId()); + $this->appConfig->setAppValueInt('wipe_drain_timeout', 0, lazy: true); + $this->langRope->expects($this->once())->method('deleteProvider')->with('someapp__prov'); + + try { + $this->service->wipe(['someapp__prov']); + } finally { + $this->appConfig->deleteAppValue('wipe_drain_timeout'); + } + + $this->assertSame(1, $queueMapper->countLocked(), 'the file being indexed is left alone'); + } + + /** + * Nothing pauses a content provider, so its queue is cleared before the wait: what is not handed out yet + * can no longer be, and what is handed out is waited for. + */ + public function testContentProviderWipeClearsTheQueueBeforeWaitingForTheHandedOutItems(): void { + $this->seedIndexedState(); + $contentQueue = Server::get(QueueContentItemMapper::class); + $this->queueContentItem('someapp', 'item2'); + foreach ($contentQueue->getFromQueue(10) as $item) { + if ($item->getAppId() === 'someapp' && $item->getItemId() === 'item1') { + $contentQueue->lock($item->getId()); + } + } + $this->appConfig->setAppValueInt('wipe_drain_timeout', 0, lazy: true); + $this->langRope->expects($this->never())->method('deleteProvider'); + + try { + $this->service->wipe(['someapp__prov']); + $this->fail('the wipe must wait for the item being indexed'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('1 documents are still being indexed', $e->getMessage()); + } finally { + $this->appConfig->deleteAppValue('wipe_drain_timeout'); + } + + $this->assertSame(['otherapp/item1', 'someapp/item1'], $this->contentQueueItems(), 'item2 must be gone, it could have been handed out during the wait'); + } + + public function testContentProviderWipeWithIgnoreHandedOutClearsItsWholeQueue(): void { + $this->seedIndexedState(); + $contentQueue = Server::get(QueueContentItemMapper::class); + foreach ($contentQueue->getFromQueue(10) as $item) { + $contentQueue->lock($item->getId()); + } + + $this->service->wipe(['someapp__prov'], ignoreHandedOut: true); + + $this->assertSame(['otherapp'], $this->contentQueueAppIds()); } public function testWaitEndsOnceTheHandedOutBatchIsReportedBack(): void { @@ -437,7 +534,7 @@ public function testWaitEndsOnceTheHandedOutBatchIsReportedBack(): void { }); $reported = []; - $this->newGuard($time)->waitForHandedOutDocuments(function (int $locked) use (&$reported) { + $this->newGuard($time)->waitForHandedOutDocuments(null, function (int $locked) use (&$reported) { $reported[] = $locked; }); @@ -484,12 +581,31 @@ public function testWipeOfOneProviderReportsItsSteps(): void { }); $this->assertSame([ - 'Pausing the hand-out of queued documents and actions to context_chat_backend', - 'Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)', 'Removing the import job and clearing the indexing queue of someapp__prov', + 'Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)', 'Deleting the documents of someapp__prov from the vector DB', 'backend:deleteProvider someapp__prov', - 'Resuming the hand-out of queued documents and actions', + ], $events, 'a content provider wipe does not pause'); + } + + public function testWipeOfTheFilesReportsItsSteps(): void { + $this->seedIndexedState(); + $events = []; + $this->langRope->method('deleteProvider')->willReturnCallback(function (string $key) use (&$events) { + $events[] = 'backend:deleteProvider ' . $key; + }); + + $this->service->wipe(['files__default'], true, function (string $message) use (&$events) { + $events[] = $message; + }); + + $this->assertSame([ + 'Pausing the hand-out of queued files to context_chat_backend', + 'Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)', + 'Removing the crawl jobs, clearing the files indexing queue and the file system events, resetting the indexing statistics', + 'Deleting the documents of files__default from the vector DB', + 'backend:deleteProvider files__default', + 'Resuming the hand-out of queued files', ], $events); } From 3c7a1dbb4fcc72fbecf94958e7a9c6eceb573a1b Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Thu, 5 Nov 2026 15:16:25 +0530 Subject: [PATCH 08/12] fix: check wipe's pause again after locking queue items Signed-off-by: kyteinsky --- lib/Controller/QueueController.php | 21 +++++ lib/Db/QueueActionMapper.php | 13 +++ lib/Db/QueueContentItemMapper.php | 13 +++ lib/Db/QueueMapper.php | 13 +++ .../integration/QueueControllerPauseTest.php | 80 ++++++++++++++++++- 5 files changed, 138 insertions(+), 2 deletions(-) diff --git a/lib/Controller/QueueController.php b/lib/Controller/QueueController.php index 41f85044..09f16615 100644 --- a/lib/Controller/QueueController.php +++ b/lib/Controller/QueueController.php @@ -153,6 +153,22 @@ public function getDocumentsQueueItems( } } + // A wipe may have started after the pause check. Do a check again here before returning the + // response to see if that happened. If the pause did happen in between the checks, + // wipe would be waiting for the locks to unlock, which we do, or proceed with the cleanup, which + // is fine too since we don't return the docs to the backend, it won't be indexing them. + // + // Only the files and a full wipe pause. A wipe of a content provider deletes the provider's + // items that are not locked, so a fetch can no longer lock them, and waits for the locked ones. + if ($files !== [] && $this->fetchGuard->areFilesPaused()) { + $queueMapper->unlock(array_keys($files)); + $files = []; + } + if ($contentItems !== [] && $this->fetchGuard->isPaused()) { + $queueContentItemMapper->unlock(array_keys($contentItems)); + $contentItems = []; + } + return new DataResponse([ 'files' => (object)$files, 'content_providers' => (object)$contentItems, @@ -249,6 +265,11 @@ public function getActionsQueueItems(QueueActionMapper $queueActionMapper, int $ } } } + // see getDocumentsQueueItems(), a full wipe may have started since the previous pause check above + if ($actions !== [] && $this->fetchGuard->isPaused()) { + $queueActionMapper->unlock(array_keys($actions)); + $actions = []; + } return new DataResponse(['actions' => (object)$actions]); } catch (Exception $e) { $this->logger->error($e->getMessage(), ['exception' => $e]); diff --git a/lib/Db/QueueActionMapper.php b/lib/Db/QueueActionMapper.php index ac367dfb..95699018 100644 --- a/lib/Db/QueueActionMapper.php +++ b/lib/Db/QueueActionMapper.php @@ -154,4 +154,17 @@ public function lock(int $id) : bool { } return false; } + + /** + * @throws Exception + */ + public function unlock(array $ids): void { + foreach (array_chunk($ids, 1000) as $chunk) { + $qb = $this->db->getQueryBuilder(); + $qb->update($this->getTableName()) + ->set('locked_at', $qb->createNamedParameter(null, IQueryBuilder::PARAM_NULL)) + ->where($qb->expr()->in('id', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY))) + ->executeStatement(); + } + } } diff --git a/lib/Db/QueueContentItemMapper.php b/lib/Db/QueueContentItemMapper.php index 135e7ba8..6b3e82ae 100644 --- a/lib/Db/QueueContentItemMapper.php +++ b/lib/Db/QueueContentItemMapper.php @@ -196,4 +196,17 @@ public function lock(int $id) : bool { } return false; } + + /** + * @throws Exception + */ + public function unlock(array $ids): void { + foreach (array_chunk($ids, 1000) as $chunk) { + $qb = $this->db->getQueryBuilder(); + $qb->update($this->getTableName()) + ->set('locked_at', $qb->createNamedParameter(null, IQueryBuilder::PARAM_NULL)) + ->where($qb->expr()->in('id', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY))) + ->executeStatement(); + } + } } diff --git a/lib/Db/QueueMapper.php b/lib/Db/QueueMapper.php index c444e827..37cb3777 100644 --- a/lib/Db/QueueMapper.php +++ b/lib/Db/QueueMapper.php @@ -244,4 +244,17 @@ public function lock(int $id) : bool { } return false; } + + /** + * @throws Exception + */ + public function unlock(array $ids): void { + foreach (array_chunk($ids, 1000) as $chunk) { + $qb = $this->db->getQueryBuilder(); + $qb->update($this->getTableName()) + ->set('locked_at', $qb->createNamedParameter(null, IQueryBuilder::PARAM_NULL)) + ->where($qb->expr()->in('id', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY))) + ->executeStatement(); + } + } } diff --git a/tests/integration/QueueControllerPauseTest.php b/tests/integration/QueueControllerPauseTest.php index 6bb15d76..b173852a 100644 --- a/tests/integration/QueueControllerPauseTest.php +++ b/tests/integration/QueueControllerPauseTest.php @@ -19,7 +19,9 @@ use OCA\ContextChat\Service\FetchGuard; use OCA\ContextChat\Type\ActionType; use OCP\AppFramework\Http; +use OCP\Files\IRootFolder; use OCP\IDBConnection; +use OCP\IUserManager; use OCP\Server; use PHPUnit\Framework\Attributes\Group; use PHPUnit\Framework\TestCase; @@ -82,16 +84,17 @@ private function clearQueues(): void { /** * Arguments are resolved by type from the DI container, like the AppFramework dispatcher does. * + * @param array $services passed instead of the container's * @return array */ - private function call(string $endpoint): array { + private function call(string $endpoint, array $services = []): array { $controller = Server::get(QueueController::class); $method = new \ReflectionMethod($controller, $endpoint); $args = []; foreach ($method->getParameters() as $parameter) { $type = $parameter->getType(); $args[] = ($type instanceof \ReflectionNamedType && !$type->isBuiltin()) - ? Server::get($type->getName()) + ? ($services[$type->getName()] ?? Server::get($type->getName())) : 64; } $response = $method->invokeArgs($controller, $args); @@ -129,4 +132,77 @@ public function testNothingIsHeldBackWithoutAWipe(): void { $this->assertCount(1, (array)$response['content_providers']); $this->assertSame(0, $this->filesInQueue(), 'the file was handed out and dropped'); } + + /** + * A full wipe that starts while a fetch is locking its batch counts the handed out documents before the fetch + * returns. The fetch must notice the pause after locking and put the items back, or they would be indexed + * after the wipe without the wipe having waited for them. + */ + public function testItemsLockedAsAFullWipeStartsArePutBack(): void { + $real = Server::get(QueueContentItemMapper::class); + $contentQueue = $this->createMock(QueueContentItemMapper::class); + $contentQueue->method('getFromQueue')->willReturnCallback(fn (int $limit) => $real->getFromQueue($limit)); + $contentQueue->method('lock')->willReturnCallback(function (int $id) use ($real): bool { + $locked = $real->lock($id); + $this->pauses[] = $this->fetchGuard->pause(); + return $locked; + }); + $contentQueue->method('unlock')->willReturnCallback(fn (array $ids) => $real->unlock($ids)); + + $response = $this->call('getDocumentsQueueItems', [QueueContentItemMapper::class => $contentQueue]); + + $this->assertSame([], (array)$response['content_providers']); + $this->assertSame([], $real->countLocked(), 'the item must be back in the queue, the wipe would wait for it'); + } + + public function testActionsLockedAsAFullWipeStartsArePutBack(): void { + $real = Server::get(QueueActionMapper::class); + $actionQueue = $this->createMock(QueueActionMapper::class); + $actionQueue->method('getFromQueue')->willReturnCallback(fn (int $limit) => $real->getFromQueue($limit)); + $actionQueue->method('lock')->willReturnCallback(function (int $id) use ($real): bool { + $locked = $real->lock($id); + $this->pauses[] = $this->fetchGuard->pause(); + return $locked; + }); + $actionQueue->method('unlock')->willReturnCallback(fn (array $ids) => $real->unlock($ids)); + + $response = $this->call('getActionsQueueItems', [QueueActionMapper::class => $actionQueue]); + + $this->assertSame([], (array)$response['actions']); + $this->assertSame(0, $real->countLocked(), 'the action must be back in the queue'); + } + + public function testFilesLockedAsAFilesWipeStartsArePutBack(): void { + $owner = 'cc_pause_' . substr(md5(uniqid('', true)), 0, 8); + $user = Server::get(IUserManager::class)->createUser($owner, 'Pass-' . $owner . '-1234567890'); + try { + $userFolder = Server::get(IRootFolder::class)->getUserFolder($owner); + $file = $userFolder->newFile('doc.txt', 'content'); + $this->clearQueues(); + $queued = new QueueFile(); + $queued->setFileId($file->getId()); + $queued->setStorageId($userFolder->getStorage()->getCache()->getNumericStorageId()); + $queued->setRootId($userFolder->getId()); + $queued->setUpdate(false); + $real = Server::get(QueueMapper::class); + $real->insertIntoQueue($queued); + + $filesQueue = $this->createMock(QueueMapper::class); + $filesQueue->method('getFromQueue')->willReturnCallback(fn (int $limit) => $real->getFromQueue($limit)); + $filesQueue->method('lock')->willReturnCallback(function (int $id) use ($real): bool { + $locked = $real->lock($id); + $this->pauses[] = $this->fetchGuard->pause(onlyFiles: true); + return $locked; + }); + $filesQueue->method('unlock')->willReturnCallback(fn (array $ids) => $real->unlock($ids)); + + $response = $this->call('getDocumentsQueueItems', [QueueMapper::class => $filesQueue]); + + $this->assertSame([], (array)$response['files']); + $this->assertSame(1, $real->count(), 'the file must be back in the queue'); + $this->assertSame(0, $real->countLocked()); + } finally { + $user?->delete(); + } + } } From 811186436401ac6097ac965583099c3f7999025c Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Thu, 5 Nov 2026 18:53:32 +0530 Subject: [PATCH 09/12] fix: wait for file crawl and provider import jobs before a wipe Signed-off-by: kyteinsky --- .../InitialContentImportJob.php | 30 +- lib/BackgroundJobs/SchedulerJob.php | 35 +- lib/BackgroundJobs/StorageCrawlJob.php | 11 + lib/Controller/QueueController.php | 5 +- lib/Service/FetchGuard.php | 223 ++++++++--- lib/Service/ReindexService.php | 29 +- .../integration/QueueControllerPauseTest.php | 15 +- tests/integration/ReindexServiceTest.php | 34 +- tests/integration/WipeBackgroundJobsTest.php | 368 ++++++++++++++++++ 9 files changed, 658 insertions(+), 92 deletions(-) create mode 100644 tests/integration/WipeBackgroundJobsTest.php diff --git a/lib/BackgroundJobs/InitialContentImportJob.php b/lib/BackgroundJobs/InitialContentImportJob.php index dbf68163..4df3b05c 100644 --- a/lib/BackgroundJobs/InitialContentImportJob.php +++ b/lib/BackgroundJobs/InitialContentImportJob.php @@ -11,6 +11,7 @@ use OCA\ContextChat\Logger; use OCA\ContextChat\Public\IContentProvider as OCAIContentProvider; +use OCA\ContextChat\Service\FetchGuard; use OCA\ContextChat\Service\ProviderConfigService; use OCP\App\IAppManager; use OCP\AppFramework\Utility\ITimeFactory; @@ -29,6 +30,7 @@ public function __construct( private IUserManager $userMan, ITimeFactory $timeFactory, private ?string $userId, + private FetchGuard $fetchGuard, ) { parent::__construct($timeFactory); } @@ -56,16 +58,26 @@ protected function run($argument): void { return; } - $registeredProviders = $this->providerConfig->getProviders(); $identifier = ProviderConfigService::getConfigKey($providerObj->getAppId(), $providerObj->getId()); - if (!isset($registeredProviders[$identifier]) - || $registeredProviders[$identifier]['isInitiated'] - ) { - $this->logger->info('[InitialContentImportJob] Provider has already been initiated, skipping content import', ['provider' => $identifier]); - return; - } + $running = $this->fetchGuard->jobStarted($identifier, $this->getId(), FetchGuard::IMPORT_RUNNING_TTL); + try { + // a wipe of the provider removes this job, a re-index after it adds it again + if ($this->fetchGuard->isPaused($identifier)) { + return; + } - $providerObj->triggerInitialImport(); - $this->providerConfig->updateProvider($providerObj->getAppId(), $providerObj->getId(), $argument, true); + $registeredProviders = $this->providerConfig->getProviders(); + if (!isset($registeredProviders[$identifier]) + || $registeredProviders[$identifier]['isInitiated'] + ) { + $this->logger->info('[InitialContentImportJob] Provider has already been initiated, skipping content import', ['provider' => $identifier]); + return; + } + + $providerObj->triggerInitialImport(); + $this->providerConfig->updateProvider($providerObj->getAppId(), $providerObj->getId(), $argument, isInitiated: true); + } finally { + $this->fetchGuard->jobEnded($running); + } } } diff --git a/lib/BackgroundJobs/SchedulerJob.php b/lib/BackgroundJobs/SchedulerJob.php index 47b0ddb6..eb26c145 100644 --- a/lib/BackgroundJobs/SchedulerJob.php +++ b/lib/BackgroundJobs/SchedulerJob.php @@ -11,6 +11,8 @@ namespace OCA\ContextChat\BackgroundJobs; use OCA\ContextChat\Logger; +use OCA\ContextChat\Service\FetchGuard; +use OCA\ContextChat\Service\ProviderConfigService; use OCA\ContextChat\Service\StorageService; use OCP\AppFramework\Services\IAppConfig; use OCP\AppFramework\Utility\ITimeFactory; @@ -26,6 +28,7 @@ public function __construct( private IJobList $jobList, private StorageService $storageService, private IAppConfig $appConfig, + private FetchGuard $fetchGuard, ) { parent::__construct($timeFactory); } @@ -34,17 +37,27 @@ public function __construct( * @throws Exception */ protected function run($argument): void { - $this->appConfig->setAppValueInt('last_indexed_time', 0, lazy: true); - foreach ($this->storageService->getMounts() as $mount) { - $this->logger->debug('Scheduling StorageCrawlJob storage_id=' . $mount['storage_id'] . ' root_id=' . $mount['root_id' ] . 'override_root=' . $mount['overridden_root']); - $this->jobList->add(StorageCrawlJob::class, [ - 'storage_id' => $mount['storage_id'], - 'root_id' => $mount['root_id' ], - 'overridden_root' => $mount['overridden_root'], - 'last_file_id' => 0, - ]); + $filesKey = ProviderConfigService::getDefaultProviderKey(); + $running = $this->fetchGuard->jobStarted($filesKey, $this->getId()); + try { + // a wipe of the files removes this job, a re-index after it adds it again + if ($this->fetchGuard->isPaused($filesKey)) { + return; + } + $this->appConfig->setAppValueInt('last_indexed_time', 0, lazy: true); + foreach ($this->storageService->getMounts() as $mount) { + $this->logger->debug('Scheduling StorageCrawlJob storage_id=' . $mount['storage_id'] . ' root_id=' . $mount['root_id' ] . 'override_root=' . $mount['overridden_root']); + $this->jobList->add(StorageCrawlJob::class, [ + 'storage_id' => $mount['storage_id'], + 'root_id' => $mount['root_id' ], + 'overridden_root' => $mount['overridden_root'], + 'last_file_id' => 0, + ]); + } + + $this->jobList->remove(self::class); + } finally { + $this->fetchGuard->jobEnded($running); } - - $this->jobList->remove(self::class); } } diff --git a/lib/BackgroundJobs/StorageCrawlJob.php b/lib/BackgroundJobs/StorageCrawlJob.php index 4c0bc8ed..25fbc11c 100644 --- a/lib/BackgroundJobs/StorageCrawlJob.php +++ b/lib/BackgroundJobs/StorageCrawlJob.php @@ -13,6 +13,8 @@ use OCA\ContextChat\Db\QueueFile; use OCA\ContextChat\Logger; use OCA\ContextChat\Service\DiagnosticService; +use OCA\ContextChat\Service\FetchGuard; +use OCA\ContextChat\Service\ProviderConfigService; use OCA\ContextChat\Service\QueueService; use OCA\ContextChat\Service\StorageService; use OCP\AppFramework\Services\IAppConfig; @@ -33,6 +35,7 @@ public function __construct( private StorageService $storageService, private DiagnosticService $diagnosticService, private IAppConfig $appConfig, + private FetchGuard $fetchGuard, ) { parent::__construct($timeFactory); } @@ -50,7 +53,13 @@ protected function run($argument): void { // Remove current iteration $this->jobList->remove(self::class, $argument); + $filesKey = ProviderConfigService::getDefaultProviderKey(); + $running = $this->fetchGuard->jobStarted($filesKey, $this->getId()); try { + // wipe of everything/for files is running, skip the crawl + if ($this->fetchGuard->isPaused($filesKey)) { + return; + } $this->diagnosticService->sendJobStart(static::class, $this->getId()); $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); @@ -64,6 +73,7 @@ protected function run($argument): void { $queueFile->setFileId($fileId); $queueFile->setUpdate(false); $this->diagnosticService->sendHeartbeat(static::class, $this->getId()); + $this->fetchGuard->jobAlive($running); try { $dbFile = $this->queue->insertIntoQueue($queueFile); $lastSuccessfulDbId = $dbFile->getId(); @@ -96,6 +106,7 @@ protected function run($argument): void { } } finally { $this->diagnosticService->sendJobEnd(static::class, $this->getId()); + $this->fetchGuard->jobEnded($running); } } diff --git a/lib/Controller/QueueController.php b/lib/Controller/QueueController.php index 09f16615..d2bcb303 100644 --- a/lib/Controller/QueueController.php +++ b/lib/Controller/QueueController.php @@ -119,9 +119,10 @@ public function getDocumentsQueueItems( } try { + $filesKey = ProviderConfigService::getDefaultProviderKey(); $files = []; // a wipe of the files holds back the files only - while (count($files) < $n && !$this->fetchGuard->areFilesPaused()) { + while (count($files) < $n && !$this->fetchGuard->isPaused($filesKey)) { $limit = $n - count($files); $documents = $queueMapper->getFromQueue($limit); if (empty($documents)) { @@ -160,7 +161,7 @@ public function getDocumentsQueueItems( // // Only the files and a full wipe pause. A wipe of a content provider deletes the provider's // items that are not locked, so a fetch can no longer lock them, and waits for the locked ones. - if ($files !== [] && $this->fetchGuard->areFilesPaused()) { + if ($files !== [] && $this->fetchGuard->isPaused($filesKey)) { $queueMapper->unlock(array_keys($files)); $files = []; } diff --git a/lib/Service/FetchGuard.php b/lib/Service/FetchGuard.php index 707c6bf0..5f105d5c 100644 --- a/lib/Service/FetchGuard.php +++ b/lib/Service/FetchGuard.php @@ -22,7 +22,8 @@ * they can only be held back from here. * * A full wipe holds back everything while a wipe of the files provider only the files. A wipe of any - * content provider does not pause, it clears the provider's queue before it waits for the handed out documents instead. + * content provider does not pause, it clears the provider's queue before it waits for the handed out documents + * instead. It only stops the provider's import job. * * A pause expires on its own after a day, so a wipe that dies half way does not stop the indexing forever. * Running the wipe again lifts it right away: it starts a new pause and lifts that one when it ends. @@ -30,9 +31,17 @@ class FetchGuard { private const ALL_KEY = 'queue_fetch_paused_until'; private const FILES_KEY = 'files_fetch_paused_until'; + // a wipe of a content provider, for its import job: ("paused_until_" => until)) + private const PROVIDER_KEY_PREFIX = 'paused_until_'; + // a background job queueing items: ("running__" => until) + private const RUNNING_KEY_PREFIX = 'running_'; // pause indexing while wipe happens, max 1 day so a crashed process does not stop it indefinitely private const PAUSE_DURATION = 24 * 60 * 60; public const DEFAULT_DRAIN_TIMEOUT = 30 * 60; // 30 mins for the currently indexing documents to be done + // bg jobs except initial import refreshes its record while it runs, a killed job blocks wipes for at most this long + public const RUNNING_TTL = 10 * 60; + // an initial import job cannot refresh its record while the app imports, so it has to last for the whole import + public const IMPORT_RUNNING_TTL = 60 * 60; public function __construct( private IDBConnection $db, @@ -44,69 +53,109 @@ public function __construct( } /** - * @param bool $onlyFiles hold back the files only, for a wipe of the files - * @return array{0: string, 1: int} the pause, to be passed to resume() + * Starts a wipe: what is queued for these providers is held back until resume() + * + * @param list|null $providerKeys null for a full wipe + * @return array the pause object, to be passed to resume() */ - public function pause(bool $onlyFiles = false): array { - $key = $onlyFiles ? self::FILES_KEY : self::ALL_KEY; + public function pause(?array $providerKeys = null): array { $until = $this->timeFactory->getTime() + self::PAUSE_DURATION; - $this->appConfig->setAppValueInt($key, $until, lazy: true); - return [$key, $until]; + $keys = $providerKeys === null ? [self::ALL_KEY] : array_map($this->getPauseKey(...), $providerKeys); + $pause = []; + foreach ($keys as $key) { + $this->appConfig->setAppValueInt($key, $until, lazy: true); + $pause[$key] = $until; + } + return $pause; } /** - * Lifts the pause, unless another one of the same kind was started since. + * Lifts the pause, only the ones started by the caller, with a matching until * - * @param array{0: string, 1: int} $pause + * @param array $pause */ public function resume(array $pause): void { - [$key, $until] = $pause; - if ($this->readUntil($key) === $until) { - $this->appConfig->deleteAppValue($key); + foreach ($pause as $key => $until) { + if ($this->readUntil($key) === $until) { + $this->appConfig->deleteAppValue($key); + } } } /** - * Whether a full wipe is running: nothing is handed out, not even actions. + * Whether a wipe holds back: + * - null: everything, nothing is handed out, not even actions. A full wipe is running. + * - the files: the files, they are neither handed out nor crawled. A full wipe or a wipe of the files is running. + * - a content provider: its import job, which must stop. A full wipe or a wipe of the provider is running. + * Its queued items are still handed out, unless it is a full wipe. */ - public function isPaused(): bool { - return $this->readUntil(self::ALL_KEY) > $this->timeFactory->getTime(); + public function isPaused(?string $providerKey = null): bool { + $now = $this->timeFactory->getTime(); + if ($this->readUntil(self::ALL_KEY) > $now) { + return true; + } + return $providerKey !== null && $this->readUntil($this->getPauseKey($providerKey)) > $now; + } + + private function getPauseKey(string $providerKey): string { + // keep the key length within 64 chars + return $providerKey === ProviderConfigService::getDefaultProviderKey() + ? self::FILES_KEY + : self::PROVIDER_KEY_PREFIX . md5($providerKey); } /** - * Whether the files are held back, by a full wipe or a wipe of the files. + * Records a background job queueing items of a provider, until it calls jobEnded(). A wipe of the + * provider waits for it with waitForRunningJobs(). + * + * The job records itself before it checks isPaused(), and a wipe pauses before it looks for running + * jobs, so either the wipe waits for the job or the job sees the wipe and stops. + * + * @return string the record, to be passed to jobAlive() and jobEnded() */ - public function areFilesPaused(): bool { - return $this->isPaused() || $this->readUntil(self::FILES_KEY) > $this->timeFactory->getTime(); + public function jobStarted(string $providerKey, int|string $jobId, int $ttl = self::RUNNING_TTL): string { + $key = self::RUNNING_KEY_PREFIX . $this->getProviderConfigPartKey($providerKey) . '_' . $jobId; + $this->appConfig->setAppValueInt($key, $this->timeFactory->getTime() + $ttl, lazy: true); + return $key; } /** - * Waits for the documents the backend took to be reported back. A document that is still being indexed - * would be written to the vector DB after it was wiped. - * - * @param list|null $providerKeys only wait for the documents of these providers, null for all of them - * @param ?callable(int): void $onWaiting called with the number of documents still being indexed, - * once before waiting and again whenever that number changes - * @throws \RuntimeException when they are not reported back in time - * @throws \OCP\DB\Exception + * Extends the record of a job that is still running, at most once per half of its lifetime. */ - public function waitForHandedOutDocuments(?array $providerKeys = null, ?callable $onWaiting = null): void { - $deadline = $this->timeFactory->getTime() + $this->getDrainTimeout(); - $reported = null; - while (($locked = $this->countHandedOutDocuments($providerKeys)) > 0) { - if ($onWaiting !== null && $locked !== $reported) { - $onWaiting($locked); - $reported = $locked; - } - if ($this->timeFactory->getTime() >= $deadline) { - throw new \RuntimeException( - $locked . ' documents are still being indexed by context_chat_backend after waiting ' - . $this->getDrainTimeout() . ' seconds. Try again later. If the backend was stopped while' - . ' indexing them, they would only be released after a day, use --ignore-handed-out then.' - ); + public function jobAlive(string $record): void { + $until = $this->appConfig->getAppValueInt($record, 0, lazy: true); + $now = $this->timeFactory->getTime(); + if ($until - $now < self::RUNNING_TTL / 2) { + $this->appConfig->setAppValueInt($record, $now + self::RUNNING_TTL, lazy: true); + } + } + + public function jobEnded(string $record): void { + $this->appConfig->deleteAppValue($record); + } + + private function getProviderConfigPartKey(string $providerKey): string { + // a provider key can be longer than an app config key may be + return $providerKey === ProviderConfigService::getDefaultProviderKey() ? 'files' : md5($providerKey); + } + + /** + * @param list $prefixes + */ + private function countRunningJobs(array $prefixes): int { + $now = $this->timeFactory->getTime(); + $count = 0; + foreach ($prefixes as $prefix) { + foreach ($this->readValues($prefix, isPrefix: true) as $key => $until) { + if ((int)$until > $now) { + $count++; + } else { + // left by a job that was killed + $this->appConfig->deleteAppValue($key); + } } - sleep(1); } + return $count; } /** @@ -126,24 +175,102 @@ private function countHandedOutDocuments(?array $providerKeys): int { return $count; } + /** + * Waits for the background jobs that queue items of these providers to end. A job that is still running + * would queue items, or mark its provider imported, after the wipe cleared them. + * + * @param list|null $providerKeys null for the jobs of all the providers + * @param ?callable(int): void $onWaiting called with the number of running jobs, once before waiting and again + * whenever that number changes + * @throws \RuntimeException when they do not end in time + */ + public function waitForRunningJobs(?array $providerKeys = null, ?callable $onWaiting = null): void { + $prefixes = $providerKeys === null + ? [self::RUNNING_KEY_PREFIX] + : array_map(fn (string $key): string => self::RUNNING_KEY_PREFIX . $this->getProviderConfigPartKey($key) . '_', $providerKeys); + $this->waitUntilNone( + fn (): int => $this->countRunningJobs($prefixes), + $onWaiting, + 'background jobs are still queueing documents', + 'If they were killed, their records expire within an hour, or use --ignore-handed-out.', + ); + } + + /** + * Waits for the documents the backend took to be reported back. A document that is still being indexed + * would be written to the vector DB after it was wiped. + * + * @param list|null $providerKeys only wait for the documents of these providers, null for all of them + * @param ?callable(int): void $onWaiting called with the number of documents still being indexed, + * once before waiting and again whenever that number changes + * @throws \RuntimeException when they are not reported back in time + * @throws \OCP\DB\Exception + */ + public function waitForHandedOutDocuments(?array $providerKeys = null, ?callable $onWaiting = null): void { + $this->waitUntilNone( + fn (): int => $this->countHandedOutDocuments($providerKeys), + $onWaiting, + 'documents are still being indexed by context_chat_backend', + 'If the backend was stopped while indexing them, they would only be released after a day, use --ignore-handed-out then.', + ); + } + + /** + * @param callable(): int $count what is left to wait for + * @param ?callable(int): void $onWaiting called with what is left, once before waiting and again whenever it changes + * @param string $progressInfoText what is being awaited, for the error message + * @param string $hint what to do if it never ends, for the error message + * @throws \RuntimeException when it is not done in time, receives the above error message + */ + private function waitUntilNone(callable $count, ?callable $onWaiting, string $progressInfoText, string $hint): void { + $timeout = $this->getDrainTimeout(); + $deadline = $this->timeFactory->getTime() + $timeout; + $reported = null; + while (($left = $count()) > 0) { + if ($onWaiting !== null && $left !== $reported) { + $onWaiting($left); + $reported = $left; + } + if ($this->timeFactory->getTime() >= $deadline) { + throw new \RuntimeException( + $left . ' ' . $progressInfoText . ' after waiting ' . $timeout . ' seconds. Try again later. ' . $hint + ); + } + sleep(1); + } + } + private function getDrainTimeout(): int { return $this->appConfig->getAppValueInt('wipe_drain_timeout', self::DEFAULT_DRAIN_TIMEOUT, lazy: true); } + private function readUntil(string $key): int { + return (int)($this->readValues($key)[$key] ?? 0); + } + /** * Read from the database instead of IAppConfig. IAppConfig values are cached by each web server * for a few seconds, and all of them must stop handing out items before the wipe starts. * This ensures that we get the most up-to-date value for the pause state. + * + * @return array configkey: value */ - private function readUntil(string $key): int { + private function readValues(string $key, bool $isPrefix = false): array { $qb = $this->db->getQueryBuilder(); - $result = $qb->select('configvalue') + $qb->select('configkey', 'configvalue') ->from('appconfig') - ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID))) - ->andWhere($qb->expr()->eq('configkey', $qb->createNamedParameter($key))) - ->executeQuery(); - $value = $result->fetchOne(); + ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID))); + if ($isPrefix) { + $qb->andWhere($qb->expr()->like('configkey', $qb->createNamedParameter($this->db->escapeLikeParameter($key) . '%'))); + } else { + $qb->andWhere($qb->expr()->eq('configkey', $qb->createNamedParameter($key))); + } + $result = $qb->executeQuery(); + $values = []; + while (($row = $result->fetch()) !== false) { + $values[(string)$row['configkey']] = (string)$row['configvalue']; + } $result->closeCursor(); - return $value === false ? 0 : (int)$value; + return $values; } } diff --git a/lib/Service/ReindexService.php b/lib/Service/ReindexService.php index c5bb2730..c8cb5e4a 100644 --- a/lib/Service/ReindexService.php +++ b/lib/Service/ReindexService.php @@ -157,23 +157,30 @@ public function wipe(?array $providerKeys = null, bool $ignoreHandedOut = false, $providerKeys = $providerKeys === null ? null : array_values(array_unique($providerKeys)); $contentProviderKeys = $providerKeys === null ? [] : array_values(array_diff($providerKeys, [$filesKey])); - $pause = null; if ($providerKeys === null) { $onStep('Pausing the hand-out of queued documents and actions to context_chat_backend'); - $pause = $this->fetchGuard->pause(); } elseif (in_array($filesKey, $providerKeys, true)) { $onStep('Pausing the hand-out of queued files to context_chat_backend'); - $pause = $this->fetchGuard->pause(onlyFiles: true); } + // also stops the crawl and import jobs of the providers + $pause = $this->fetchGuard->pause($providerKeys); try { + if ($ignoreHandedOut) { + $onStep('Not waiting for the background jobs and the documents context_chat_backend may still be indexing (--ignore-handed-out)'); + } else { + // a running crawl or import would queue items after the queues are cleared + $onStep('Checking for background jobs still queueing documents'); + $this->fetchGuard->waitForRunningJobs($providerKeys, static function (int $running) use ($onStep): void { + $onStep('Waiting for ' . $running . ' background jobs to finish queueing documents'); + }); + } + foreach ($contentProviderKeys as $providerKey) { $onStep('Removing the import job and clearing the indexing queue of ' . $providerKey); $this->clearContentProviderState($providerKey); } - if ($ignoreHandedOut) { - $onStep('Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)'); - } else { + if (!$ignoreHandedOut) { $onStep('Checking for documents context_chat_backend is still indexing'); $this->fetchGuard->waitForHandedOutDocuments($providerKeys, static function (int $locked) use ($onStep): void { $onStep('Waiting for context_chat_backend to finish indexing ' . $locked . ' documents'); @@ -200,12 +207,12 @@ public function wipe(?array $providerKeys = null, bool $ignoreHandedOut = false, $this->langRopeService->deleteProvider($providerKey); } } finally { - if ($pause !== null) { - $onStep($providerKeys === null - ? 'Resuming the hand-out of queued documents and actions' - : 'Resuming the hand-out of queued files'); - $this->fetchGuard->resume($pause); + if ($providerKeys === null) { + $onStep('Resuming the hand-out of queued documents and actions'); + } elseif (in_array($filesKey, $providerKeys, true)) { + $onStep('Resuming the hand-out of queued files'); } + $this->fetchGuard->resume($pause); } } diff --git a/tests/integration/QueueControllerPauseTest.php b/tests/integration/QueueControllerPauseTest.php index b173852a..5c7d7b9d 100644 --- a/tests/integration/QueueControllerPauseTest.php +++ b/tests/integration/QueueControllerPauseTest.php @@ -107,7 +107,7 @@ private function filesInQueue(): int { } public function testFilesWipeHoldsBackOnlyTheFiles(): void { - $this->pauses[] = $this->fetchGuard->pause(onlyFiles: true); + $this->pauses[] = $this->fetchGuard->pause(['files__default']); $response = $this->call('getDocumentsQueueItems'); @@ -116,6 +116,17 @@ public function testFilesWipeHoldsBackOnlyTheFiles(): void { $this->assertCount(1, (array)$this->call('getActionsQueueItems')['actions']); } + public function testContentProviderWipeHoldsBackNothing(): void { + // it only stops the provider's import, its wipe deletes the queued items it does not wait for + $this->pauses[] = $this->fetchGuard->pause(['someapp__prov']); + + $response = $this->call('getDocumentsQueueItems'); + + $this->assertCount(1, (array)$response['content_providers']); + $this->assertSame(0, $this->filesInQueue()); + $this->assertCount(1, (array)$this->call('getActionsQueueItems')['actions']); + } + public function testFullWipeHoldsBackEverything(): void { $this->pauses[] = $this->fetchGuard->pause(); @@ -191,7 +202,7 @@ public function testFilesLockedAsAFilesWipeStartsArePutBack(): void { $filesQueue->method('getFromQueue')->willReturnCallback(fn (int $limit) => $real->getFromQueue($limit)); $filesQueue->method('lock')->willReturnCallback(function (int $id) use ($real): bool { $locked = $real->lock($id); - $this->pauses[] = $this->fetchGuard->pause(onlyFiles: true); + $this->pauses[] = $this->fetchGuard->pause(['files__default']); return $locked; }); $filesQueue->method('unlock')->willReturnCallback(fn (array $ids) => $real->unlock($ids)); diff --git a/tests/integration/ReindexServiceTest.php b/tests/integration/ReindexServiceTest.php index 5d7254d1..581e4cec 100644 --- a/tests/integration/ReindexServiceTest.php +++ b/tests/integration/ReindexServiceTest.php @@ -409,6 +409,21 @@ public function testResumeDoesNotLiftANewerPause(): void { $this->assertFalse($guard->isPaused()); } + public function testResumeDoesNotLiftANewerPauseOfTheSameProvider(): void { + $now = self::NOW; + $guard = $this->newGuard($this->clock($now)); + + $first = $guard->pause(['someapp__prov', 'files__default']); + $now += 10; + $second = $guard->pause(['someapp__prov']); + $guard->resume($first); + + $this->assertTrue($guard->isPaused('someapp__prov'), 'the second wipe is still running'); + $this->assertFalse($guard->isPaused('files__default')); + $guard->resume($second); + $this->assertFalse($guard->isPaused('someapp__prov')); + } + public function testPauseExpiresOnItsOwn(): void { $now = self::NOW; $guard = $this->newGuard($this->clock($now)); @@ -425,35 +440,35 @@ public function testPauseOutlastsAScopedWipeOfManySlowProviders(): void { $now = self::NOW; $guard = $this->newGuard($this->clock($now)); - $pause = $guard->pause(onlyFiles: true); + $pause = $guard->pause(['files__default']); // the drain timeout, then 20 deleteProvider requests that each run into the request timeout $now += FetchGuard::DEFAULT_DRAIN_TIMEOUT + 20 * Application::CC_DEFAULT_REQUEST_TIMEOUT; - $this->assertTrue($guard->areFilesPaused(), 'indexing must not restart while providers are still being wiped'); + $this->assertTrue($guard->isPaused('files__default'), 'indexing must not restart while providers are still being wiped'); $guard->resume($pause); } public function testFullAndFilesPausesAreLiftedSeparately(): void { - $files = $this->fetchGuard->pause(onlyFiles: true); + $files = $this->fetchGuard->pause(['files__default']); $all = $this->fetchGuard->pause(); $this->fetchGuard->resume($all); - $this->assertTrue($this->fetchGuard->areFilesPaused(), 'the files wipe is still running'); + $this->assertTrue($this->fetchGuard->isPaused('files__default'), 'the files wipe is still running'); $this->fetchGuard->resume($files); - $this->assertFalse($this->fetchGuard->areFilesPaused()); + $this->assertFalse($this->fetchGuard->isPaused('files__default')); } public function testFilesWipePausesOnlyTheFiles(): void { $this->seedIndexedState(); $seen = null; $this->langRope->method('deleteProvider')->willReturnCallback(function () use (&$seen) { - $seen = [$this->fetchGuard->areFilesPaused(), $this->fetchGuard->isPaused()]; + $seen = [$this->fetchGuard->isPaused('files__default'), $this->fetchGuard->isPaused()]; }); $this->service->wipe(['files__default']); $this->assertSame([true, false], $seen); - $this->assertFalse($this->fetchGuard->areFilesPaused()); + $this->assertFalse($this->fetchGuard->isPaused('files__default')); } public function testContentProviderWipeOnlyWaitsForItsOwnDocuments(): void { @@ -559,6 +574,7 @@ public function testWipeReportsEachStepBeforeDoingIt(): void { $this->assertSame([ 'Pausing the hand-out of queued documents and actions to context_chat_backend', + 'Checking for background jobs still queueing documents', 'Checking for documents context_chat_backend is still indexing', 'Removing the scheduled crawl and content import jobs', 'Clearing the indexing, actions and file system events queues', @@ -581,8 +597,8 @@ public function testWipeOfOneProviderReportsItsSteps(): void { }); $this->assertSame([ + 'Not waiting for the background jobs and the documents context_chat_backend may still be indexing (--ignore-handed-out)', 'Removing the import job and clearing the indexing queue of someapp__prov', - 'Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)', 'Deleting the documents of someapp__prov from the vector DB', 'backend:deleteProvider someapp__prov', ], $events, 'a content provider wipe does not pause'); @@ -601,7 +617,7 @@ public function testWipeOfTheFilesReportsItsSteps(): void { $this->assertSame([ 'Pausing the hand-out of queued files to context_chat_backend', - 'Not waiting for the documents context_chat_backend may still be indexing (--ignore-handed-out)', + 'Not waiting for the background jobs and the documents context_chat_backend may still be indexing (--ignore-handed-out)', 'Removing the crawl jobs, clearing the files indexing queue and the file system events, resetting the indexing statistics', 'Deleting the documents of files__default from the vector DB', 'backend:deleteProvider files__default', diff --git a/tests/integration/WipeBackgroundJobsTest.php b/tests/integration/WipeBackgroundJobsTest.php new file mode 100644 index 00000000..ae032856 --- /dev/null +++ b/tests/integration/WipeBackgroundJobsTest.php @@ -0,0 +1,368 @@ + 1, 'root_id' => 1, 'overridden_root' => 1]; + private const PROVIDER_CLASS = 'OCA\\SomeApp\\ContentProvider'; + + private IJobList $jobList; + private IAppConfig $appConfig; + private FetchGuard $fetchGuard; + private QueueMapper $queueMapper; + private ProviderConfigService $providerConfig; + private string $savedProviders; + + protected function setUp(): void { + parent::setUp(); + $this->jobList = Server::get(IJobList::class); + $container = (new Application())->getContainer(); + $this->appConfig = $container->get(IAppConfig::class); + $this->providerConfig = $container->get(ProviderConfigService::class); + $this->fetchGuard = Server::get(FetchGuard::class); + $this->queueMapper = Server::get(QueueMapper::class); + $this->savedProviders = $this->appConfig->getAppValueString('providers', '', lazy: true); + $this->clearState(); + } + + protected function tearDown(): void { + $this->clearState(); + $this->appConfig->setAppValueString('providers', $this->savedProviders, lazy: true); + parent::tearDown(); + } + + private function clearState(): void { + $this->queueMapper->clearQueue(); + foreach ([SchedulerJob::class, StorageCrawlJob::class, InitialContentImportJob::class] as $jobClass) { + $this->jobList->remove($jobClass); + } + $this->appConfig->deleteAppValue('last_enqueued_db_id'); + $qb = Server::get(IDBConnection::class)->getQueryBuilder(); + $qb->delete('appconfig') + ->where($qb->expr()->eq('appid', $qb->createNamedParameter(Application::APP_ID))) + ->andWhere($qb->expr()->orX( + $qb->expr()->like('configkey', $qb->createNamedParameter('running\_%')), + $qb->expr()->like('configkey', $qb->createNamedParameter('wipe\_%')), + $qb->expr()->in('configkey', $qb->createNamedParameter(['queue_fetch_paused_until', 'files_fetch_paused_until'], IQueryBuilder::PARAM_STR_ARRAY)), + )) + ->executeStatement(); + } + + /** @param ?callable(): void $meanwhile called between the two files of the batch */ + private function runCrawl(?callable $meanwhile = null): void { + $storage = $this->createMock(StorageService::class); + $storage->method('getFilesInMount')->willReturnCallback(static function () use ($meanwhile): \Generator { + yield 101; + if ($meanwhile !== null) { + $meanwhile(); + } + yield 102; + }); + $job = Server::get(StorageCrawlJob::class); + $job->setId('7'); + $this->replaceDependency($job, 'storageService', $storage); + $this->invokeRun($job, self::MOUNT + ['last_file_id' => 0]); + } + + private function runScheduler(): void { + $storage = $this->createMock(StorageService::class); + $storage->method('getMounts')->willReturnCallback(static function (): \Generator { + yield self::MOUNT; + }); + $job = Server::get(SchedulerJob::class); + $this->replaceDependency($job, 'storageService', $storage); + $this->invokeRun($job, null); + } + + /** @param ?callable(): void $meanwhile called during the provider's import */ + private function runImport(?callable $meanwhile = null): void { + $provider = $this->createMock(IContentProvider::class); + $provider->method('getAppId')->willReturn('someapp'); + $provider->method('getId')->willReturn('prov'); + $provider->method('triggerInitialImport')->willReturnCallback(static function () use ($meanwhile): void { + if ($meanwhile !== null) { + $meanwhile(); + } + }); + \OC::$server->registerService(self::PROVIDER_CLASS, static fn () => $provider); + $appManager = $this->createMock(IAppManager::class); + $appManager->method('isEnabledForUser')->willReturn(true); + + $job = Server::get(InitialContentImportJob::class); + $this->replaceDependency($job, 'appManager', $appManager); + $this->invokeRun($job, self::PROVIDER_CLASS); + } + + private function replaceDependency(object $object, string $property, object $value): void { + (new \ReflectionProperty($object, $property))->setValue($object, $value); + } + + private function invokeRun(object $job, mixed $argument): void { + (new \ReflectionMethod($job, 'run'))->invoke($job, $argument); + } + + private function countCrawlJobs(): int { + return count(iterator_to_array($this->jobList->getJobsIterator(StorageCrawlJob::class, null, 0), false)); + } + + private function isImported(): bool { + return $this->providerConfig->getProviders()['someapp__prov']['isInitiated']; + } + + /** @return ITimeFactory a clock that starts at $now and moves when $now is changed */ + private function clock(int &$now): ITimeFactory { + $time = $this->createMock(ITimeFactory::class); + $time->method('getTime')->willReturnCallback(static function () use (&$now): int { + return $now; + }); + return $time; + } + + private function guardWith(ITimeFactory $time): FetchGuard { + return new FetchGuard( + Server::get(IDBConnection::class), $this->appConfig, $time, + $this->queueMapper, Server::get(QueueContentItemMapper::class), + ); + } + + /** + * @param list|null $providerKeys + * @return int the running jobs a wipe of these providers would wait for + */ + private function countRunningJobs(?array $providerKeys, ?FetchGuard $guard = null): int { + $counted = null; + $this->appConfig->setAppValueInt('wipe_drain_timeout', 0, lazy: true); + try { + ($guard ?? $this->fetchGuard)->waitForRunningJobs($providerKeys, static function (int $running) use (&$counted): void { + $counted = $running; + }); + } catch (\RuntimeException) { + } finally { + $this->appConfig->deleteAppValue('wipe_drain_timeout'); + } + return $counted ?? 0; + } + + public function testCrawlQueuesAndContinuesWithoutAWipe(): void { + $this->runCrawl(); + + $this->assertSame(2, $this->queueMapper->count()); + $this->assertSame(1, $this->countCrawlJobs(), 'the next batch is scheduled'); + $this->assertSame(0, $this->countRunningJobs(null), 'the crawl is not recorded as running once it ended'); + } + + public function testCrawlStopsDuringAFilesWipe(): void { + $pause = $this->fetchGuard->pause(['files__default']); + try { + $this->runCrawl(); + } finally { + $this->fetchGuard->resume($pause); + } + + $this->assertSame(0, $this->queueMapper->count()); + $this->assertSame(0, $this->countCrawlJobs()); + } + + public function testCrawlGoesOnDuringAWipeOfAContentProvider(): void { + $wipe = $this->fetchGuard->pause(['someapp__prov']); + try { + $this->runCrawl(); + } finally { + $this->fetchGuard->resume($wipe); + } + + $this->assertSame(2, $this->queueMapper->count()); + } + + public function testCrawlIsRecordedAsRunningWhileItQueues(): void { + $running = null; + $this->runCrawl(function () use (&$running) { + $running = [ + $this->countRunningJobs([ProviderConfigService::getDefaultProviderKey()]), + $this->countRunningJobs(['someapp__prov']), + ]; + }); + + $this->assertSame([1, 0], $running, 'a wipe of the files waits for the crawl, a wipe of another provider does not'); + $this->assertSame(0, $this->countRunningJobs(null)); + } + + public function testCrawlRecordIsRemovedWhenTheCrawlFails(): void { + try { + $this->runCrawl(static function (): void { + throw new \RuntimeException('storage gone'); + }); + $this->fail('the failure must not be swallowed'); + } catch (\RuntimeException $e) { + $this->assertSame('storage gone', $e->getMessage()); + } + + $this->assertSame(0, $this->countRunningJobs(null)); + } + + public function testSchedulerDoesNothingDuringAFullWipe(): void { + $pause = $this->fetchGuard->pause(); + try { + $this->runScheduler(); + } finally { + $this->fetchGuard->resume($pause); + } + + $this->assertSame(0, $this->countCrawlJobs()); + } + + public function testSchedulerAddsTheCrawlJobsWithoutAWipe(): void { + $this->runScheduler(); + + $this->assertSame(1, $this->countCrawlJobs()); + $this->assertSame(0, $this->countRunningJobs(null)); + } + + public function testImportMarksTheProviderImportedWithoutAWipe(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, false); + + $this->runImport(); + + $this->assertTrue($this->isImported()); + $this->assertSame(0, $this->countRunningJobs(null)); + } + + public function testImportStopsDuringAWipeOfItsProvider(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, false); + $imported = false; + $wipe = $this->fetchGuard->pause(['someapp__prov']); + try { + $this->runImport(static function () use (&$imported): void { + $imported = true; + }); + } finally { + $this->fetchGuard->resume($wipe); + } + + $this->assertFalse($imported, 'nothing may be submitted while the provider is wiped'); + $this->assertFalse($this->isImported()); + } + + public function testImportStopsDuringAFullWipe(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, false); + $pause = $this->fetchGuard->pause(); + try { + $this->runImport(); + } finally { + $this->fetchGuard->resume($pause); + } + + $this->assertFalse($this->isImported()); + } + + public function testImportIsRecordedAsRunningForItsProviderOnly(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, false); + $running = null; + $this->runImport(function () use (&$running) { + $running = [ + $this->countRunningJobs(['someapp__prov']), + $this->countRunningJobs([ProviderConfigService::getDefaultProviderKey()]), + $this->countRunningJobs(null), + ]; + }); + + $this->assertSame([1, 0, 1], $running); + } + + public function testWipeWaitsForARunningImportBeforeClearingTheQueue(): void { + $this->providerConfig->updateProvider('someapp', 'prov', self::PROVIDER_CLASS, true); + $record = $this->fetchGuard->jobStarted('someapp__prov', '42', FetchGuard::IMPORT_RUNNING_TTL); + $this->appConfig->setAppValueInt('wipe_drain_timeout', 0, lazy: true); + $langRope = $this->createMock(LangRopeService::class); + $langRope->expects($this->never())->method('deleteProvider'); + $service = Server::get(ReindexService::class); + $this->replaceDependency($service, 'langRopeService', $langRope); + + try { + $service->wipe(['someapp__prov']); + $this->fail('the wipe must wait for the running import'); + } catch (\RuntimeException $e) { + $this->assertStringContainsString('1 background jobs are still queueing documents', $e->getMessage()); + } finally { + $this->appConfig->deleteAppValue('wipe_drain_timeout'); + $this->fetchGuard->jobEnded($record); + } + + $this->assertTrue($this->isImported(), 'nothing may be reset when the wipe gives up'); + $this->assertFalse($this->fetchGuard->isPaused('someapp__prov'), 'the import may run again'); + } + + public function testWipeWithIgnoreHandedOutDoesNotWaitForRunningJobs(): void { + $record = $this->fetchGuard->jobStarted(ProviderConfigService::getDefaultProviderKey(), '42'); + $langRope = $this->createMock(LangRopeService::class); + $langRope->expects($this->once())->method('deleteProvider'); + $service = Server::get(ReindexService::class); + $this->replaceDependency($service, 'langRopeService', $langRope); + + try { + $service->wipe([ProviderConfigService::getDefaultProviderKey()], ignoreHandedOut: true); + } finally { + $this->fetchGuard->jobEnded($record); + } + } + + public function testAKilledJobBlocksWipesUntilItsRecordExpires(): void { + $now = 1_800_000_000; + $guard = $this->guardWith($this->clock($now)); + $guard->jobStarted(ProviderConfigService::getDefaultProviderKey(), '42'); + + $now += FetchGuard::RUNNING_TTL - 1; + $this->assertSame(1, $this->countRunningJobs(null, $guard)); + + $now += 1; + $this->assertSame(0, $this->countRunningJobs(null, $guard), 'a job that never ended must not block wipes forever'); + } + + public function testARunningCrawlKeepsItsRecordAlive(): void { + $now = 1_800_000_000; + $guard = $this->guardWith($this->clock($now)); + $record = $guard->jobStarted(ProviderConfigService::getDefaultProviderKey(), '42'); + + for ($i = 0; $i < 4; $i++) { + $now += FetchGuard::RUNNING_TTL / 2; + $guard->jobAlive($record); + } + + $this->assertSame(1, $this->countRunningJobs(null, $guard), 'a crawl still working must not be taken as killed'); + $guard->jobEnded($record); + } +} From e05b24072d22b63da3e7619ddb9c6cba591bfc2c Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Thu, 5 Nov 2026 19:20:11 +0530 Subject: [PATCH 10/12] fix: store wiped as a special state different from a fresh install Signed-off-by: kyteinsky --- lib/BackgroundJobs/SchedulerJob.php | 2 +- lib/Service/ReindexService.php | 4 ++ lib/Service/StatisticsService.php | 9 +-- tests/integration/ReindexServiceTest.php | 4 +- tests/integration/WipeBackgroundJobsTest.php | 64 +++++++++++++++++++- 5 files changed, 74 insertions(+), 9 deletions(-) diff --git a/lib/BackgroundJobs/SchedulerJob.php b/lib/BackgroundJobs/SchedulerJob.php index eb26c145..3abb335f 100644 --- a/lib/BackgroundJobs/SchedulerJob.php +++ b/lib/BackgroundJobs/SchedulerJob.php @@ -44,7 +44,6 @@ protected function run($argument): void { if ($this->fetchGuard->isPaused($filesKey)) { return; } - $this->appConfig->setAppValueInt('last_indexed_time', 0, lazy: true); foreach ($this->storageService->getMounts() as $mount) { $this->logger->debug('Scheduling StorageCrawlJob storage_id=' . $mount['storage_id'] . ' root_id=' . $mount['root_id' ] . 'override_root=' . $mount['overridden_root']); $this->jobList->add(StorageCrawlJob::class, [ @@ -56,6 +55,7 @@ protected function run($argument): void { } $this->jobList->remove(self::class); + $this->appConfig->setAppValueInt('last_indexed_time', 0, lazy: true); } finally { $this->fetchGuard->jobEnded($running); } diff --git a/lib/Service/ReindexService.php b/lib/Service/ReindexService.php index c8cb5e4a..7c2cbf0f 100644 --- a/lib/Service/ReindexService.php +++ b/lib/Service/ReindexService.php @@ -290,6 +290,10 @@ private function resetStats(): void { foreach (self::STATS_CONFIG_KEYS as $key) { $this->appConfig->deleteAppValue($key); } + // Indicate wipe as a different value, `last_indexed_time` set to 0 and queues empty triggers the initial + // indexing in QueueController to be set as completed. + // A re-index sets this back to 0, in SchedulerJob specifically. + $this->appConfig->setAppValueInt('last_indexed_time', -1, lazy: true); // the time taken for the next complete index is measured from here $this->appConfig->setAppValueInt('installed_time', $this->timeFactory->getTime(), lazy: true); } diff --git a/lib/Service/StatisticsService.php b/lib/Service/StatisticsService.php index c3086b52..cce67161 100644 --- a/lib/Service/StatisticsService.php +++ b/lib/Service/StatisticsService.php @@ -36,11 +36,12 @@ public function getStatistics(): array { $stats = []; $stats['installed_at'] = $this->appConfig->getAppValueInt('installed_time', 0, lazy: true); - if ($this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true) === 0) { - $stats['initial_indexing_complete'] = false; - } else { + // 0 while the initial indexing runs, -1 after a wipe of the files until re-index starts again + if (($lastIndexedTime = $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true)) > 0) { $stats['initial_indexing_complete'] = true; - $stats['intial_indexing_completed_at'] = $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true); + $stats['intial_indexing_completed_at'] = $lastIndexedTime; + } else { + $stats['initial_indexing_complete'] = false; } $stats['eligible_files_count'] = $this->getEligibleFilesCount(); diff --git a/tests/integration/ReindexServiceTest.php b/tests/integration/ReindexServiceTest.php index 581e4cec..b575890f 100644 --- a/tests/integration/ReindexServiceTest.php +++ b/tests/integration/ReindexServiceTest.php @@ -210,7 +210,7 @@ public function testWipeClearsAllLocalStateAndTheBackend(): void { $this->assertFalse($this->jobList->has(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0])); $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); $this->assertFalse($this->jobList->has(InitialContentImportJob::class, self::OTHER_PROVIDER_CLASS)); - $this->assertSame(0, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true)); + $this->assertSame(-1, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true), 'not complete, and not 0 until the crawl is scheduled'); $this->assertSame(-1, $this->appConfig->getAppValueInt('last_enqueued_db_id', -1, lazy: true)); $this->assertSame(0, $this->appConfig->getAppValueInt('eligible_files_count_time', 0, lazy: true)); $this->assertSame(self::NOW, $this->appConfig->getAppValueInt('installed_time', 0, lazy: true)); @@ -272,7 +272,7 @@ public function testWipeFilesOnlyLeavesContentProvidersAndActions(): void { $this->assertSame(1, $this->rowCount('context_chat_action_queue'), 'actions of other providers and users must be kept'); $this->assertFalse($this->jobList->has(StorageCrawlJob::class, ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1, 'last_file_id' => 0])); $this->assertTrue($this->jobList->has(InitialContentImportJob::class, self::PROVIDER_CLASS)); - $this->assertSame(0, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true)); + $this->assertSame(-1, $this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true)); $this->assertTrue($this->providerConfig->getProviders()['someapp__prov']['isInitiated']); } diff --git a/tests/integration/WipeBackgroundJobsTest.php b/tests/integration/WipeBackgroundJobsTest.php index ae032856..d03d7aaf 100644 --- a/tests/integration/WipeBackgroundJobsTest.php +++ b/tests/integration/WipeBackgroundJobsTest.php @@ -13,12 +13,14 @@ use OCA\ContextChat\BackgroundJobs\InitialContentImportJob; use OCA\ContextChat\BackgroundJobs\SchedulerJob; use OCA\ContextChat\BackgroundJobs\StorageCrawlJob; +use OCA\ContextChat\Controller\QueueController; use OCA\ContextChat\Db\QueueContentItemMapper; use OCA\ContextChat\Db\QueueMapper; use OCA\ContextChat\Service\FetchGuard; use OCA\ContextChat\Service\LangRopeService; use OCA\ContextChat\Service\ProviderConfigService; use OCA\ContextChat\Service\ReindexService; +use OCA\ContextChat\Service\StatisticsService; use OCA\ContextChat\Service\StorageService; use OCP\App\IAppManager; use OCP\AppFramework\Services\IAppConfig; @@ -39,6 +41,8 @@ class WipeBackgroundJobsTest extends TestCase { private const MOUNT = ['storage_id' => 1, 'root_id' => 1, 'overridden_root' => 1]; private const PROVIDER_CLASS = 'OCA\\SomeApp\\ContentProvider'; + // the indexing stats a files wipe resets, put back after each test + private const STATS_KEYS = ['last_indexed_time', 'installed_time', 'eligible_files_count', 'eligible_files_count_time']; private IJobList $jobList; private IAppConfig $appConfig; @@ -46,6 +50,8 @@ class WipeBackgroundJobsTest extends TestCase { private QueueMapper $queueMapper; private ProviderConfigService $providerConfig; private string $savedProviders; + /** @var array */ + private array $savedStats = []; protected function setUp(): void { parent::setUp(); @@ -56,12 +62,18 @@ protected function setUp(): void { $this->fetchGuard = Server::get(FetchGuard::class); $this->queueMapper = Server::get(QueueMapper::class); $this->savedProviders = $this->appConfig->getAppValueString('providers', '', lazy: true); + foreach (self::STATS_KEYS as $key) { + $this->savedStats[$key] = $this->appConfig->hasAppKey($key, lazy: true) ? $this->appConfig->getAppValueInt($key, 0, lazy: true) : null; + } $this->clearState(); } protected function tearDown(): void { $this->clearState(); $this->appConfig->setAppValueString('providers', $this->savedProviders, lazy: true); + foreach ($this->savedStats as $key => $value) { + $value === null ? $this->appConfig->deleteAppValue($key) : $this->appConfig->setAppValueInt($key, $value, lazy: true); + } parent::tearDown(); } @@ -98,9 +110,13 @@ private function runCrawl(?callable $meanwhile = null): void { $this->invokeRun($job, self::MOUNT + ['last_file_id' => 0]); } - private function runScheduler(): void { + /** @param ?callable(): void $meanwhile called once the job runs, before it adds the crawl jobs */ + private function runScheduler(?callable $meanwhile = null): void { $storage = $this->createMock(StorageService::class); - $storage->method('getMounts')->willReturnCallback(static function (): \Generator { + $storage->method('getMounts')->willReturnCallback(static function () use ($meanwhile): \Generator { + if ($meanwhile !== null) { + $meanwhile(); + } yield self::MOUNT; }); $job = Server::get(SchedulerJob::class); @@ -365,4 +381,48 @@ public function testARunningCrawlKeepsItsRecordAlive(): void { $this->assertSame(1, $this->countRunningJobs(null, $guard), 'a crawl still working must not be taken as killed'); $guard->jobEnded($record); } + + /** + * The backend reports a batch back, which is when the initial indexing is checked for completion. + */ + private function backendReportsABatch(): void { + $controller = Server::get(QueueController::class); + $method = new \ReflectionMethod($controller, 'deleteDocumentsQueueItems'); + $args = []; + foreach ($method->getParameters() as $parameter) { + $type = $parameter->getType(); + $args[] = ($type instanceof \ReflectionNamedType && !$type->isBuiltin()) ? Server::get($type->getName()) : []; + } + $method->invokeArgs($controller, $args); + } + + private function isInitialIndexingComplete(): bool { + $stats = Server::get(StatisticsService::class); + $this->replaceDependency($stats, 'langRopeService', $this->createMock(LangRopeService::class)); + $this->replaceDependency($stats, 'storageService', $this->createMock(StorageService::class)); + return $stats->getStatistics()['initial_indexing_complete']; + } + + public function testIndexingIsNotCompleteAfterAFilesWipeUntilTheFilesAreCrawledAgain(): void { + $langRope = $this->createMock(LangRopeService::class); + $service = Server::get(ReindexService::class); + $this->replaceDependency($service, 'langRopeService', $langRope); + + $service->wipe([ProviderConfigService::getDefaultProviderKey()]); + $this->backendReportsABatch(); + $this->assertFalse($this->isInitialIndexingComplete(), 'nothing was crawled yet'); + + // context_chat:reindex, then the scheduler adds the crawl jobs and the crawl queues the files + $service->scheduleReindex([ProviderConfigService::getDefaultProviderKey()]); + $this->runScheduler(function (): void { + $this->backendReportsABatch(); + $this->assertFalse($this->isInitialIndexingComplete(), 'the crawl jobs are not added yet'); + }); + $this->backendReportsABatch(); + $this->assertFalse($this->isInitialIndexingComplete(), 'the crawl jobs have not run yet'); + + $this->jobList->remove(StorageCrawlJob::class); + $this->backendReportsABatch(); + $this->assertTrue($this->isInitialIndexingComplete(), 'the crawl is done and the queue is empty'); + } } From 42abc60bd9ba73b0c110eec7e2bf075388f00ac1 Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Thu, 5 Nov 2026 19:22:17 +0530 Subject: [PATCH 11/12] fix: CCB responses must be a valid JSON Signed-off-by: kyteinsky --- lib/Service/LangRopeService.php | 8 ++ .../LangRopeServiceResponseTest.php | 99 +++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 tests/integration/LangRopeServiceResponseTest.php diff --git a/lib/Service/LangRopeService.php b/lib/Service/LangRopeService.php index 84e8cf93..2485325d 100644 --- a/lib/Service/LangRopeService.php +++ b/lib/Service/LangRopeService.php @@ -180,6 +180,14 @@ private function requestToExApp( } } + if (!is_array($finalBody)) { + $this->logger->error('Malformed response from Context Chat Backend (ExApp), expected a JSON object or array', [ + 'response' => $response->getBody(), + 'route' => $route, + 'method' => $method, + ]); + throw new RuntimeException('Malformed response from Context Chat Backend (ExApp) for ' . $route . ': expected a JSON object or array'); + } return $finalBody; } diff --git a/tests/integration/LangRopeServiceResponseTest.php b/tests/integration/LangRopeServiceResponseTest.php new file mode 100644 index 00000000..6ed7c5e7 --- /dev/null +++ b/tests/integration/LangRopeServiceResponseTest.php @@ -0,0 +1,99 @@ +appConfig = (new Application())->getContainer()->get(IAppConfig::class); + $this->savedBackendInit = $this->appConfig->getAppValueString('backend_init', 'false', lazy: true); + $this->appConfig->setAppValueString('backend_init', 'true', lazy: true); + \OC::$server->registerService(PublicFunctions::class, static fn () => new PublicFunctions()); + } + + protected function tearDown(): void { + $this->appConfig->setAppValueString('backend_init', $this->savedBackendInit, lazy: true); + parent::tearDown(); + } + + private function service(): LangRopeService { + $service = Server::get(LangRopeService::class); + $appManager = $this->createMock(IAppManager::class); + $appManager->method('isEnabledForUser')->willReturn(true); + (new \ReflectionProperty($service, 'appManager'))->setValue($service, $appManager); + return $service; + } + + private function backendAnswers(string $body): void { + $response = $this->createMock(IResponse::class); + $response->method('getHeader')->willReturnCallback(static fn (string $name) => $name === 'Content-Type' ? 'application/json' : ''); + $response->method('getBody')->willReturn($body); + $response->method('getStatusCode')->willReturn(200); + PublicFunctions::$response = $response; + } + + /** @return array */ + public static function malformedBodies(): array { + return [ + 'null' => ['null'], + 'a number' => ['3'], + 'a string' => ['"ok"'], + 'not JSON' => ['proxy error'], + ]; + } + + #[DataProvider('malformedBodies')] + public function testMalformedResponseIsReportedAsABackendError(string $body): void { + $this->backendAnswers($body); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage('Malformed response from Context Chat Backend (ExApp) for /countIndexedDocuments'); + $this->service()->getIndexedDocumentsCounts(); + } + + public function testCountsAreReturned(): void { + $this->backendAnswers('{"files__default": 3}'); + + $this->assertSame(['files__default' => 3], $this->service()->getIndexedDocumentsCounts()); + } + } +} From 7be51e9ca68cc765899c332a3761ddfc806b7635 Mon Sep 17 00:00:00 2001 From: kyteinsky Date: Thu, 5 Nov 2026 20:10:16 +0530 Subject: [PATCH 12/12] tests: fix tests Signed-off-by: kyteinsky --- tests/integration/QueueControllerFetchGuardTest.php | 3 ++- tests/integration/ReindexServiceTest.php | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/QueueControllerFetchGuardTest.php b/tests/integration/QueueControllerFetchGuardTest.php index ef5dc8fb..983dd02b 100644 --- a/tests/integration/QueueControllerFetchGuardTest.php +++ b/tests/integration/QueueControllerFetchGuardTest.php @@ -34,7 +34,8 @@ class QueueControllerFetchGuardTest extends TestCase { private QueueMapper $queueMapper; private QueueActionMapper $actionMapper; private FetchGuard $guard; - private ?int $pausedUntil = null; + /** @var array|null */ + private ?array $pausedUntil = null; protected function setUp(): void { parent::setUp(); diff --git a/tests/integration/ReindexServiceTest.php b/tests/integration/ReindexServiceTest.php index b575890f..c4b452ba 100644 --- a/tests/integration/ReindexServiceTest.php +++ b/tests/integration/ReindexServiceTest.php @@ -430,7 +430,7 @@ public function testPauseExpiresOnItsOwn(): void { $pause = $guard->pause(); $this->assertTrue($guard->isPaused()); - $now += FetchGuard::PAUSE_DURATION; + $now += 24 * 60 * 60; $this->assertFalse($guard->isPaused(), 'a wipe that died must not stop the indexing forever'); $guard->resume($pause);