From 4cad77ba7a26287b832bcaf423a40773ae68f451 Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 00:23:27 -0400 Subject: [PATCH 01/14] Add persistent checkpoints to operations --- README.md | 62 ++++- ...651_create_operation_checkpoints_table.php | 30 +++ src/Commands/ForgetCommand.php | 4 +- src/Operation.php | 37 +++ src/OperationRepository.php | 53 +++- tests/Feature/CheckpointTest.php | 253 ++++++++++++++++++ tests/Feature/ForgetTest.php | 61 ++++- tests/Feature/InstallTest.php | 6 +- 8 files changed, 487 insertions(+), 19 deletions(-) create mode 100644 database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php create mode 100644 tests/Feature/CheckpointTest.php diff --git a/README.md b/README.md index 9eaffa3..fa5b5b6 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Install the package with Composer: composer require directorytree/operations ``` -Publish and run the migration: +Publish and run the migrations: ```bash php artisan vendor:publish --tag=operations-migrations @@ -219,9 +219,63 @@ The command asks for confirmation. Use `--force` to skip the prompt: php artisan operations:forget 2026_10_02_120000_backfill_company_names --force ``` -Forgetting only deletes the completion record. It does not undo previous effects, delete the file, or execute the operation. If the file is present, the next `operations:run` will execute it again. Make sure it is safe to repeat. +Forgetting deletes the completion record and any saved checkpoints. It does not undo previous effects, delete the file, or execute the operation. If the file is present, the next `operations:run` will execute it again from the beginning. Make sure it is safe to repeat. -You can also forget records whose files have been removed. The command fails if no completion record matches the supplied name. Failed operations do not need to be forgotten; they are already pending. +You can also forget records whose files have been removed, or clear checkpoints from an unfinished operation. The command fails if neither a completion record nor checkpoints match the supplied name. Failed operations are already pending; only forget them if you want to discard their saved progress. + +### Checkpoints + +Use `checkpoint()` inside an operation to save progress between attempts. Pass a key to read a value, an optional default for a missing key, or an array to persist values: + +```php +$cursor = $this->checkpoint('cursor'); +$lastId = $this->checkpoint('last_id', 0); + +$this->checkpoint(['last_id' => 15000]); + +$this->checkpoint([ + 'cursor' => $nextCursor, + 'processed' => $processed, +]); +``` + +Keys are scoped to the operation's filename. Writes update the supplied keys and preserve other saved values. Values are stored as JSON and read back as scalars or arrays; use JSON-compatible values such as strings, numbers, booleans, arrays, and `null`. Saving `null` retains the key, so reading it returns `null` even when a default is provided. + +For example, save the last processed ID to resume a backfill: + +```php +use App\Models\Company; +use DirectoryTree\Operations\Operation; +use Illuminate\Console\Command; +use Illuminate\Support\Collection; + +return new class extends Operation +{ + public function handle(Command $command): void + { + Company::query() + ->where('id', '>', $this->checkpoint('last_id', 0)) + ->chunkById(1000, function (Collection $companies) { + foreach ($companies as $company) { + $company->update(['display_name' => $company->name]); + } + + $this->checkpoint(['last_id' => $companies->last()->id]); + }); + } +}; +``` + +After a failure, `operations:run` starts `handle()` again. Your code reads the saved checkpoint to decide where to continue. Saving a checkpoint does not mark the operation complete or prevent concurrent execution. Checkpoints remain after completion until the operation is forgotten. + +In the example above, an interrupted chunk may run again. Write work that can safely repeat, or wrap each chunk's database changes and checkpoint write in a transaction on the same connection. Checkpoints use `operations.connection` and participate in its active transaction. With `WithinTransaction`, a failure rolls back all checkpoints written during that attempt along with the operation's database changes. External effects, such as API calls, are not rolled back. + +Existing installations must publish and run the new checkpoint migration before using checkpoints or `operations:forget`: + +```bash +php artisan vendor:publish --tag=operations-migrations +php artisan migrate +``` ### Transactions @@ -317,4 +371,4 @@ return [ ]; ``` -`connection` selects the database connection for the operations table and optional transactions. `null` uses the application's default connection. It does not change the connection used by your application models. +`connection` selects the database connection for the operations and checkpoint tables, and optional transactions. `null` uses the application's default connection. It does not change the connection used by your application models. diff --git a/database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php b/database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php new file mode 100644 index 0000000..9919f13 --- /dev/null +++ b/database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php @@ -0,0 +1,30 @@ +create('operation_checkpoints', function (Blueprint $table) { + $table->id(); + $table->string('operation'); + $table->string('name'); + $table->json('value'); + $table->unique(['operation', 'name']); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::connection(config('operations.connection'))->dropIfExists('operation_checkpoints'); + } +}; diff --git a/src/Commands/ForgetCommand.php b/src/Commands/ForgetCommand.php index f523eee..897b4df 100644 --- a/src/Commands/ForgetCommand.php +++ b/src/Commands/ForgetCommand.php @@ -21,7 +21,7 @@ class ForgetCommand extends Command * * @var string */ - protected $description = 'Delete the completion record of an operation'; + protected $description = 'Delete the completion record and checkpoints of an operation'; /** * Execute the console command. @@ -39,7 +39,7 @@ public function handle(OperationRepository $repository): int } if (! $repository->forget($name)) { - $this->components->error("No completion record found for operation [{$name}]."); + $this->components->error("No completion record or checkpoints found for operation [{$name}]."); return self::FAILURE; } diff --git a/src/Operation.php b/src/Operation.php index 7670b6b..11f199b 100644 --- a/src/Operation.php +++ b/src/Operation.php @@ -6,8 +6,45 @@ abstract class Operation { + /** + * The operation's filename without the extension. + */ + protected string $name; + + /** + * The repository executing the operation. + */ + protected OperationRepository $repository; + /** * Run the operation. */ abstract public function handle(Command $command): void; + + /** + * Read a checkpoint or persist an array of checkpoint values. + * + * @param string|array $key + */ + public function checkpoint(string|array $key, mixed $default = null): mixed + { + if (is_array($key)) { + $this->repository->saveCheckpoints($this->name, $key); + + return null; + } + + return $this->repository->getCheckpoint($this->name, $key, $default); + } + + /** + * Set the operation's execution context. + */ + public function setContext(string $name, OperationRepository $repository): static + { + $this->name = $name; + $this->repository = $repository; + + return $this; + } } diff --git a/src/OperationRepository.php b/src/OperationRepository.php index ba4f561..7072f2b 100644 --- a/src/OperationRepository.php +++ b/src/OperationRepository.php @@ -29,13 +29,56 @@ public function completed(): Collection } /** - * Delete the completion record for the given operation. + * Get a checkpoint from the writer connection. */ - public function forget(string $name): bool + public function getCheckpoint(string $operation, string $name, mixed $default = null): mixed { - return $this->connection->table('operations') + $value = $this->connection->table('operation_checkpoints') + ->useWritePdo() + ->where('operation', $operation) ->where('name', $name) - ->delete() > 0; + ->value('value'); + + return $value === null ? value($default) : json_decode($value, true, flags: JSON_THROW_ON_ERROR); + } + + /** + * Persist checkpoint values without replacing other keys. + * + * @param array $values + */ + public function saveCheckpoints(string $operation, array $values): void + { + $checkpoints = []; + + foreach ($values as $name => $value) { + $checkpoints[] = [ + 'operation' => $operation, + 'name' => $name, + 'value' => json_encode($value, JSON_THROW_ON_ERROR), + ]; + } + + $this->connection->table('operation_checkpoints') + ->upsert($checkpoints, ['operation', 'name'], ['value']); + } + + /** + * Delete the completion record and checkpoints for the given operation. + */ + public function forget(string $name): bool + { + return $this->connection->transaction(function () use ($name) { + $deleted = $this->connection->table('operations') + ->where('name', $name) + ->delete(); + + $deleted += $this->connection->table('operation_checkpoints') + ->where('operation', $name) + ->delete(); + + return $deleted > 0; + }); } /** @@ -43,6 +86,8 @@ public function forget(string $name): bool */ public function run(string $name, Operation $operation, Command $command): void { + $operation->setContext($name, $this); + $run = function () use ($name, $operation, $command) { $operation->handle($command); diff --git a/tests/Feature/CheckpointTest.php b/tests/Feature/CheckpointTest.php new file mode 100644 index 0000000..af17d11 --- /dev/null +++ b/tests/Feature/CheckpointTest.php @@ -0,0 +1,253 @@ + 'next-page', + 'count' => 0, + 'ratio' => 1.5, + 'enabled' => false, + 'empty' => '', + 'nullable' => null, + 'ids' => [1, 2, 3], + 'metadata' => ['page' => 2, 'finished' => false], + ]; + + app()->instance('checkpoint_values', $values); + app()->instance('observed', collect()); + + File::ensureDirectoryExists(config('operations.path')); + File::put(config('operations.path').'/2026_10_05_120000_example.php', <<<'PHP' + push($this->checkpoint('missing')); + app('observed')->push($this->checkpoint('missing', 42)); + + $this->checkpoint(app('checkpoint_values')); + $this->checkpoint(['cursor' => 'last-page']); + $this->checkpoint([]); + + foreach (app('checkpoint_values') as $key => $value) { + app('observed')->push($this->checkpoint($key, 'missing')); + } + } + }; + PHP); + + artisan('operations:run')->assertSuccessful(); + + $values['cursor'] = 'last-page'; + + expect(app('observed')->all())->toBe([null, 42, ...array_values($values)]) + ->and(DB::table('operation_checkpoints')->count())->toBe(count($values)) + ->and(DB::table('operation_checkpoints')->where('name', 'missing')->exists())->toBeFalse(); +}); + +test('a failed operation resumes from its saved checkpoint on the next run', function () { + Schema::create('examples', function (Blueprint $table) { + $table->id(); + }); + + app()->instance('should_fail', true); + + File::ensureDirectoryExists(config('operations.path')); + File::put(config('operations.path').'/2026_10_05_120000_backfill.php', <<<'PHP' + checkpoint('last_id', 0) + 1; $id <= 3; $id++) { + if ($id === 3 && app('should_fail')) { + throw new \RuntimeException('Backfill interrupted.'); + } + + \Illuminate\Support\Facades\DB::table('examples')->insert(['id' => $id]); + $this->checkpoint(['last_id' => $id]); + } + } + }; + PHP); + + expect(fn () => artisan('operations:run')->run())->toThrow(RuntimeException::class, 'Backfill interrupted.'); + + expect(DB::table('examples')->pluck('id')->all())->toBe([1, 2]) + ->and(json_decode(DB::table('operation_checkpoints')->value('value')))->toBe(2) + ->and(DB::table('operations')->count())->toBe(0); + + app()->instance('should_fail', false); + + artisan('operations:run')->assertSuccessful(); + artisan('operations:run')->expectsOutputToContain('No pending operations.')->assertSuccessful(); + + expect(DB::table('examples')->pluck('id')->all())->toBe([1, 2, 3]) + ->and(json_decode(DB::table('operation_checkpoints')->value('value')))->toBe(3) + ->and(DB::table('operations')->count())->toBe(1); +}); + +test('checkpoint names are scoped to the operation filename', function () { + app()->instance('observed', collect()); + + File::ensureDirectoryExists(config('operations.path')); + + foreach (['2026_10_05_120000_first', '2026_10_05_120001_second'] as $name) { + File::put(config('operations.path')."/{$name}.php", <<<'PHP' + push($this->checkpoint('cursor')); + $this->checkpoint(['cursor' => basename(__FILE__, '.php')]); + app('observed')->push($this->checkpoint('cursor')); + } + }; + PHP); + } + + artisan('operations:run')->assertSuccessful(); + + expect(app('observed')->all())->toBe([ + null, '2026_10_05_120000_first', + null, '2026_10_05_120001_second', + ])->and(DB::table('operation_checkpoints')->count())->toBe(2); +}); + +test('a failed chunk rolls back its work and checkpoint without losing earlier progress', function () { + Schema::create('examples', function (Blueprint $table) { + $table->id(); + }); + + File::ensureDirectoryExists(config('operations.path')); + File::put(config('operations.path').'/2026_10_05_120000_backfill.php', <<<'PHP' + insert(['id' => $id]); + $this->checkpoint(['last_id' => $id]); + + if ($id === 2) { + throw new \RuntimeException('Chunk failed.'); + } + }); + } + } + }; + PHP); + + expect(fn () => artisan('operations:run')->run())->toThrow(RuntimeException::class, 'Chunk failed.'); + + expect(DB::table('examples')->pluck('id')->all())->toBe([1]) + ->and(json_decode(DB::table('operation_checkpoints')->value('value')))->toBe(1) + ->and(DB::table('operations')->count())->toBe(0); +}); + +test('checkpoints participate in the whole operation transaction', function () { + $name = '2026_10_05_120000_backfill'; + + DB::table('operation_checkpoints')->insert([ + 'operation' => $name, + 'name' => 'last_id', + 'value' => '10', + ]); + + File::ensureDirectoryExists(config('operations.path')); + File::put(config('operations.path')."/{$name}.php", <<<'PHP' + checkpoint(['last_id' => 20, 'finished' => true]); + + throw new \RuntimeException('Backfill failed.'); + } + }; + PHP); + + expect(fn () => artisan('operations:run')->run())->toThrow(RuntimeException::class, 'Backfill failed.'); + + expect(DB::table('operation_checkpoints')->count())->toBe(1) + ->and(json_decode(DB::table('operation_checkpoints')->value('value')))->toBe(10) + ->and(DB::table('operations')->count())->toBe(0); +}); + +test('invalid JSON values fail before any checkpoints are changed', function () { + File::ensureDirectoryExists(config('operations.path')); + File::put(config('operations.path').'/2026_10_05_120000_example.php', <<<'PHP' + checkpoint(['last_id' => 10]); + $this->checkpoint(['last_id' => 20, 'invalid' => NAN]); + } + }; + PHP); + + expect(fn () => artisan('operations:run')->run())->toThrow(JsonException::class); + + expect(DB::table('operation_checkpoints')->count())->toBe(1) + ->and(json_decode(DB::table('operation_checkpoints')->value('value')))->toBe(10) + ->and(DB::table('operations')->count())->toBe(0); +}); + +test('checkpoints use the configured connection and read from its writer', function () { + File::ensureDirectoryExists(config('operations.path')); + $writer = config('operations.path').'/writer.sqlite'; + $reader = config('operations.path').'/reader.sqlite'; + File::put($writer, ''); + File::put($reader, ''); + + config([ + 'database.connections.operations' => [ + 'driver' => 'sqlite', + 'database' => $writer, + 'read' => ['database' => $reader], + 'write' => ['database' => $writer], + 'prefix' => '', + ], + 'operations.connection' => 'operations', + ]); + + foreach (File::glob(__DIR__.'/../../database/migrations/*.php') as $path) { + File::getRequire($path)->up(); + } + + app()->instance('observed', collect()); + + File::put(config('operations.path').'/2026_10_05_120000_example.php', <<<'PHP' + push($this->checkpoint('cursor', 'initial')); + $this->checkpoint(['cursor' => 'next-page']); + app('observed')->push($this->checkpoint('cursor')); + } + }; + PHP); + + artisan('operations:run')->assertSuccessful(); + + expect(app('observed')->all())->toBe(['initial', 'next-page']) + ->and(DB::connection('operations')->table('operation_checkpoints')->useWritePdo()->count())->toBe(1) + ->and(DB::table('operation_checkpoints')->count())->toBe(0); + + config(['operations.connection' => null]); +}); diff --git a/tests/Feature/ForgetTest.php b/tests/Feature/ForgetTest.php index 8c30353..faea8c4 100644 --- a/tests/Feature/ForgetTest.php +++ b/tests/Feature/ForgetTest.php @@ -69,13 +69,19 @@ public function handle(\Illuminate\Console\Command $command): void app()->instance('env', $environment); $name = '2026_10_02_120000_example'; DB::table('operations')->insert(['name' => $name, 'completed_at' => now()]); + DB::table('operation_checkpoints')->insert([ + 'operation' => $name, + 'name' => 'last_id', + 'value' => '10', + ]); artisan('operations:forget', ['name' => $name]) ->expectsConfirmation("Forget operation [{$name}]?", 'no') ->expectsOutputToContain('Command cancelled.') ->assertFailed(); - expect(DB::table('operations')->pluck('name')->all())->toBe([$name]); + expect(DB::table('operations')->pluck('name')->all())->toBe([$name]) + ->and(DB::table('operation_checkpoints')->pluck('operation')->all())->toBe([$name]); app()->instance('env', 'testing'); })->with(['testing', 'production']); @@ -106,11 +112,11 @@ public function handle(\Illuminate\Console\Command $command): void app()->instance('env', 'testing'); }); -test('forgetting fails when the exact completion record does not exist', function (string $name) { +test('forgetting fails when no completion record or checkpoints exist for the exact name', function (string $name) { DB::table('operations')->insert(['name' => '2026_10_02_120000_example', 'completed_at' => now()]); artisan('operations:forget', ['name' => $name, '--force' => true]) - ->expectsOutputToContain("No completion record found for operation [{$name}].") + ->expectsOutputToContain("No completion record or checkpoints found for operation [{$name}].") ->assertFailed(); expect(DB::table('operations')->pluck('name')->all())->toBe(['2026_10_02_120000_example']); @@ -126,15 +132,58 @@ public function handle(\Illuminate\Console\Command $command): void 'operations.connection' => 'operations', ]); - $migration = File::getRequire(__DIR__.'/../../database/migrations/2026_10_02_165413_create_operations_table.php'); - $migration->up(); + foreach (File::glob(__DIR__.'/../../database/migrations/*.php') as $path) { + File::getRequire($path)->up(); + } $name = '2026_10_02_120000_example'; DB::table('operations')->insert(['name' => $name, 'completed_at' => now()]); DB::connection('operations')->table('operations')->insert(['name' => $name, 'completed_at' => now()]); + DB::table('operation_checkpoints')->insert(['operation' => $name, 'name' => 'last_id', 'value' => '10']); + DB::connection('operations')->table('operation_checkpoints')->insert(['operation' => $name, 'name' => 'last_id', 'value' => '20']); artisan('operations:forget', ['name' => $name, '--force' => true])->assertSuccessful(); expect(DB::connection('operations')->table('operations')->count())->toBe(0) - ->and(DB::table('operations')->pluck('name')->all())->toBe([$name]); + ->and(DB::connection('operations')->table('operation_checkpoints')->count())->toBe(0) + ->and(DB::table('operations')->pluck('name')->all())->toBe([$name]) + ->and(DB::table('operation_checkpoints')->pluck('operation')->all())->toBe([$name]); }); + +test('forgetting clears checkpoints for completed and unfinished operations', function (bool $completed) { + $name = '2026_10_05_120000_example'; + $other = '2026_10_05_120001_other'; + + if ($completed) { + DB::table('operations')->insert(['name' => $name, 'completed_at' => now()]); + } + + DB::table('operation_checkpoints')->insert([ + ['operation' => $name, 'name' => 'last_id', 'value' => '10'], + ['operation' => $name, 'name' => 'finished', 'value' => 'true'], + ['operation' => $other, 'name' => 'last_id', 'value' => '20'], + ]); + + app()->instance('observed', collect()); + + File::ensureDirectoryExists(config('operations.path')); + File::put(config('operations.path')."/{$name}.php", <<<'PHP' + push($this->checkpoint('last_id', 0)); + } + }; + PHP); + + artisan('operations:forget', ['name' => $name, '--force' => true])->assertSuccessful(); + + expect(DB::table('operations')->count())->toBe(0) + ->and(DB::table('operation_checkpoints')->pluck('operation')->all())->toBe([$other]); + + artisan('operations:run', ['operation' => $name])->assertSuccessful(); + + expect(app('observed')->all())->toBe([0]); +})->with(['completed' => true, 'unfinished' => false]); diff --git a/tests/Feature/InstallTest.php b/tests/Feature/InstallTest.php index f5beae8..46aec79 100644 --- a/tests/Feature/InstallTest.php +++ b/tests/Feature/InstallTest.php @@ -5,16 +5,16 @@ use function Pest\Laravel\artisan; -test('the migration can be published with a current timestamp', function () { +test('the migrations can be published with a current timestamp', function (string $table) { Date::setTestNow('2026-10-02 13:45:00'); artisan('vendor:publish', ['--tag' => 'operations-migrations'])->assertSuccessful(); - $files = File::glob(database_path('migrations/*_create_operations_table.php')); + $files = File::glob(database_path("migrations/*_create_{$table}_table.php")); expect($files)->toHaveCount(1) ->and(basename($files[0]))->toStartWith('2026_10_02_1345'); -}); +})->with(['operations', 'operation_checkpoints']); test('the package configuration can be published', function () { artisan('vendor:publish', ['--tag' => 'operations-config'])->assertSuccessful(); From f107fb9eefd980aa61a7c4720c3f06c78f0296b9 Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 00:24:41 -0400 Subject: [PATCH 02/14] Publish the checkpoint migration separately for upgrades --- README.md | 2 +- src/OperationsServiceProvider.php | 6 +++++- tests/Feature/InstallTest.php | 11 +++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fa5b5b6..43a2ede 100644 --- a/README.md +++ b/README.md @@ -273,7 +273,7 @@ In the example above, an interrupted chunk may run again. Write work that can sa Existing installations must publish and run the new checkpoint migration before using checkpoints or `operations:forget`: ```bash -php artisan vendor:publish --tag=operations-migrations +php artisan vendor:publish --tag=operations-checkpoints-migration php artisan migrate ``` diff --git a/src/OperationsServiceProvider.php b/src/OperationsServiceProvider.php index 80d4021..db7ab4f 100644 --- a/src/OperationsServiceProvider.php +++ b/src/OperationsServiceProvider.php @@ -40,8 +40,12 @@ public function boot(): void ], 'operations-config'); $this->publishesMigrations([ - __DIR__.'/../database/migrations' => database_path('migrations'), + __DIR__.'/../database/migrations/2026_10_02_165413_create_operations_table.php' => database_path('migrations/2026_10_02_165413_create_operations_table.php'), ], 'operations-migrations'); + + $this->publishesMigrations([ + __DIR__.'/../database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php' => database_path('migrations/2026_10_05_041651_create_operation_checkpoints_table.php'), + ], ['operations-migrations', 'operations-checkpoints-migration']); } } } diff --git a/tests/Feature/InstallTest.php b/tests/Feature/InstallTest.php index 46aec79..77f834d 100644 --- a/tests/Feature/InstallTest.php +++ b/tests/Feature/InstallTest.php @@ -21,3 +21,14 @@ expect(File::get(config_path('operations.php')))->toBe(File::get(__DIR__.'/../../config/operations.php')); }); + +test('existing installations can publish only the checkpoint migration', function () { + File::ensureDirectoryExists(database_path('migrations')); + $original = database_path('migrations/2026_10_02_120000_create_operations_table.php'); + File::copy(__DIR__.'/../../database/migrations/2026_10_02_165413_create_operations_table.php', $original); + + artisan('vendor:publish', ['--tag' => 'operations-checkpoints-migration'])->assertSuccessful(); + + expect(File::glob(database_path('migrations/*_create_operations_table.php')))->toBe([$original]) + ->and(File::glob(database_path('migrations/*_create_operation_checkpoints_table.php')))->toHaveCount(1); +}); From 6142aff29d66cfdff9dcf13aad7bd396212e6469 Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 11:31:42 -0400 Subject: [PATCH 03/14] Make operation checkpoints opt-in --- README.md | 21 ++++--- src/Concerns/HasCheckpoints.php | 45 +++++++++++++++ src/Operation.php | 37 ------------- src/OperationRepository.php | 13 +++-- src/OperationsServiceProvider.php | 2 +- tests/Feature/CheckpointTest.php | 91 +++++++++++++++++++++++++++++++ tests/Feature/ForgetTest.php | 2 + tests/Feature/InstallTest.php | 16 ++++-- 8 files changed, 172 insertions(+), 55 deletions(-) create mode 100644 src/Concerns/HasCheckpoints.php diff --git a/README.md b/README.md index 43a2ede..8e38477 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Install the package with Composer: composer require directorytree/operations ``` -Publish and run the migrations: +Publish and run the migration: ```bash php artisan vendor:publish --tag=operations-migrations @@ -225,7 +225,14 @@ You can also forget records whose files have been removed, or clear checkpoints ### Checkpoints -Use `checkpoint()` inside an operation to save progress between attempts. Pass a key to read a value, an optional default for a missing key, or an array to persist values: +Checkpoints are optional. Publish and run their migration only if you need to save progress between attempts: + +```bash +php artisan vendor:publish --tag=operations-checkpoints-migration +php artisan migrate +``` + +Add the `DirectoryTree\Operations\Concerns\HasCheckpoints` trait to an operation to use `checkpoint()`. Pass a key to read a value, an optional default for a missing key, or an array to persist values: ```php $cursor = $this->checkpoint('cursor'); @@ -245,12 +252,15 @@ For example, save the last processed ID to resume a backfill: ```php use App\Models\Company; +use DirectoryTree\Operations\Concerns\HasCheckpoints; use DirectoryTree\Operations\Operation; use Illuminate\Console\Command; use Illuminate\Support\Collection; return new class extends Operation { + use HasCheckpoints; + public function handle(Command $command): void { Company::query() @@ -270,12 +280,7 @@ After a failure, `operations:run` starts `handle()` again. Your code reads the s In the example above, an interrupted chunk may run again. Write work that can safely repeat, or wrap each chunk's database changes and checkpoint write in a transaction on the same connection. Checkpoints use `operations.connection` and participate in its active transaction. With `WithinTransaction`, a failure rolls back all checkpoints written during that attempt along with the operation's database changes. External effects, such as API calls, are not rolled back. -Existing installations must publish and run the new checkpoint migration before using checkpoints or `operations:forget`: - -```bash -php artisan vendor:publish --tag=operations-checkpoints-migration -php artisan migrate -``` +The standard `operations-migrations` tag only publishes the operations table. Running, checking status, and forgetting operations work without the checkpoint table; `operations:forget` clears checkpoints when that table exists. Calling `checkpoint()` before installing its migration raises a database exception. ### Transactions diff --git a/src/Concerns/HasCheckpoints.php b/src/Concerns/HasCheckpoints.php new file mode 100644 index 0000000..5235664 --- /dev/null +++ b/src/Concerns/HasCheckpoints.php @@ -0,0 +1,45 @@ + $key + */ + public function checkpoint(string|array $key, mixed $default = null): mixed + { + if (is_array($key)) { + $this->checkpointRepository->saveCheckpoints($this->operationName, $key); + + return null; + } + + return $this->checkpointRepository->getCheckpoint($this->operationName, $key, $default); + } + + /** + * Set the operation's checkpoint context. + */ + public function setCheckpointContext(string $name, OperationRepository $repository): static + { + $this->operationName = $name; + $this->checkpointRepository = $repository; + + return $this; + } +} diff --git a/src/Operation.php b/src/Operation.php index 11f199b..7670b6b 100644 --- a/src/Operation.php +++ b/src/Operation.php @@ -6,45 +6,8 @@ abstract class Operation { - /** - * The operation's filename without the extension. - */ - protected string $name; - - /** - * The repository executing the operation. - */ - protected OperationRepository $repository; - /** * Run the operation. */ abstract public function handle(Command $command): void; - - /** - * Read a checkpoint or persist an array of checkpoint values. - * - * @param string|array $key - */ - public function checkpoint(string|array $key, mixed $default = null): mixed - { - if (is_array($key)) { - $this->repository->saveCheckpoints($this->name, $key); - - return null; - } - - return $this->repository->getCheckpoint($this->name, $key, $default); - } - - /** - * Set the operation's execution context. - */ - public function setContext(string $name, OperationRepository $repository): static - { - $this->name = $name; - $this->repository = $repository; - - return $this; - } } diff --git a/src/OperationRepository.php b/src/OperationRepository.php index 7072f2b..b9b3d85 100644 --- a/src/OperationRepository.php +++ b/src/OperationRepository.php @@ -2,6 +2,7 @@ namespace DirectoryTree\Operations; +use DirectoryTree\Operations\Concerns\HasCheckpoints; use DirectoryTree\Operations\Contracts\WithinTransaction; use Illuminate\Console\Command; use Illuminate\Database\ConnectionInterface; @@ -73,9 +74,11 @@ public function forget(string $name): bool ->where('name', $name) ->delete(); - $deleted += $this->connection->table('operation_checkpoints') - ->where('operation', $name) - ->delete(); + if ($this->connection->getSchemaBuilder()->hasTable('operation_checkpoints')) { + $deleted += $this->connection->table('operation_checkpoints') + ->where('operation', $name) + ->delete(); + } return $deleted > 0; }); @@ -86,7 +89,9 @@ public function forget(string $name): bool */ public function run(string $name, Operation $operation, Command $command): void { - $operation->setContext($name, $this); + if (in_array(HasCheckpoints::class, class_uses_recursive($operation))) { + $operation->setCheckpointContext($name, $this); + } $run = function () use ($name, $operation, $command) { $operation->handle($command); diff --git a/src/OperationsServiceProvider.php b/src/OperationsServiceProvider.php index db7ab4f..b198dea 100644 --- a/src/OperationsServiceProvider.php +++ b/src/OperationsServiceProvider.php @@ -45,7 +45,7 @@ public function boot(): void $this->publishesMigrations([ __DIR__.'/../database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php' => database_path('migrations/2026_10_05_041651_create_operation_checkpoints_table.php'), - ], ['operations-migrations', 'operations-checkpoints-migration']); + ], 'operations-checkpoints-migration'); } } } diff --git a/tests/Feature/CheckpointTest.php b/tests/Feature/CheckpointTest.php index af17d11..e51eafa 100644 --- a/tests/Feature/CheckpointTest.php +++ b/tests/Feature/CheckpointTest.php @@ -1,5 +1,6 @@ [ + 'driver' => 'sqlite', + 'database' => ':memory:', + 'prefix' => '', + ], + 'operations.connection' => 'operations', + ]); + + File::getRequire(__DIR__.'/../../database/migrations/2026_10_02_165413_create_operations_table.php')->up(); + } else { + Schema::drop('operation_checkpoints'); + } + + $connection = DB::connection(config('operations.connection')); + $name = '2026_10_05_120000_example'; + + File::ensureDirectoryExists(config('operations.path')); + File::put(config('operations.path')."/{$name}.php", <<<'PHP' + info('Operation executed.'); + } + }; + PHP); + + artisan('operations:run')->expectsOutputToContain('Operation executed.')->assertSuccessful(); + artisan('operations:run')->expectsOutputToContain('No pending operations.')->assertSuccessful(); + + artisan('operations:status')->expectsTable( + ['Operation', 'Status', 'Completed at', 'File'], + [[$name, 'Completed', $connection->table('operations')->value('completed_at'), 'Present']], + )->assertSuccessful(); + + File::delete(config('operations.path')."/{$name}.php"); + + artisan('operations:forget', ['name' => $name, '--force' => true])->assertSuccessful(); + artisan('operations:forget', ['name' => $name, '--force' => true])->assertFailed(); + + expect($connection->table('operations')->count())->toBe(0) + ->and($connection->getSchemaBuilder()->hasTable('operation_checkpoints'))->toBeFalse(); + + config(['operations.connection' => null]); +})->with(['default connection' => false, 'separate connection' => true]); + +test('using checkpoints without their migration raises a database exception', function (string|array $argument) { + Schema::drop('operation_checkpoints'); + app()->instance('checkpoint_argument', $argument); + + File::ensureDirectoryExists(config('operations.path')); + File::put(config('operations.path').'/2026_10_05_120000_example.php', <<<'PHP' + checkpoint(app('checkpoint_argument')); + } + }; + PHP); + + expect(fn () => artisan('operations:run')->run())->toThrow(QueryException::class, 'operation_checkpoints'); + + expect(DB::table('operations')->count())->toBe(0); +})->with([ + 'read' => ['cursor'], + 'write' => [['cursor' => 'next-page']], +]); + test('checkpoints read defaults and persist JSON values while preserving other keys', function () { $values = [ 'cursor' => 'next-page', @@ -27,6 +104,8 @@ push($this->checkpoint('missing')); @@ -64,6 +143,8 @@ public function handle(\Illuminate\Console\Command $command): void checkpoint('last_id', 0) + 1; $id <= 3; $id++) { @@ -104,6 +185,8 @@ public function handle(\Illuminate\Console\Command $command): void push($this->checkpoint('cursor')); @@ -132,6 +215,8 @@ public function handle(\Illuminate\Console\Command $command): void checkpoint(['last_id' => 20, 'finished' => true]); @@ -191,6 +278,8 @@ public function handle(\Illuminate\Console\Command $command): void checkpoint(['last_id' => 10]); @@ -234,6 +323,8 @@ public function handle(\Illuminate\Console\Command $command): void push($this->checkpoint('cursor', 'initial')); diff --git a/tests/Feature/ForgetTest.php b/tests/Feature/ForgetTest.php index faea8c4..c972be0 100644 --- a/tests/Feature/ForgetTest.php +++ b/tests/Feature/ForgetTest.php @@ -171,6 +171,8 @@ public function handle(\Illuminate\Console\Command $command): void push($this->checkpoint('last_id', 0)); diff --git a/tests/Feature/InstallTest.php b/tests/Feature/InstallTest.php index 77f834d..0de5940 100644 --- a/tests/Feature/InstallTest.php +++ b/tests/Feature/InstallTest.php @@ -5,16 +5,17 @@ use function Pest\Laravel\artisan; -test('the migrations can be published with a current timestamp', function (string $table) { +test('the standard migration can be published without opting into checkpoints', function () { Date::setTestNow('2026-10-02 13:45:00'); artisan('vendor:publish', ['--tag' => 'operations-migrations'])->assertSuccessful(); - $files = File::glob(database_path("migrations/*_create_{$table}_table.php")); + $files = File::glob(database_path('migrations/*_create_operations_table.php')); expect($files)->toHaveCount(1) - ->and(basename($files[0]))->toStartWith('2026_10_02_1345'); -})->with(['operations', 'operation_checkpoints']); + ->and(basename($files[0]))->toStartWith('2026_10_02_1345') + ->and(File::glob(database_path('migrations/*_create_operation_checkpoints_table.php')))->toBe([]); +}); test('the package configuration can be published', function () { artisan('vendor:publish', ['--tag' => 'operations-config'])->assertSuccessful(); @@ -23,12 +24,17 @@ }); test('existing installations can publish only the checkpoint migration', function () { + Date::setTestNow('2026-10-05 12:00:00'); + File::ensureDirectoryExists(database_path('migrations')); $original = database_path('migrations/2026_10_02_120000_create_operations_table.php'); File::copy(__DIR__.'/../../database/migrations/2026_10_02_165413_create_operations_table.php', $original); artisan('vendor:publish', ['--tag' => 'operations-checkpoints-migration'])->assertSuccessful(); + $files = File::glob(database_path('migrations/*_create_operation_checkpoints_table.php')); + expect(File::glob(database_path('migrations/*_create_operations_table.php')))->toBe([$original]) - ->and(File::glob(database_path('migrations/*_create_operation_checkpoints_table.php')))->toHaveCount(1); + ->and($files)->toHaveCount(1) + ->and(basename($files[0]))->toStartWith('2026_10_05_1200'); }); From fc3faf667053326f7e67db7b5f4f8f7f6b229792 Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 11:48:05 -0400 Subject: [PATCH 04/14] Separate operation and checkpoint repositories --- src/CheckpointRepository.php | 64 +++++++++++++++++++++++++++++++ src/Concerns/HasCheckpoints.php | 10 ++--- src/OperationRepository.php | 48 +++-------------------- src/OperationsServiceProvider.php | 7 +++- 4 files changed, 80 insertions(+), 49 deletions(-) create mode 100644 src/CheckpointRepository.php diff --git a/src/CheckpointRepository.php b/src/CheckpointRepository.php new file mode 100644 index 0000000..32ca43b --- /dev/null +++ b/src/CheckpointRepository.php @@ -0,0 +1,64 @@ +connection->table('operation_checkpoints') + ->useWritePdo() + ->where('operation', $operation) + ->where('name', $name) + ->value('value'); + + return $value === null ? value($default) : json_decode($value, true, flags: JSON_THROW_ON_ERROR); + } + + /** + * Persist checkpoint values without replacing other keys. + * + * @param array $values + */ + public function save(string $operation, array $values): void + { + $checkpoints = []; + + foreach ($values as $name => $value) { + $checkpoints[] = [ + 'operation' => $operation, + 'name' => $name, + 'value' => json_encode($value, JSON_THROW_ON_ERROR), + ]; + } + + $this->connection->table('operation_checkpoints') + ->upsert($checkpoints, ['operation', 'name'], ['value']); + } + + /** + * Delete an operation's checkpoints when checkpoint storage is installed. + */ + public function forget(string $operation): bool + { + if (! $this->connection->getSchemaBuilder()->hasTable('operation_checkpoints')) { + return false; + } + + return $this->connection->table('operation_checkpoints') + ->where('operation', $operation) + ->delete() > 0; + } +} diff --git a/src/Concerns/HasCheckpoints.php b/src/Concerns/HasCheckpoints.php index 5235664..c1b5404 100644 --- a/src/Concerns/HasCheckpoints.php +++ b/src/Concerns/HasCheckpoints.php @@ -2,7 +2,7 @@ namespace DirectoryTree\Operations\Concerns; -use DirectoryTree\Operations\OperationRepository; +use DirectoryTree\Operations\CheckpointRepository; trait HasCheckpoints { @@ -14,7 +14,7 @@ trait HasCheckpoints /** * The repository storing the operation's checkpoints. */ - protected OperationRepository $checkpointRepository; + protected CheckpointRepository $checkpointRepository; /** * Read a checkpoint or persist an array of checkpoint values. @@ -24,18 +24,18 @@ trait HasCheckpoints public function checkpoint(string|array $key, mixed $default = null): mixed { if (is_array($key)) { - $this->checkpointRepository->saveCheckpoints($this->operationName, $key); + $this->checkpointRepository->save($this->operationName, $key); return null; } - return $this->checkpointRepository->getCheckpoint($this->operationName, $key, $default); + return $this->checkpointRepository->get($this->operationName, $key, $default); } /** * Set the operation's checkpoint context. */ - public function setCheckpointContext(string $name, OperationRepository $repository): static + public function setCheckpointContext(string $name, CheckpointRepository $repository): static { $this->operationName = $name; $this->checkpointRepository = $repository; diff --git a/src/OperationRepository.php b/src/OperationRepository.php index b9b3d85..77b773c 100644 --- a/src/OperationRepository.php +++ b/src/OperationRepository.php @@ -14,7 +14,8 @@ class OperationRepository * Create a new operation repository. */ public function __construct( - protected ConnectionInterface $connection + protected ConnectionInterface $connection, + protected CheckpointRepository $checkpoints ) {} /** @@ -29,41 +30,6 @@ public function completed(): Collection ->pluck('completed_at', 'name'); } - /** - * Get a checkpoint from the writer connection. - */ - public function getCheckpoint(string $operation, string $name, mixed $default = null): mixed - { - $value = $this->connection->table('operation_checkpoints') - ->useWritePdo() - ->where('operation', $operation) - ->where('name', $name) - ->value('value'); - - return $value === null ? value($default) : json_decode($value, true, flags: JSON_THROW_ON_ERROR); - } - - /** - * Persist checkpoint values without replacing other keys. - * - * @param array $values - */ - public function saveCheckpoints(string $operation, array $values): void - { - $checkpoints = []; - - foreach ($values as $name => $value) { - $checkpoints[] = [ - 'operation' => $operation, - 'name' => $name, - 'value' => json_encode($value, JSON_THROW_ON_ERROR), - ]; - } - - $this->connection->table('operation_checkpoints') - ->upsert($checkpoints, ['operation', 'name'], ['value']); - } - /** * Delete the completion record and checkpoints for the given operation. */ @@ -74,13 +40,9 @@ public function forget(string $name): bool ->where('name', $name) ->delete(); - if ($this->connection->getSchemaBuilder()->hasTable('operation_checkpoints')) { - $deleted += $this->connection->table('operation_checkpoints') - ->where('operation', $name) - ->delete(); - } + $forgotten = $this->checkpoints->forget($name); - return $deleted > 0; + return $deleted > 0 || $forgotten; }); } @@ -90,7 +52,7 @@ public function forget(string $name): bool public function run(string $name, Operation $operation, Command $command): void { if (in_array(HasCheckpoints::class, class_uses_recursive($operation))) { - $operation->setCheckpointContext($name, $this); + $operation->setCheckpointContext($name, $this->checkpoints); } $run = function () use ($name, $operation, $command) { diff --git a/src/OperationsServiceProvider.php b/src/OperationsServiceProvider.php index b198dea..ff2f063 100644 --- a/src/OperationsServiceProvider.php +++ b/src/OperationsServiceProvider.php @@ -17,9 +17,14 @@ public function register(): void { $this->mergeConfigFrom(__DIR__.'/../config/operations.php', 'operations'); - $this->app->bind(OperationRepository::class, fn ($app) => new OperationRepository( + $this->app->bind(CheckpointRepository::class, fn ($app) => new CheckpointRepository( $app['db']->connection($app['config']['operations.connection']) )); + + $this->app->bind(OperationRepository::class, fn ($app) => new OperationRepository( + $app['db']->connection($app['config']['operations.connection']), + $app->make(CheckpointRepository::class) + )); } /** From 77634a65a4feffa661e3a326b1ecb12df4c969ed Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 11:53:29 -0400 Subject: [PATCH 05/14] Rename checkpoint repository property to checkpoints --- src/Concerns/HasCheckpoints.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Concerns/HasCheckpoints.php b/src/Concerns/HasCheckpoints.php index c1b5404..7d32179 100644 --- a/src/Concerns/HasCheckpoints.php +++ b/src/Concerns/HasCheckpoints.php @@ -14,7 +14,7 @@ trait HasCheckpoints /** * The repository storing the operation's checkpoints. */ - protected CheckpointRepository $checkpointRepository; + protected CheckpointRepository $checkpoints; /** * Read a checkpoint or persist an array of checkpoint values. @@ -24,12 +24,12 @@ trait HasCheckpoints public function checkpoint(string|array $key, mixed $default = null): mixed { if (is_array($key)) { - $this->checkpointRepository->save($this->operationName, $key); + $this->checkpoints->save($this->operationName, $key); return null; } - return $this->checkpointRepository->get($this->operationName, $key, $default); + return $this->checkpoints->get($this->operationName, $key, $default); } /** @@ -38,7 +38,7 @@ public function checkpoint(string|array $key, mixed $default = null): mixed public function setCheckpointContext(string $name, CheckpointRepository $repository): static { $this->operationName = $name; - $this->checkpointRepository = $repository; + $this->checkpoints = $repository; return $this; } From b570817fdf1eb8fe979515102de055f6a9ad1058 Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 11:54:31 -0400 Subject: [PATCH 06/14] Rename operation name property to name --- src/Concerns/HasCheckpoints.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Concerns/HasCheckpoints.php b/src/Concerns/HasCheckpoints.php index 7d32179..668b219 100644 --- a/src/Concerns/HasCheckpoints.php +++ b/src/Concerns/HasCheckpoints.php @@ -9,7 +9,7 @@ trait HasCheckpoints /** * The operation's filename without the extension. */ - protected string $operationName; + protected string $name; /** * The repository storing the operation's checkpoints. @@ -24,12 +24,12 @@ trait HasCheckpoints public function checkpoint(string|array $key, mixed $default = null): mixed { if (is_array($key)) { - $this->checkpoints->save($this->operationName, $key); + $this->checkpoints->save($this->name, $key); return null; } - return $this->checkpoints->get($this->operationName, $key, $default); + return $this->checkpoints->get($this->name, $key, $default); } /** @@ -37,7 +37,7 @@ public function checkpoint(string|array $key, mixed $default = null): mixed */ public function setCheckpointContext(string $name, CheckpointRepository $repository): static { - $this->operationName = $name; + $this->name = $name; $this->checkpoints = $repository; return $this; From 03be7b951b0f27000c179254e9a703894b4c48d6 Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 11:57:44 -0400 Subject: [PATCH 07/14] Scope checkpoint repositories to individual operations --- src/CheckpointRepository.php | 21 +++++++++++---------- src/Concerns/HasCheckpoints.php | 16 +++++----------- src/OperationRepository.php | 15 +++++++++++---- src/OperationsServiceProvider.php | 7 +------ 4 files changed, 28 insertions(+), 31 deletions(-) diff --git a/src/CheckpointRepository.php b/src/CheckpointRepository.php index 32ca43b..10ce71f 100644 --- a/src/CheckpointRepository.php +++ b/src/CheckpointRepository.php @@ -10,18 +10,19 @@ class CheckpointRepository * Create a new checkpoint repository. */ public function __construct( - protected ConnectionInterface $connection + protected ConnectionInterface $connection, + protected string $name ) {} /** * Get a checkpoint from the writer connection. */ - public function get(string $operation, string $name, mixed $default = null): mixed + public function get(string $key, mixed $default = null): mixed { $value = $this->connection->table('operation_checkpoints') ->useWritePdo() - ->where('operation', $operation) - ->where('name', $name) + ->where('operation', $this->name) + ->where('name', $key) ->value('value'); return $value === null ? value($default) : json_decode($value, true, flags: JSON_THROW_ON_ERROR); @@ -32,14 +33,14 @@ public function get(string $operation, string $name, mixed $default = null): mix * * @param array $values */ - public function save(string $operation, array $values): void + public function save(array $values): void { $checkpoints = []; - foreach ($values as $name => $value) { + foreach ($values as $key => $value) { $checkpoints[] = [ - 'operation' => $operation, - 'name' => $name, + 'operation' => $this->name, + 'name' => $key, 'value' => json_encode($value, JSON_THROW_ON_ERROR), ]; } @@ -51,14 +52,14 @@ public function save(string $operation, array $values): void /** * Delete an operation's checkpoints when checkpoint storage is installed. */ - public function forget(string $operation): bool + public function forget(): bool { if (! $this->connection->getSchemaBuilder()->hasTable('operation_checkpoints')) { return false; } return $this->connection->table('operation_checkpoints') - ->where('operation', $operation) + ->where('operation', $this->name) ->delete() > 0; } } diff --git a/src/Concerns/HasCheckpoints.php b/src/Concerns/HasCheckpoints.php index 668b219..fbefd0e 100644 --- a/src/Concerns/HasCheckpoints.php +++ b/src/Concerns/HasCheckpoints.php @@ -6,11 +6,6 @@ trait HasCheckpoints { - /** - * The operation's filename without the extension. - */ - protected string $name; - /** * The repository storing the operation's checkpoints. */ @@ -24,21 +19,20 @@ trait HasCheckpoints public function checkpoint(string|array $key, mixed $default = null): mixed { if (is_array($key)) { - $this->checkpoints->save($this->name, $key); + $this->checkpoints->save($key); return null; } - return $this->checkpoints->get($this->name, $key, $default); + return $this->checkpoints->get($key, $default); } /** - * Set the operation's checkpoint context. + * Set the operation's checkpoint repository. */ - public function setCheckpointContext(string $name, CheckpointRepository $repository): static + public function setCheckpoints(CheckpointRepository $checkpoints): static { - $this->name = $name; - $this->checkpoints = $repository; + $this->checkpoints = $checkpoints; return $this; } diff --git a/src/OperationRepository.php b/src/OperationRepository.php index 77b773c..198ee8e 100644 --- a/src/OperationRepository.php +++ b/src/OperationRepository.php @@ -14,8 +14,7 @@ class OperationRepository * Create a new operation repository. */ public function __construct( - protected ConnectionInterface $connection, - protected CheckpointRepository $checkpoints + protected ConnectionInterface $connection ) {} /** @@ -40,7 +39,7 @@ public function forget(string $name): bool ->where('name', $name) ->delete(); - $forgotten = $this->checkpoints->forget($name); + $forgotten = $this->checkpoints($name)->forget(); return $deleted > 0 || $forgotten; }); @@ -52,7 +51,7 @@ public function forget(string $name): bool public function run(string $name, Operation $operation, Command $command): void { if (in_array(HasCheckpoints::class, class_uses_recursive($operation))) { - $operation->setCheckpointContext($name, $this->checkpoints); + $operation->setCheckpoints($this->checkpoints($name)); } $run = function () use ($name, $operation, $command) { @@ -70,4 +69,12 @@ public function run(string $name, Operation $operation, Command $command): void $run(); } } + + /** + * Create a checkpoint repository for the given operation. + */ + protected function checkpoints(string $name): CheckpointRepository + { + return new CheckpointRepository($this->connection, $name); + } } diff --git a/src/OperationsServiceProvider.php b/src/OperationsServiceProvider.php index ff2f063..b198dea 100644 --- a/src/OperationsServiceProvider.php +++ b/src/OperationsServiceProvider.php @@ -17,13 +17,8 @@ public function register(): void { $this->mergeConfigFrom(__DIR__.'/../config/operations.php', 'operations'); - $this->app->bind(CheckpointRepository::class, fn ($app) => new CheckpointRepository( - $app['db']->connection($app['config']['operations.connection']) - )); - $this->app->bind(OperationRepository::class, fn ($app) => new OperationRepository( - $app['db']->connection($app['config']['operations.connection']), - $app->make(CheckpointRepository::class) + $app['db']->connection($app['config']['operations.connection']) )); } From 778a1071b15c49487741252bd3ab1db970f97d8a Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 12:02:43 -0400 Subject: [PATCH 08/14] Expose checkpoints through the repository API --- README.md | 18 +++++----- src/CheckpointRepository.php | 6 ++-- src/Concerns/HasCheckpoints.php | 16 --------- tests/Feature/CheckpointTest.php | 58 ++++++++++++++++++-------------- tests/Feature/ForgetTest.php | 2 +- 5 files changed, 47 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 8e38477..2de2c07 100644 --- a/README.md +++ b/README.md @@ -232,15 +232,15 @@ php artisan vendor:publish --tag=operations-checkpoints-migration php artisan migrate ``` -Add the `DirectoryTree\Operations\Concerns\HasCheckpoints` trait to an operation to use `checkpoint()`. Pass a key to read a value, an optional default for a missing key, or an array to persist values: +Add the `DirectoryTree\Operations\Concerns\HasCheckpoints` trait to access the operation's checkpoint repository. Use `get()` to read a value with an optional default, and `put()` to save a value or an array of values: ```php -$cursor = $this->checkpoint('cursor'); -$lastId = $this->checkpoint('last_id', 0); +$cursor = $this->checkpoints->get('cursor'); +$lastId = $this->checkpoints->get('last_id', 0); -$this->checkpoint(['last_id' => 15000]); +$this->checkpoints->put('last_id', 15000); -$this->checkpoint([ +$this->checkpoints->put([ 'cursor' => $nextCursor, 'processed' => $processed, ]); @@ -264,23 +264,23 @@ return new class extends Operation public function handle(Command $command): void { Company::query() - ->where('id', '>', $this->checkpoint('last_id', 0)) + ->where('id', '>', $this->checkpoints->get('last_id', 0)) ->chunkById(1000, function (Collection $companies) { foreach ($companies as $company) { $company->update(['display_name' => $company->name]); } - $this->checkpoint(['last_id' => $companies->last()->id]); + $this->checkpoints->put('last_id', $companies->last()->id); }); } }; ``` -After a failure, `operations:run` starts `handle()` again. Your code reads the saved checkpoint to decide where to continue. Saving a checkpoint does not mark the operation complete or prevent concurrent execution. Checkpoints remain after completion until the operation is forgotten. +After a failure, `operations:run` starts `handle()` again. Your code reads the saved checkpoint to decide where to continue. Saving a checkpoint does not mark the operation complete or prevent concurrent execution. Checkpoints remain after completion until the operation is forgotten. You can also call `$this->checkpoints->forget()` to clear all of the operation's checkpoints without changing its completion record. In the example above, an interrupted chunk may run again. Write work that can safely repeat, or wrap each chunk's database changes and checkpoint write in a transaction on the same connection. Checkpoints use `operations.connection` and participate in its active transaction. With `WithinTransaction`, a failure rolls back all checkpoints written during that attempt along with the operation's database changes. External effects, such as API calls, are not rolled back. -The standard `operations-migrations` tag only publishes the operations table. Running, checking status, and forgetting operations work without the checkpoint table; `operations:forget` clears checkpoints when that table exists. Calling `checkpoint()` before installing its migration raises a database exception. +The standard `operations-migrations` tag only publishes the operations table. Running, checking status, and forgetting operations work without the checkpoint table; `operations:forget` clears checkpoints when that table exists. Reading or writing checkpoints before installing their migration raises a database exception. ### Transactions diff --git a/src/CheckpointRepository.php b/src/CheckpointRepository.php index 10ce71f..1693762 100644 --- a/src/CheckpointRepository.php +++ b/src/CheckpointRepository.php @@ -31,10 +31,12 @@ public function get(string $key, mixed $default = null): mixed /** * Persist checkpoint values without replacing other keys. * - * @param array $values + * @param string|array $key */ - public function save(array $values): void + public function put(string|array $key, mixed $value = null): void { + $values = is_array($key) ? $key : [$key => $value]; + $checkpoints = []; foreach ($values as $key => $value) { diff --git a/src/Concerns/HasCheckpoints.php b/src/Concerns/HasCheckpoints.php index fbefd0e..df0b6f5 100644 --- a/src/Concerns/HasCheckpoints.php +++ b/src/Concerns/HasCheckpoints.php @@ -11,22 +11,6 @@ trait HasCheckpoints */ protected CheckpointRepository $checkpoints; - /** - * Read a checkpoint or persist an array of checkpoint values. - * - * @param string|array $key - */ - public function checkpoint(string|array $key, mixed $default = null): mixed - { - if (is_array($key)) { - $this->checkpoints->save($key); - - return null; - } - - return $this->checkpoints->get($key, $default); - } - /** * Set the operation's checkpoint repository. */ diff --git a/tests/Feature/CheckpointTest.php b/tests/Feature/CheckpointTest.php index e51eafa..f654e6a 100644 --- a/tests/Feature/CheckpointTest.php +++ b/tests/Feature/CheckpointTest.php @@ -58,9 +58,9 @@ public function handle(\Illuminate\Console\Command $command): void config(['operations.connection' => null]); })->with(['default connection' => false, 'separate connection' => true]); -test('using checkpoints without their migration raises a database exception', function (string|array $argument) { +test('using checkpoints without their migration raises a database exception', function (string $method) { Schema::drop('operation_checkpoints'); - app()->instance('checkpoint_argument', $argument); + app()->instance('checkpoint_method', $method); File::ensureDirectoryExists(config('operations.path')); File::put(config('operations.path').'/2026_10_05_120000_example.php', <<<'PHP' @@ -71,7 +71,7 @@ public function handle(\Illuminate\Console\Command $command): void public function handle(\Illuminate\Console\Command $command): void { - $this->checkpoint(app('checkpoint_argument')); + $this->checkpoints->{app('checkpoint_method')}('cursor', 'next-page'); } }; PHP); @@ -80,11 +80,11 @@ public function handle(\Illuminate\Console\Command $command): void expect(DB::table('operations')->count())->toBe(0); })->with([ - 'read' => ['cursor'], - 'write' => [['cursor' => 'next-page']], + 'read' => 'get', + 'write' => 'put', ]); -test('checkpoints read defaults and persist JSON values while preserving other keys', function () { +test('checkpoints read defaults and persist JSON values while preserving other keys', function (bool $multiple) { $values = [ 'cursor' => 'next-page', 'count' => 0, @@ -97,6 +97,7 @@ public function handle(\Illuminate\Console\Command $command): void ]; app()->instance('checkpoint_values', $values); + app()->instance('multiple', $multiple); app()->instance('observed', collect()); File::ensureDirectoryExists(config('operations.path')); @@ -108,15 +109,22 @@ public function handle(\Illuminate\Console\Command $command): void public function handle(\Illuminate\Console\Command $command): void { - app('observed')->push($this->checkpoint('missing')); - app('observed')->push($this->checkpoint('missing', 42)); + app('observed')->push($this->checkpoints->get('missing')); + app('observed')->push($this->checkpoints->get('missing', 42)); + + if (app('multiple')) { + $this->checkpoints->put(app('checkpoint_values')); + } else { + foreach (app('checkpoint_values') as $key => $value) { + $this->checkpoints->put($key, $value); + } + } - $this->checkpoint(app('checkpoint_values')); - $this->checkpoint(['cursor' => 'last-page']); - $this->checkpoint([]); + $this->checkpoints->put('cursor', 'last-page'); + $this->checkpoints->put([]); foreach (app('checkpoint_values') as $key => $value) { - app('observed')->push($this->checkpoint($key, 'missing')); + app('observed')->push($this->checkpoints->get($key, 'missing')); } } }; @@ -129,7 +137,7 @@ public function handle(\Illuminate\Console\Command $command): void expect(app('observed')->all())->toBe([null, 42, ...array_values($values)]) ->and(DB::table('operation_checkpoints')->count())->toBe(count($values)) ->and(DB::table('operation_checkpoints')->where('name', 'missing')->exists())->toBeFalse(); -}); +})->with(['multiple values' => true, 'single values' => false]); test('a failed operation resumes from its saved checkpoint on the next run', function () { Schema::create('examples', function (Blueprint $table) { @@ -147,13 +155,13 @@ public function handle(\Illuminate\Console\Command $command): void public function handle(\Illuminate\Console\Command $command): void { - for ($id = $this->checkpoint('last_id', 0) + 1; $id <= 3; $id++) { + for ($id = $this->checkpoints->get('last_id', 0) + 1; $id <= 3; $id++) { if ($id === 3 && app('should_fail')) { throw new \RuntimeException('Backfill interrupted.'); } \Illuminate\Support\Facades\DB::table('examples')->insert(['id' => $id]); - $this->checkpoint(['last_id' => $id]); + $this->checkpoints->put('last_id', $id); } } }; @@ -189,9 +197,9 @@ public function handle(\Illuminate\Console\Command $command): void public function handle(\Illuminate\Console\Command $command): void { - app('observed')->push($this->checkpoint('cursor')); - $this->checkpoint(['cursor' => basename(__FILE__, '.php')]); - app('observed')->push($this->checkpoint('cursor')); + app('observed')->push($this->checkpoints->get('cursor')); + $this->checkpoints->put('cursor', basename(__FILE__, '.php')); + app('observed')->push($this->checkpoints->get('cursor')); } }; PHP); @@ -222,7 +230,7 @@ public function handle(\Illuminate\Console\Command $command): void foreach ([1, 2] as $id) { \Illuminate\Support\Facades\DB::transaction(function () use ($id) { \Illuminate\Support\Facades\DB::table('examples')->insert(['id' => $id]); - $this->checkpoint(['last_id' => $id]); + $this->checkpoints->put('last_id', $id); if ($id === 2) { throw new \RuntimeException('Chunk failed.'); @@ -258,7 +266,7 @@ public function handle(\Illuminate\Console\Command $command): void public function handle(\Illuminate\Console\Command $command): void { - $this->checkpoint(['last_id' => 20, 'finished' => true]); + $this->checkpoints->put(['last_id' => 20, 'finished' => true]); throw new \RuntimeException('Backfill failed.'); } @@ -282,8 +290,8 @@ public function handle(\Illuminate\Console\Command $command): void public function handle(\Illuminate\Console\Command $command): void { - $this->checkpoint(['last_id' => 10]); - $this->checkpoint(['last_id' => 20, 'invalid' => NAN]); + $this->checkpoints->put('last_id', 10); + $this->checkpoints->put(['last_id' => 20, 'invalid' => NAN]); } }; PHP); @@ -327,9 +335,9 @@ public function handle(\Illuminate\Console\Command $command): void public function handle(\Illuminate\Console\Command $command): void { - app('observed')->push($this->checkpoint('cursor', 'initial')); - $this->checkpoint(['cursor' => 'next-page']); - app('observed')->push($this->checkpoint('cursor')); + app('observed')->push($this->checkpoints->get('cursor', 'initial')); + $this->checkpoints->put('cursor', 'next-page'); + app('observed')->push($this->checkpoints->get('cursor')); } }; PHP); diff --git a/tests/Feature/ForgetTest.php b/tests/Feature/ForgetTest.php index c972be0..dab0a63 100644 --- a/tests/Feature/ForgetTest.php +++ b/tests/Feature/ForgetTest.php @@ -175,7 +175,7 @@ public function handle(\Illuminate\Console\Command $command): void public function handle(\Illuminate\Console\Command $command): void { - app('observed')->push($this->checkpoint('last_id', 0)); + app('observed')->push($this->checkpoints->get('last_id', 0)); } }; PHP); From 1d5892e8390feb0f8c209acde3838ebcce70ba14 Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 12:13:30 -0400 Subject: [PATCH 09/14] Handle optional checkpoint storage in the operation repository --- README.md | 2 +- src/CheckpointRepository.php | 6 +----- src/OperationRepository.php | 3 ++- tests/Feature/CheckpointTest.php | 10 ++++++---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 2de2c07..a50109d 100644 --- a/README.md +++ b/README.md @@ -280,7 +280,7 @@ After a failure, `operations:run` starts `handle()` again. Your code reads the s In the example above, an interrupted chunk may run again. Write work that can safely repeat, or wrap each chunk's database changes and checkpoint write in a transaction on the same connection. Checkpoints use `operations.connection` and participate in its active transaction. With `WithinTransaction`, a failure rolls back all checkpoints written during that attempt along with the operation's database changes. External effects, such as API calls, are not rolled back. -The standard `operations-migrations` tag only publishes the operations table. Running, checking status, and forgetting operations work without the checkpoint table; `operations:forget` clears checkpoints when that table exists. Reading or writing checkpoints before installing their migration raises a database exception. +The standard `operations-migrations` tag only publishes the operations table. Running, checking status, and forgetting operations work without the checkpoint table; `operations:forget` clears checkpoints when that table exists. Calling the checkpoint repository's `get()`, `put()`, or `forget()` methods before installing its migration raises a database exception. ### Transactions diff --git a/src/CheckpointRepository.php b/src/CheckpointRepository.php index 1693762..c2d9d3d 100644 --- a/src/CheckpointRepository.php +++ b/src/CheckpointRepository.php @@ -52,14 +52,10 @@ public function put(string|array $key, mixed $value = null): void } /** - * Delete an operation's checkpoints when checkpoint storage is installed. + * Delete the operation's checkpoints. */ public function forget(): bool { - if (! $this->connection->getSchemaBuilder()->hasTable('operation_checkpoints')) { - return false; - } - return $this->connection->table('operation_checkpoints') ->where('operation', $this->name) ->delete() > 0; diff --git a/src/OperationRepository.php b/src/OperationRepository.php index 198ee8e..3a81fb5 100644 --- a/src/OperationRepository.php +++ b/src/OperationRepository.php @@ -39,7 +39,8 @@ public function forget(string $name): bool ->where('name', $name) ->delete(); - $forgotten = $this->checkpoints($name)->forget(); + $forgotten = $this->connection->getSchemaBuilder()->hasTable('operation_checkpoints') + && $this->checkpoints($name)->forget(); return $deleted > 0 || $forgotten; }); diff --git a/tests/Feature/CheckpointTest.php b/tests/Feature/CheckpointTest.php index f654e6a..b9306cd 100644 --- a/tests/Feature/CheckpointTest.php +++ b/tests/Feature/CheckpointTest.php @@ -58,9 +58,10 @@ public function handle(\Illuminate\Console\Command $command): void config(['operations.connection' => null]); })->with(['default connection' => false, 'separate connection' => true]); -test('using checkpoints without their migration raises a database exception', function (string $method) { +test('using checkpoints without their migration raises a database exception', function (string $method, array $arguments) { Schema::drop('operation_checkpoints'); app()->instance('checkpoint_method', $method); + app()->instance('checkpoint_arguments', $arguments); File::ensureDirectoryExists(config('operations.path')); File::put(config('operations.path').'/2026_10_05_120000_example.php', <<<'PHP' @@ -71,7 +72,7 @@ public function handle(\Illuminate\Console\Command $command): void public function handle(\Illuminate\Console\Command $command): void { - $this->checkpoints->{app('checkpoint_method')}('cursor', 'next-page'); + $this->checkpoints->{app('checkpoint_method')}(...app('checkpoint_arguments')); } }; PHP); @@ -80,8 +81,9 @@ public function handle(\Illuminate\Console\Command $command): void expect(DB::table('operations')->count())->toBe(0); })->with([ - 'read' => 'get', - 'write' => 'put', + 'read' => ['get', ['cursor']], + 'write' => ['put', ['cursor', 'next-page']], + 'forget' => ['forget', []], ]); test('checkpoints read defaults and persist JSON values while preserving other keys', function (bool $multiple) { From 999443994a8432851094d5a657b9156f1521f0bf Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 12:14:31 -0400 Subject: [PATCH 10/14] Rename checkpoint repository operation property --- src/CheckpointRepository.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/CheckpointRepository.php b/src/CheckpointRepository.php index c2d9d3d..964d63a 100644 --- a/src/CheckpointRepository.php +++ b/src/CheckpointRepository.php @@ -11,7 +11,7 @@ class CheckpointRepository */ public function __construct( protected ConnectionInterface $connection, - protected string $name + protected string $operation ) {} /** @@ -21,7 +21,7 @@ public function get(string $key, mixed $default = null): mixed { $value = $this->connection->table('operation_checkpoints') ->useWritePdo() - ->where('operation', $this->name) + ->where('operation', $this->operation) ->where('name', $key) ->value('value'); @@ -41,7 +41,7 @@ public function put(string|array $key, mixed $value = null): void foreach ($values as $key => $value) { $checkpoints[] = [ - 'operation' => $this->name, + 'operation' => $this->operation, 'name' => $key, 'value' => json_encode($value, JSON_THROW_ON_ERROR), ]; @@ -57,7 +57,7 @@ public function put(string|array $key, mixed $value = null): void public function forget(): bool { return $this->connection->table('operation_checkpoints') - ->where('operation', $this->name) + ->where('operation', $this->operation) ->delete() > 0; } } From 49eb6f742dded59de414ffbdc4eb1a0ed1c2ae62 Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 12:15:11 -0400 Subject: [PATCH 11/14] Rename checkpoint name column to key --- ..._05_041651_create_operation_checkpoints_table.php | 4 ++-- src/CheckpointRepository.php | 6 +++--- tests/Feature/CheckpointTest.php | 6 +++--- tests/Feature/ForgetTest.php | 12 ++++++------ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php b/database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php index 9919f13..db05912 100644 --- a/database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php +++ b/database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php @@ -14,9 +14,9 @@ public function up(): void Schema::connection(config('operations.connection'))->create('operation_checkpoints', function (Blueprint $table) { $table->id(); $table->string('operation'); - $table->string('name'); + $table->string('key'); $table->json('value'); - $table->unique(['operation', 'name']); + $table->unique(['operation', 'key']); }); } diff --git a/src/CheckpointRepository.php b/src/CheckpointRepository.php index 964d63a..7201c13 100644 --- a/src/CheckpointRepository.php +++ b/src/CheckpointRepository.php @@ -22,7 +22,7 @@ public function get(string $key, mixed $default = null): mixed $value = $this->connection->table('operation_checkpoints') ->useWritePdo() ->where('operation', $this->operation) - ->where('name', $key) + ->where('key', $key) ->value('value'); return $value === null ? value($default) : json_decode($value, true, flags: JSON_THROW_ON_ERROR); @@ -42,13 +42,13 @@ public function put(string|array $key, mixed $value = null): void foreach ($values as $key => $value) { $checkpoints[] = [ 'operation' => $this->operation, - 'name' => $key, + 'key' => $key, 'value' => json_encode($value, JSON_THROW_ON_ERROR), ]; } $this->connection->table('operation_checkpoints') - ->upsert($checkpoints, ['operation', 'name'], ['value']); + ->upsert($checkpoints, ['operation', 'key'], ['value']); } /** diff --git a/tests/Feature/CheckpointTest.php b/tests/Feature/CheckpointTest.php index b9306cd..30ea589 100644 --- a/tests/Feature/CheckpointTest.php +++ b/tests/Feature/CheckpointTest.php @@ -138,7 +138,7 @@ public function handle(\Illuminate\Console\Command $command): void expect(app('observed')->all())->toBe([null, 42, ...array_values($values)]) ->and(DB::table('operation_checkpoints')->count())->toBe(count($values)) - ->and(DB::table('operation_checkpoints')->where('name', 'missing')->exists())->toBeFalse(); + ->and(DB::table('operation_checkpoints')->where('key', 'missing')->exists())->toBeFalse(); })->with(['multiple values' => true, 'single values' => false]); test('a failed operation resumes from its saved checkpoint on the next run', function () { @@ -185,7 +185,7 @@ public function handle(\Illuminate\Console\Command $command): void ->and(DB::table('operations')->count())->toBe(1); }); -test('checkpoint names are scoped to the operation filename', function () { +test('checkpoint keys are scoped to the operation filename', function () { app()->instance('observed', collect()); File::ensureDirectoryExists(config('operations.path')); @@ -255,7 +255,7 @@ public function handle(\Illuminate\Console\Command $command): void DB::table('operation_checkpoints')->insert([ 'operation' => $name, - 'name' => 'last_id', + 'key' => 'last_id', 'value' => '10', ]); diff --git a/tests/Feature/ForgetTest.php b/tests/Feature/ForgetTest.php index dab0a63..9ce74e2 100644 --- a/tests/Feature/ForgetTest.php +++ b/tests/Feature/ForgetTest.php @@ -71,7 +71,7 @@ public function handle(\Illuminate\Console\Command $command): void DB::table('operations')->insert(['name' => $name, 'completed_at' => now()]); DB::table('operation_checkpoints')->insert([ 'operation' => $name, - 'name' => 'last_id', + 'key' => 'last_id', 'value' => '10', ]); @@ -139,8 +139,8 @@ public function handle(\Illuminate\Console\Command $command): void $name = '2026_10_02_120000_example'; DB::table('operations')->insert(['name' => $name, 'completed_at' => now()]); DB::connection('operations')->table('operations')->insert(['name' => $name, 'completed_at' => now()]); - DB::table('operation_checkpoints')->insert(['operation' => $name, 'name' => 'last_id', 'value' => '10']); - DB::connection('operations')->table('operation_checkpoints')->insert(['operation' => $name, 'name' => 'last_id', 'value' => '20']); + DB::table('operation_checkpoints')->insert(['operation' => $name, 'key' => 'last_id', 'value' => '10']); + DB::connection('operations')->table('operation_checkpoints')->insert(['operation' => $name, 'key' => 'last_id', 'value' => '20']); artisan('operations:forget', ['name' => $name, '--force' => true])->assertSuccessful(); @@ -159,9 +159,9 @@ public function handle(\Illuminate\Console\Command $command): void } DB::table('operation_checkpoints')->insert([ - ['operation' => $name, 'name' => 'last_id', 'value' => '10'], - ['operation' => $name, 'name' => 'finished', 'value' => 'true'], - ['operation' => $other, 'name' => 'last_id', 'value' => '20'], + ['operation' => $name, 'key' => 'last_id', 'value' => '10'], + ['operation' => $name, 'key' => 'finished', 'value' => 'true'], + ['operation' => $other, 'key' => 'last_id', 'value' => '20'], ]); app()->instance('observed', collect()); From 69e3b49f3c8abc03e2a2c5b05d4cc078502a9cd3 Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 12:19:33 -0400 Subject: [PATCH 12/14] Use operation fixtures throughout the test suite --- tests/Feature/CheckpointTest.php | 181 ++++-------------- tests/Feature/ConnectionTest.php | 39 ++-- tests/Feature/ForgetTest.php | 37 ++-- tests/Feature/IsolationTest.php | 31 ++- tests/Feature/OutputTest.php | 68 ++----- tests/Feature/RunSelectedTest.php | 24 +-- tests/Feature/RunTest.php | 92 +++------ tests/Feature/StatusTest.php | 5 +- .../operations/checkpoint_connection.php | 17 ++ .../Fixtures/operations/checkpoint_method.php | 15 ++ .../Fixtures/operations/checkpoint_values.php | 31 +++ tests/Fixtures/operations/console_output.php | 23 +++ tests/Fixtures/operations/dispatch_job.php | 14 ++ .../operations/duplicate_completion.php | 18 ++ tests/Fixtures/operations/empty_operation.php | 9 + .../operations/failing_checkpoint_chunk.php | 25 +++ .../failing_checkpoint_transaction.php | 18 ++ tests/Fixtures/operations/failing_output.php | 24 +++ .../operations/failing_transaction.php | 16 ++ .../operations/invalid_checkpoint_value.php | 16 ++ .../Fixtures/operations/invalid_operation.php | 3 + tests/Fixtures/operations/quiet_output.php | 18 ++ tests/Fixtures/operations/read_checkpoint.php | 15 ++ .../Fixtures/operations/record_execution.php | 12 ++ .../operations/record_execution_order.php | 14 ++ .../operations/resumable_backfill.php | 23 +++ .../operations/resumable_operations.php | 17 ++ .../operations/scoped_checkpoints.php | 17 ++ .../operations/selected_operation.php | 12 ++ .../operations/successful_operation.php | 12 ++ .../operations/successful_transaction.php | 14 ++ tests/Fixtures/operations/throw_on_load.php | 3 + 32 files changed, 515 insertions(+), 348 deletions(-) create mode 100644 tests/Fixtures/operations/checkpoint_connection.php create mode 100644 tests/Fixtures/operations/checkpoint_method.php create mode 100644 tests/Fixtures/operations/checkpoint_values.php create mode 100644 tests/Fixtures/operations/console_output.php create mode 100644 tests/Fixtures/operations/dispatch_job.php create mode 100644 tests/Fixtures/operations/duplicate_completion.php create mode 100644 tests/Fixtures/operations/empty_operation.php create mode 100644 tests/Fixtures/operations/failing_checkpoint_chunk.php create mode 100644 tests/Fixtures/operations/failing_checkpoint_transaction.php create mode 100644 tests/Fixtures/operations/failing_output.php create mode 100644 tests/Fixtures/operations/failing_transaction.php create mode 100644 tests/Fixtures/operations/invalid_checkpoint_value.php create mode 100644 tests/Fixtures/operations/invalid_operation.php create mode 100644 tests/Fixtures/operations/quiet_output.php create mode 100644 tests/Fixtures/operations/read_checkpoint.php create mode 100644 tests/Fixtures/operations/record_execution.php create mode 100644 tests/Fixtures/operations/record_execution_order.php create mode 100644 tests/Fixtures/operations/resumable_backfill.php create mode 100644 tests/Fixtures/operations/resumable_operations.php create mode 100644 tests/Fixtures/operations/scoped_checkpoints.php create mode 100644 tests/Fixtures/operations/selected_operation.php create mode 100644 tests/Fixtures/operations/successful_operation.php create mode 100644 tests/Fixtures/operations/successful_transaction.php create mode 100644 tests/Fixtures/operations/throw_on_load.php diff --git a/tests/Feature/CheckpointTest.php b/tests/Feature/CheckpointTest.php index 30ea589..491780e 100644 --- a/tests/Feature/CheckpointTest.php +++ b/tests/Feature/CheckpointTest.php @@ -28,16 +28,10 @@ $name = '2026_10_05_120000_example'; File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path')."/{$name}.php", <<<'PHP' - info('Operation executed.'); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/successful_operation.php', + config('operations.path')."/{$name}.php", + ); artisan('operations:run')->expectsOutputToContain('Operation executed.')->assertSuccessful(); artisan('operations:run')->expectsOutputToContain('No pending operations.')->assertSuccessful(); @@ -64,18 +58,10 @@ public function handle(\Illuminate\Console\Command $command): void app()->instance('checkpoint_arguments', $arguments); File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_05_120000_example.php', <<<'PHP' - checkpoints->{app('checkpoint_method')}(...app('checkpoint_arguments')); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/checkpoint_method.php', + config('operations.path').'/2026_10_05_120000_example.php', + ); expect(fn () => artisan('operations:run')->run())->toThrow(QueryException::class, 'operation_checkpoints'); @@ -103,34 +89,10 @@ public function handle(\Illuminate\Console\Command $command): void app()->instance('observed', collect()); File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_05_120000_example.php', <<<'PHP' - push($this->checkpoints->get('missing')); - app('observed')->push($this->checkpoints->get('missing', 42)); - - if (app('multiple')) { - $this->checkpoints->put(app('checkpoint_values')); - } else { - foreach (app('checkpoint_values') as $key => $value) { - $this->checkpoints->put($key, $value); - } - } - - $this->checkpoints->put('cursor', 'last-page'); - $this->checkpoints->put([]); - - foreach (app('checkpoint_values') as $key => $value) { - app('observed')->push($this->checkpoints->get($key, 'missing')); - } - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/checkpoint_values.php', + config('operations.path').'/2026_10_05_120000_example.php', + ); artisan('operations:run')->assertSuccessful(); @@ -149,25 +111,10 @@ public function handle(\Illuminate\Console\Command $command): void app()->instance('should_fail', true); File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_05_120000_backfill.php', <<<'PHP' - checkpoints->get('last_id', 0) + 1; $id <= 3; $id++) { - if ($id === 3 && app('should_fail')) { - throw new \RuntimeException('Backfill interrupted.'); - } - - \Illuminate\Support\Facades\DB::table('examples')->insert(['id' => $id]); - $this->checkpoints->put('last_id', $id); - } - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/resumable_backfill.php', + config('operations.path').'/2026_10_05_120000_backfill.php', + ); expect(fn () => artisan('operations:run')->run())->toThrow(RuntimeException::class, 'Backfill interrupted.'); @@ -191,20 +138,10 @@ public function handle(\Illuminate\Console\Command $command): void File::ensureDirectoryExists(config('operations.path')); foreach (['2026_10_05_120000_first', '2026_10_05_120001_second'] as $name) { - File::put(config('operations.path')."/{$name}.php", <<<'PHP' - push($this->checkpoints->get('cursor')); - $this->checkpoints->put('cursor', basename(__FILE__, '.php')); - app('observed')->push($this->checkpoints->get('cursor')); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/scoped_checkpoints.php', + config('operations.path')."/{$name}.php", + ); } artisan('operations:run')->assertSuccessful(); @@ -221,27 +158,10 @@ public function handle(\Illuminate\Console\Command $command): void }); File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_05_120000_backfill.php', <<<'PHP' - insert(['id' => $id]); - $this->checkpoints->put('last_id', $id); - - if ($id === 2) { - throw new \RuntimeException('Chunk failed.'); - } - }); - } - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/failing_checkpoint_chunk.php', + config('operations.path').'/2026_10_05_120000_backfill.php', + ); expect(fn () => artisan('operations:run')->run())->toThrow(RuntimeException::class, 'Chunk failed.'); @@ -260,20 +180,10 @@ public function handle(\Illuminate\Console\Command $command): void ]); File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path')."/{$name}.php", <<<'PHP' - checkpoints->put(['last_id' => 20, 'finished' => true]); - - throw new \RuntimeException('Backfill failed.'); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/failing_checkpoint_transaction.php', + config('operations.path')."/{$name}.php", + ); expect(fn () => artisan('operations:run')->run())->toThrow(RuntimeException::class, 'Backfill failed.'); @@ -284,19 +194,10 @@ public function handle(\Illuminate\Console\Command $command): void test('invalid JSON values fail before any checkpoints are changed', function () { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_05_120000_example.php', <<<'PHP' - checkpoints->put('last_id', 10); - $this->checkpoints->put(['last_id' => 20, 'invalid' => NAN]); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/invalid_checkpoint_value.php', + config('operations.path').'/2026_10_05_120000_example.php', + ); expect(fn () => artisan('operations:run')->run())->toThrow(JsonException::class); @@ -329,20 +230,10 @@ public function handle(\Illuminate\Console\Command $command): void app()->instance('observed', collect()); - File::put(config('operations.path').'/2026_10_05_120000_example.php', <<<'PHP' - push($this->checkpoints->get('cursor', 'initial')); - $this->checkpoints->put('cursor', 'next-page'); - app('observed')->push($this->checkpoints->get('cursor')); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/checkpoint_connection.php', + config('operations.path').'/2026_10_05_120000_example.php', + ); artisan('operations:run')->assertSuccessful(); diff --git a/tests/Feature/ConnectionTest.php b/tests/Feature/ConnectionTest.php index 187b2d2..e6ab812 100644 --- a/tests/Feature/ConnectionTest.php +++ b/tests/Feature/ConnectionTest.php @@ -21,13 +21,10 @@ $migration->up(); File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_example.php', <<<'PHP' - assertSuccessful(); artisan('operations:run')->expectsOutputToContain('No pending operations.')->assertSuccessful(); @@ -49,16 +46,10 @@ public function handle(\Illuminate\Console\Command $command): void {} }); File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_backfill.php', <<<'PHP' - insert(['id' => 1]); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/successful_transaction.php', + config('operations.path').'/2026_10_02_120000_backfill.php', + ); artisan('operations:run')->assertSuccessful(); @@ -88,16 +79,10 @@ public function handle(\Illuminate\Console\Command $command): void $migration = File::getRequire(__DIR__.'/../../database/migrations/2026_10_02_165413_create_operations_table.php'); $migration->up(); - File::put(config('operations.path').'/2026_10_02_120000_example.php', <<<'PHP' - push('example'); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/record_execution.php', + config('operations.path').'/2026_10_02_120000_example.php', + ); app()->instance('executed', collect()); diff --git a/tests/Feature/ForgetTest.php b/tests/Feature/ForgetTest.php index 9ce74e2..bfbc916 100644 --- a/tests/Feature/ForgetTest.php +++ b/tests/Feature/ForgetTest.php @@ -18,7 +18,10 @@ File::ensureDirectoryExists(config('operations.path')); $path = config('operations.path')."/{$name}.php"; - File::put($path, ' $name]) ->expectsOutputToContain('Forgetting does not undo previous effects.') @@ -27,23 +30,17 @@ ->assertSuccessful(); expect(DB::table('operations')->pluck('name')->all())->toBe([$other]) - ->and(File::get($path))->toBe('and(File::get($path))->toBe(File::get(__DIR__.'/../Fixtures/operations/throw_on_load.php')); }); test('a forgotten operation runs again only when the runner is invoked', function () { $name = '2026_10_02_120000_example'; File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path')."/{$name}.php", <<<'PHP' - push('example'); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/record_execution.php', + config('operations.path')."/{$name}.php", + ); app()->instance('executed', collect()); @@ -167,18 +164,10 @@ public function handle(\Illuminate\Console\Command $command): void app()->instance('observed', collect()); File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path')."/{$name}.php", <<<'PHP' - push($this->checkpoints->get('last_id', 0)); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/read_checkpoint.php', + config('operations.path')."/{$name}.php", + ); artisan('operations:forget', ['name' => $name, '--force' => true])->assertSuccessful(); diff --git a/tests/Feature/IsolationTest.php b/tests/Feature/IsolationTest.php index e7e92bb..237459e 100644 --- a/tests/Feature/IsolationTest.php +++ b/tests/Feature/IsolationTest.php @@ -9,11 +9,10 @@ test('isolated invocations skip pending operations when another command holds the lock', function (bool|string $isolated, int $exitCode) { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_backfill.php', <<<'PHP' - true]) ->expectsOutputToContain('Completed 1 operation(s).') diff --git a/tests/Feature/OutputTest.php b/tests/Feature/OutputTest.php index 72ce7be..a4116b6 100644 --- a/tests/Feature/OutputTest.php +++ b/tests/Feature/OutputTest.php @@ -8,26 +8,10 @@ test('operations can print messages tables and progress between their start and completion lines', function () { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_output.php', <<<'PHP' - info('Backfilling company names...'); - $command->table(['Updated'], [[2]]); - - $output = $command->getOutput(); - $output->progressStart(2); - $output->progressAdvance(2); - $output->progressFinish(); - - $command->info('Company names updated.'); - app()->instance('forced', $command->option('force')); - app()->instance('completed_during_operation', \Illuminate\Support\Facades\DB::table('operations')->count()); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/console_output.php', + config('operations.path').'/2026_10_02_120000_output.php', + ); $output = new BufferedOutput; $exitCode = Artisan::call('operations:run', ['--force' => true], $output); @@ -45,22 +29,10 @@ public function handle(\Illuminate\Console\Command $command): void test('quiet mode suppresses both operation output and runner status lines', function () { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_quiet.php', <<<'PHP' - info('Backfilling company names...'); - $command->table(['Updated'], [[1]]); - - $output = $command->getOutput(); - $output->progressStart(1); - $output->progressAdvance(); - $output->progressFinish(); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/quiet_output.php', + config('operations.path').'/2026_10_02_120000_quiet.php', + ); $output = new BufferedOutput; $exitCode = Artisan::call('operations:run', ['--quiet' => true], $output); @@ -72,26 +44,10 @@ public function handle(\Illuminate\Console\Command $command): void test('failed operations never print a successful completion line', function (string $failure, string $exception) { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_failure.php', <<<'PHP' - info('Starting the backfill...'); - - if (app('failure') === 'operation') { - throw new \RuntimeException('Backfill failed.'); - } - - // Cause the runner's completion insert to fail on the unique name. - \Illuminate\Support\Facades\DB::table('operations')->insert([ - 'name' => basename(__FILE__, '.php'), - 'completed_at' => now(), - ]); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/failing_output.php', + config('operations.path').'/2026_10_02_120000_failure.php', + ); app()->instance('failure', $failure); $output = new BufferedOutput; diff --git a/tests/Feature/RunSelectedTest.php b/tests/Feature/RunSelectedTest.php index c33aff1..c96eed0 100644 --- a/tests/Feature/RunSelectedTest.php +++ b/tests/Feature/RunSelectedTest.php @@ -13,19 +13,16 @@ File::ensureDirectoryExists(config('operations.path')); foreach (['2026_10_03_110000_unrelated', '2026_10_03_130000_unrelated'] as $name) { - File::put(config('operations.path')."/{$name}.php", 'info('Selected operation executed.'); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/selected_operation.php', + config('operations.path').'/2026_10_03_120000_selected.php', + ); artisan('operations:run', ['operation' => '2026_10_03_120000_selected']) ->expectsOutputToContain('Selected operation executed.') @@ -49,7 +46,10 @@ public function handle(\Illuminate\Console\Command $command): void File::ensureDirectoryExists(config('operations.path')); foreach (['2026_10_03_120000_backfill', '2026_10_03_130000_backfill'] as $name) { - File::put(config('operations.path')."/{$name}.php", ' artisan('operations:run', [ diff --git a/tests/Feature/RunTest.php b/tests/Feature/RunTest.php index 1350f23..f9aaa7a 100644 --- a/tests/Feature/RunTest.php +++ b/tests/Feature/RunTest.php @@ -17,17 +17,10 @@ File::ensureDirectoryExists(config('operations.path')); foreach (['2026_10_02_120002_second', '2026_10_02_120001_first'] as $name) { - File::put(config('operations.path')."/{$name}.php", <<<'PHP' - push(basename(__FILE__, '.php')); - app('ledger_counts')->push(\Illuminate\Support\Facades\DB::table('operations')->count()); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/record_execution_order.php', + config('operations.path')."/{$name}.php", + ); } app()->instance('executed', collect()); @@ -45,21 +38,10 @@ public function handle(\Illuminate\Console\Command $command): void File::ensureDirectoryExists(config('operations.path')); foreach (['2026_10_02_120001_first', '2026_10_02_120002_second', '2026_10_02_120003_third'] as $name) { - File::put(config('operations.path')."/{$name}.php", <<<'PHP' - push($name); - - if (str_ends_with($name, 'second') && app('should_fail')) { - throw new \RuntimeException('Backfill failed.'); - } - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/resumable_operations.php', + config('operations.path')."/{$name}.php", + ); } app()->instance('attempts', collect()); @@ -87,18 +69,10 @@ public function handle(\Illuminate\Console\Command $command): void }); File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_backfill.php', <<<'PHP' - insert(['id' => 1]); - - throw new \RuntimeException('Backfill failed.'); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/failing_transaction.php', + config('operations.path').'/2026_10_02_120000_backfill.php', + ); expect(fn () => Artisan::call('operations:run', ['operation' => $operation]))->toThrow(RuntimeException::class, 'Backfill failed.'); @@ -112,20 +86,10 @@ public function handle(\Illuminate\Console\Command $command): void }); File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_backfill.php', <<<'PHP' - insert(['id' => 1]); - \Illuminate\Support\Facades\DB::table('operations')->insert([ - 'name' => basename(__FILE__, '.php'), - 'completed_at' => now(), - ]); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/duplicate_completion.php', + config('operations.path').'/2026_10_02_120000_backfill.php', + ); expect(fn () => Artisan::call('operations:run', ['operation' => $operation]))->toThrow(QueryException::class); @@ -137,16 +101,10 @@ public function handle(\Illuminate\Console\Command $command): void Bus::fake(); File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_dispatch.php', <<<'PHP' - assertSuccessful(); @@ -158,7 +116,10 @@ public function handle(\Illuminate\Console\Command $command): void test('completed operation files are not loaded on subsequent runs', function (?string $operation) { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_old.php', 'insert(['name' => '2026_10_02_120000_old', 'completed_at' => now()]); @@ -168,7 +129,10 @@ public function handle(\Illuminate\Console\Command $command): void test('invalid operation files fail without being recorded', function () { File::ensureDirectoryExists(config('operations.path')); $path = config('operations.path').'/2026_10_02_120000_invalid.php'; - File::put($path, ' app(Runner::class)->run(app(RunCommand::class)))->toThrow(UnexpectedValueException::class); diff --git a/tests/Feature/StatusTest.php b/tests/Feature/StatusTest.php index 8306f74..046400c 100644 --- a/tests/Feature/StatusTest.php +++ b/tests/Feature/StatusTest.php @@ -7,7 +7,10 @@ test('status includes pending operations and completed operations whose files were removed', function () { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120002_pending.php', 'insert([ 'name' => '2026_10_02_120001_completed', 'completed_at' => '2026-10-02 12:00:00', diff --git a/tests/Fixtures/operations/checkpoint_connection.php b/tests/Fixtures/operations/checkpoint_connection.php new file mode 100644 index 0000000..95798e4 --- /dev/null +++ b/tests/Fixtures/operations/checkpoint_connection.php @@ -0,0 +1,17 @@ +push($this->checkpoints->get('cursor', 'initial')); + $this->checkpoints->put('cursor', 'next-page'); + app('observed')->push($this->checkpoints->get('cursor')); + } +}; diff --git a/tests/Fixtures/operations/checkpoint_method.php b/tests/Fixtures/operations/checkpoint_method.php new file mode 100644 index 0000000..b91bc89 --- /dev/null +++ b/tests/Fixtures/operations/checkpoint_method.php @@ -0,0 +1,15 @@ +checkpoints->{app('checkpoint_method')}(...app('checkpoint_arguments')); + } +}; diff --git a/tests/Fixtures/operations/checkpoint_values.php b/tests/Fixtures/operations/checkpoint_values.php new file mode 100644 index 0000000..ff8b14d --- /dev/null +++ b/tests/Fixtures/operations/checkpoint_values.php @@ -0,0 +1,31 @@ +push($this->checkpoints->get('missing')); + app('observed')->push($this->checkpoints->get('missing', 42)); + + if (app('multiple')) { + $this->checkpoints->put(app('checkpoint_values')); + } else { + foreach (app('checkpoint_values') as $key => $value) { + $this->checkpoints->put($key, $value); + } + } + + $this->checkpoints->put('cursor', 'last-page'); + $this->checkpoints->put([]); + + foreach (app('checkpoint_values') as $key => $value) { + app('observed')->push($this->checkpoints->get($key, 'missing')); + } + } +}; diff --git a/tests/Fixtures/operations/console_output.php b/tests/Fixtures/operations/console_output.php new file mode 100644 index 0000000..633a0fb --- /dev/null +++ b/tests/Fixtures/operations/console_output.php @@ -0,0 +1,23 @@ +info('Backfilling company names...'); + $command->table(['Updated'], [[2]]); + + $output = $command->getOutput(); + $output->progressStart(2); + $output->progressAdvance(2); + $output->progressFinish(); + + $command->info('Company names updated.'); + app()->instance('forced', $command->option('force')); + app()->instance('completed_during_operation', DB::table('operations')->count()); + } +}; diff --git a/tests/Fixtures/operations/dispatch_job.php b/tests/Fixtures/operations/dispatch_job.php new file mode 100644 index 0000000..c193f7a --- /dev/null +++ b/tests/Fixtures/operations/dispatch_job.php @@ -0,0 +1,14 @@ +insert(['id' => 1]); + DB::table('operations')->insert([ + 'name' => basename(__FILE__, '.php'), + 'completed_at' => now(), + ]); + } +}; diff --git a/tests/Fixtures/operations/empty_operation.php b/tests/Fixtures/operations/empty_operation.php new file mode 100644 index 0000000..0652e3f --- /dev/null +++ b/tests/Fixtures/operations/empty_operation.php @@ -0,0 +1,9 @@ +insert(['id' => $id]); + $this->checkpoints->put('last_id', $id); + + if ($id === 2) { + throw new RuntimeException('Chunk failed.'); + } + }); + } + } +}; diff --git a/tests/Fixtures/operations/failing_checkpoint_transaction.php b/tests/Fixtures/operations/failing_checkpoint_transaction.php new file mode 100644 index 0000000..c2e86db --- /dev/null +++ b/tests/Fixtures/operations/failing_checkpoint_transaction.php @@ -0,0 +1,18 @@ +checkpoints->put(['last_id' => 20, 'finished' => true]); + + throw new RuntimeException('Backfill failed.'); + } +}; diff --git a/tests/Fixtures/operations/failing_output.php b/tests/Fixtures/operations/failing_output.php new file mode 100644 index 0000000..342d8d1 --- /dev/null +++ b/tests/Fixtures/operations/failing_output.php @@ -0,0 +1,24 @@ +info('Starting the backfill...'); + + if (app('failure') === 'operation') { + throw new RuntimeException('Backfill failed.'); + } + + // Cause the runner's completion insert to fail on the unique name. + DB::table('operations')->insert([ + 'name' => basename(__FILE__, '.php'), + 'completed_at' => now(), + ]); + } +}; diff --git a/tests/Fixtures/operations/failing_transaction.php b/tests/Fixtures/operations/failing_transaction.php new file mode 100644 index 0000000..58b368a --- /dev/null +++ b/tests/Fixtures/operations/failing_transaction.php @@ -0,0 +1,16 @@ +insert(['id' => 1]); + + throw new RuntimeException('Backfill failed.'); + } +}; diff --git a/tests/Fixtures/operations/invalid_checkpoint_value.php b/tests/Fixtures/operations/invalid_checkpoint_value.php new file mode 100644 index 0000000..b052c4a --- /dev/null +++ b/tests/Fixtures/operations/invalid_checkpoint_value.php @@ -0,0 +1,16 @@ +checkpoints->put('last_id', 10); + $this->checkpoints->put(['last_id' => 20, 'invalid' => NAN]); + } +}; diff --git a/tests/Fixtures/operations/invalid_operation.php b/tests/Fixtures/operations/invalid_operation.php new file mode 100644 index 0000000..13436bc --- /dev/null +++ b/tests/Fixtures/operations/invalid_operation.php @@ -0,0 +1,3 @@ +info('Backfilling company names...'); + $command->table(['Updated'], [[1]]); + + $output = $command->getOutput(); + $output->progressStart(1); + $output->progressAdvance(); + $output->progressFinish(); + } +}; diff --git a/tests/Fixtures/operations/read_checkpoint.php b/tests/Fixtures/operations/read_checkpoint.php new file mode 100644 index 0000000..fad1852 --- /dev/null +++ b/tests/Fixtures/operations/read_checkpoint.php @@ -0,0 +1,15 @@ +push($this->checkpoints->get('last_id', 0)); + } +}; diff --git a/tests/Fixtures/operations/record_execution.php b/tests/Fixtures/operations/record_execution.php new file mode 100644 index 0000000..2971b89 --- /dev/null +++ b/tests/Fixtures/operations/record_execution.php @@ -0,0 +1,12 @@ +push('example'); + } +}; diff --git a/tests/Fixtures/operations/record_execution_order.php b/tests/Fixtures/operations/record_execution_order.php new file mode 100644 index 0000000..a66b366 --- /dev/null +++ b/tests/Fixtures/operations/record_execution_order.php @@ -0,0 +1,14 @@ +push(basename(__FILE__, '.php')); + app('ledger_counts')->push(DB::table('operations')->count()); + } +}; diff --git a/tests/Fixtures/operations/resumable_backfill.php b/tests/Fixtures/operations/resumable_backfill.php new file mode 100644 index 0000000..a9ca6c5 --- /dev/null +++ b/tests/Fixtures/operations/resumable_backfill.php @@ -0,0 +1,23 @@ +checkpoints->get('last_id', 0) + 1; $id <= 3; $id++) { + if ($id === 3 && app('should_fail')) { + throw new RuntimeException('Backfill interrupted.'); + } + + DB::table('examples')->insert(['id' => $id]); + $this->checkpoints->put('last_id', $id); + } + } +}; diff --git a/tests/Fixtures/operations/resumable_operations.php b/tests/Fixtures/operations/resumable_operations.php new file mode 100644 index 0000000..a4ad388 --- /dev/null +++ b/tests/Fixtures/operations/resumable_operations.php @@ -0,0 +1,17 @@ +push($name); + + if (str_ends_with($name, 'second') && app('should_fail')) { + throw new RuntimeException('Backfill failed.'); + } + } +}; diff --git a/tests/Fixtures/operations/scoped_checkpoints.php b/tests/Fixtures/operations/scoped_checkpoints.php new file mode 100644 index 0000000..2ba084d --- /dev/null +++ b/tests/Fixtures/operations/scoped_checkpoints.php @@ -0,0 +1,17 @@ +push($this->checkpoints->get('cursor')); + $this->checkpoints->put('cursor', basename(__FILE__, '.php')); + app('observed')->push($this->checkpoints->get('cursor')); + } +}; diff --git a/tests/Fixtures/operations/selected_operation.php b/tests/Fixtures/operations/selected_operation.php new file mode 100644 index 0000000..850c784 --- /dev/null +++ b/tests/Fixtures/operations/selected_operation.php @@ -0,0 +1,12 @@ +info('Selected operation executed.'); + } +}; diff --git a/tests/Fixtures/operations/successful_operation.php b/tests/Fixtures/operations/successful_operation.php new file mode 100644 index 0000000..d38e47f --- /dev/null +++ b/tests/Fixtures/operations/successful_operation.php @@ -0,0 +1,12 @@ +info('Operation executed.'); + } +}; diff --git a/tests/Fixtures/operations/successful_transaction.php b/tests/Fixtures/operations/successful_transaction.php new file mode 100644 index 0000000..5b1c8b6 --- /dev/null +++ b/tests/Fixtures/operations/successful_transaction.php @@ -0,0 +1,14 @@ +insert(['id' => 1]); + } +}; diff --git a/tests/Fixtures/operations/throw_on_load.php b/tests/Fixtures/operations/throw_on_load.php new file mode 100644 index 0000000..aa9a765 --- /dev/null +++ b/tests/Fixtures/operations/throw_on_load.php @@ -0,0 +1,3 @@ + Date: Mon, 5 Oct 2026 12:24:15 -0400 Subject: [PATCH 13/14] Expose checkpoint storage installation status --- src/CheckpointRepository.php | 9 +++++++++ src/OperationRepository.php | 5 +++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/CheckpointRepository.php b/src/CheckpointRepository.php index 7201c13..61b73a7 100644 --- a/src/CheckpointRepository.php +++ b/src/CheckpointRepository.php @@ -51,6 +51,15 @@ public function put(string|array $key, mixed $value = null): void ->upsert($checkpoints, ['operation', 'key'], ['value']); } + /** + * Determine whether the checkpoint table is installed. + */ + public function installed(): bool + { + return $this->connection->getSchemaBuilder() + ->hasTable('operation_checkpoints'); + } + /** * Delete the operation's checkpoints. */ diff --git a/src/OperationRepository.php b/src/OperationRepository.php index 3a81fb5..601c231 100644 --- a/src/OperationRepository.php +++ b/src/OperationRepository.php @@ -39,8 +39,9 @@ public function forget(string $name): bool ->where('name', $name) ->delete(); - $forgotten = $this->connection->getSchemaBuilder()->hasTable('operation_checkpoints') - && $this->checkpoints($name)->forget(); + $checkpoints = $this->checkpoints($name); + + $forgotten = $checkpoints->installed() && $checkpoints->forget(); return $deleted > 0 || $forgotten; }); From be5667ebcf1de2ab2ee6d89f241dd6245756b66c Mon Sep 17 00:00:00 2001 From: Steve Bauman Date: Mon, 5 Oct 2026 12:38:01 -0400 Subject: [PATCH 14/14] Test repositories directly and retain focused command integration tests --- tests/Feature/CheckpointRepositoryTest.php | 106 ++++++++ tests/Feature/CheckpointTest.php | 167 +------------ tests/Feature/ConnectionTest.php | 82 +++--- tests/Feature/ForgetTest.php | 60 ----- tests/Feature/OperationRepositoryTest.php | 233 ++++++++++++++++++ tests/Feature/RunTest.php | 56 ----- .../operations/checkpoint_connection.php | 17 -- .../Fixtures/operations/checkpoint_method.php | 15 -- .../Fixtures/operations/checkpoint_values.php | 31 --- tests/Fixtures/operations/dispatch_job.php | 14 -- .../operations/duplicate_completion.php | 18 -- .../operations/failing_checkpoint_chunk.php | 25 -- .../failing_checkpoint_transaction.php | 18 -- .../operations/failing_transaction.php | 16 -- .../operations/invalid_checkpoint_value.php | 16 -- tests/Fixtures/operations/read_checkpoint.php | 15 -- .../operations/scoped_checkpoints.php | 17 -- .../operations/successful_transaction.php | 14 -- 18 files changed, 381 insertions(+), 539 deletions(-) create mode 100644 tests/Feature/CheckpointRepositoryTest.php create mode 100644 tests/Feature/OperationRepositoryTest.php delete mode 100644 tests/Fixtures/operations/checkpoint_connection.php delete mode 100644 tests/Fixtures/operations/checkpoint_method.php delete mode 100644 tests/Fixtures/operations/checkpoint_values.php delete mode 100644 tests/Fixtures/operations/dispatch_job.php delete mode 100644 tests/Fixtures/operations/duplicate_completion.php delete mode 100644 tests/Fixtures/operations/failing_checkpoint_chunk.php delete mode 100644 tests/Fixtures/operations/failing_checkpoint_transaction.php delete mode 100644 tests/Fixtures/operations/failing_transaction.php delete mode 100644 tests/Fixtures/operations/invalid_checkpoint_value.php delete mode 100644 tests/Fixtures/operations/read_checkpoint.php delete mode 100644 tests/Fixtures/operations/scoped_checkpoints.php delete mode 100644 tests/Fixtures/operations/successful_transaction.php diff --git a/tests/Feature/CheckpointRepositoryTest.php b/tests/Feature/CheckpointRepositoryTest.php new file mode 100644 index 0000000..77546b2 --- /dev/null +++ b/tests/Feature/CheckpointRepositoryTest.php @@ -0,0 +1,106 @@ +installed())->toBeTrue(); + + Schema::drop('operation_checkpoints'); + + expect($checkpoints->installed())->toBeFalse(); +}); + +test('using checkpoints without their migration raises a database exception', function (string $method, array $arguments) { + Schema::drop('operation_checkpoints'); + + $checkpoints = new CheckpointRepository(DB::connection(), 'backfill'); + + expect(fn () => $checkpoints->{$method}(...$arguments)) + ->toThrow(QueryException::class, 'operation_checkpoints'); +})->with([ + 'read' => ['get', ['cursor']], + 'write' => ['put', ['cursor', 'next-page']], + 'forget' => ['forget', []], +]); + +test('checkpoints read defaults and persist JSON values while preserving other keys', function (bool $multiple) { + $checkpoints = new CheckpointRepository(DB::connection(), 'backfill'); + $values = [ + 'cursor' => 'next-page', + 'count' => 0, + 'ratio' => 1.5, + 'enabled' => false, + 'empty' => '', + 'nullable' => null, + 'ids' => [1, 2, 3], + 'metadata' => ['page' => 2, 'finished' => false], + ]; + + expect($checkpoints->get('missing'))->toBeNull() + ->and($checkpoints->get('missing', 42))->toBe(42); + + if ($multiple) { + $checkpoints->put($values); + } else { + foreach ($values as $key => $value) { + $checkpoints->put($key, $value); + } + } + + $checkpoints->put('cursor', 'last-page'); + $checkpoints->put([]); + $values['cursor'] = 'last-page'; + + foreach ($values as $key => $value) { + expect($checkpoints->get($key, 'missing'))->toBe($value); + } + + expect(DB::table('operation_checkpoints')->count())->toBe(count($values)) + ->and(DB::table('operation_checkpoints')->where('key', 'missing')->exists())->toBeFalse(); +})->with(['multiple values' => true, 'single values' => false]); + +test('checkpoint values are isolated between operations', function () { + $first = new CheckpointRepository(DB::connection(), 'first'); + $second = new CheckpointRepository(DB::connection(), 'second'); + + $first->put('cursor', 'first-page'); + + expect($second->get('cursor'))->toBeNull(); + + $second->put('cursor', 'second-page'); + + expect($first->get('cursor'))->toBe('first-page') + ->and($second->get('cursor'))->toBe('second-page'); +}); + +test('forgetting checkpoints preserves other operations and completion records', function () { + $first = new CheckpointRepository(DB::connection(), 'first'); + $second = new CheckpointRepository(DB::connection(), 'second'); + + DB::table('operations')->insert(['name' => 'first', 'completed_at' => now()]); + $first->put(['cursor' => 'first-page', 'processed' => 10]); + $second->put('cursor', 'second-page'); + + expect($first->forget())->toBeTrue() + ->and($first->forget())->toBeFalse() + ->and($first->get('cursor'))->toBeNull() + ->and($first->get('processed'))->toBeNull() + ->and($second->get('cursor'))->toBe('second-page') + ->and(DB::table('operations')->pluck('name')->all())->toBe(['first']); +}); + +test('invalid JSON values fail before any checkpoints are changed', function () { + $checkpoints = new CheckpointRepository(DB::connection(), 'backfill'); + $checkpoints->put('last_id', 10); + + expect(fn () => $checkpoints->put(['last_id' => 20, 'invalid' => NAN]))->toThrow(JsonException::class); + + expect($checkpoints->get('last_id'))->toBe(10) + ->and($checkpoints->get('invalid'))->toBeNull() + ->and(DB::table('operation_checkpoints')->count())->toBe(1); +}); diff --git a/tests/Feature/CheckpointTest.php b/tests/Feature/CheckpointTest.php index 491780e..69b880e 100644 --- a/tests/Feature/CheckpointTest.php +++ b/tests/Feature/CheckpointTest.php @@ -1,6 +1,5 @@ null]); })->with(['default connection' => false, 'separate connection' => true]); -test('using checkpoints without their migration raises a database exception', function (string $method, array $arguments) { - Schema::drop('operation_checkpoints'); - app()->instance('checkpoint_method', $method); - app()->instance('checkpoint_arguments', $arguments); - - File::ensureDirectoryExists(config('operations.path')); - File::copy( - __DIR__.'/../Fixtures/operations/checkpoint_method.php', - config('operations.path').'/2026_10_05_120000_example.php', - ); - - expect(fn () => artisan('operations:run')->run())->toThrow(QueryException::class, 'operation_checkpoints'); - - expect(DB::table('operations')->count())->toBe(0); -})->with([ - 'read' => ['get', ['cursor']], - 'write' => ['put', ['cursor', 'next-page']], - 'forget' => ['forget', []], -]); - -test('checkpoints read defaults and persist JSON values while preserving other keys', function (bool $multiple) { - $values = [ - 'cursor' => 'next-page', - 'count' => 0, - 'ratio' => 1.5, - 'enabled' => false, - 'empty' => '', - 'nullable' => null, - 'ids' => [1, 2, 3], - 'metadata' => ['page' => 2, 'finished' => false], - ]; - - app()->instance('checkpoint_values', $values); - app()->instance('multiple', $multiple); - app()->instance('observed', collect()); - - File::ensureDirectoryExists(config('operations.path')); - File::copy( - __DIR__.'/../Fixtures/operations/checkpoint_values.php', - config('operations.path').'/2026_10_05_120000_example.php', - ); - - artisan('operations:run')->assertSuccessful(); - - $values['cursor'] = 'last-page'; - - expect(app('observed')->all())->toBe([null, 42, ...array_values($values)]) - ->and(DB::table('operation_checkpoints')->count())->toBe(count($values)) - ->and(DB::table('operation_checkpoints')->where('key', 'missing')->exists())->toBeFalse(); -})->with(['multiple values' => true, 'single values' => false]); - test('a failed operation resumes from its saved checkpoint on the next run', function () { Schema::create('examples', function (Blueprint $table) { $table->id(); @@ -129,117 +77,6 @@ expect(DB::table('examples')->pluck('id')->all())->toBe([1, 2, 3]) ->and(json_decode(DB::table('operation_checkpoints')->value('value')))->toBe(3) - ->and(DB::table('operations')->count())->toBe(1); -}); - -test('checkpoint keys are scoped to the operation filename', function () { - app()->instance('observed', collect()); - - File::ensureDirectoryExists(config('operations.path')); - - foreach (['2026_10_05_120000_first', '2026_10_05_120001_second'] as $name) { - File::copy( - __DIR__.'/../Fixtures/operations/scoped_checkpoints.php', - config('operations.path')."/{$name}.php", - ); - } - - artisan('operations:run')->assertSuccessful(); - - expect(app('observed')->all())->toBe([ - null, '2026_10_05_120000_first', - null, '2026_10_05_120001_second', - ])->and(DB::table('operation_checkpoints')->count())->toBe(2); -}); - -test('a failed chunk rolls back its work and checkpoint without losing earlier progress', function () { - Schema::create('examples', function (Blueprint $table) { - $table->id(); - }); - - File::ensureDirectoryExists(config('operations.path')); - File::copy( - __DIR__.'/../Fixtures/operations/failing_checkpoint_chunk.php', - config('operations.path').'/2026_10_05_120000_backfill.php', - ); - - expect(fn () => artisan('operations:run')->run())->toThrow(RuntimeException::class, 'Chunk failed.'); - - expect(DB::table('examples')->pluck('id')->all())->toBe([1]) - ->and(json_decode(DB::table('operation_checkpoints')->value('value')))->toBe(1) - ->and(DB::table('operations')->count())->toBe(0); -}); - -test('checkpoints participate in the whole operation transaction', function () { - $name = '2026_10_05_120000_backfill'; - - DB::table('operation_checkpoints')->insert([ - 'operation' => $name, - 'key' => 'last_id', - 'value' => '10', - ]); - - File::ensureDirectoryExists(config('operations.path')); - File::copy( - __DIR__.'/../Fixtures/operations/failing_checkpoint_transaction.php', - config('operations.path')."/{$name}.php", - ); - - expect(fn () => artisan('operations:run')->run())->toThrow(RuntimeException::class, 'Backfill failed.'); - - expect(DB::table('operation_checkpoints')->count())->toBe(1) - ->and(json_decode(DB::table('operation_checkpoints')->value('value')))->toBe(10) - ->and(DB::table('operations')->count())->toBe(0); -}); - -test('invalid JSON values fail before any checkpoints are changed', function () { - File::ensureDirectoryExists(config('operations.path')); - File::copy( - __DIR__.'/../Fixtures/operations/invalid_checkpoint_value.php', - config('operations.path').'/2026_10_05_120000_example.php', - ); - - expect(fn () => artisan('operations:run')->run())->toThrow(JsonException::class); - - expect(DB::table('operation_checkpoints')->count())->toBe(1) - ->and(json_decode(DB::table('operation_checkpoints')->value('value')))->toBe(10) - ->and(DB::table('operations')->count())->toBe(0); -}); - -test('checkpoints use the configured connection and read from its writer', function () { - File::ensureDirectoryExists(config('operations.path')); - $writer = config('operations.path').'/writer.sqlite'; - $reader = config('operations.path').'/reader.sqlite'; - File::put($writer, ''); - File::put($reader, ''); - - config([ - 'database.connections.operations' => [ - 'driver' => 'sqlite', - 'database' => $writer, - 'read' => ['database' => $reader], - 'write' => ['database' => $writer], - 'prefix' => '', - ], - 'operations.connection' => 'operations', - ]); - - foreach (File::glob(__DIR__.'/../../database/migrations/*.php') as $path) { - File::getRequire($path)->up(); - } - - app()->instance('observed', collect()); - - File::copy( - __DIR__.'/../Fixtures/operations/checkpoint_connection.php', - config('operations.path').'/2026_10_05_120000_example.php', - ); - - artisan('operations:run')->assertSuccessful(); - - expect(app('observed')->all())->toBe(['initial', 'next-page']) - ->and(DB::connection('operations')->table('operation_checkpoints')->useWritePdo()->count())->toBe(1) - ->and(DB::table('operation_checkpoints')->count())->toBe(0); - - config(['operations.connection' => null]); + ->and(DB::table('operations')->pluck('name')->all())->toBe(['2026_10_05_120000_backfill']) + ->and(DB::table('operation_checkpoints')->value('operation'))->toBe('2026_10_05_120000_backfill'); }); diff --git a/tests/Feature/ConnectionTest.php b/tests/Feature/ConnectionTest.php index e6ab812..958dcd4 100644 --- a/tests/Feature/ConnectionTest.php +++ b/tests/Feature/ConnectionTest.php @@ -1,13 +1,14 @@ [ 'driver' => 'sqlite', @@ -20,16 +21,16 @@ $migration = File::getRequire(__DIR__.'/../../database/migrations/2026_10_02_165413_create_operations_table.php'); $migration->up(); - File::ensureDirectoryExists(config('operations.path')); - File::copy( - __DIR__.'/../Fixtures/operations/empty_operation.php', - config('operations.path').'/2026_10_02_120000_example.php', - ); + $operation = new class extends Operation + { + public function handle(Command $command): void {} + }; - artisan('operations:run')->assertSuccessful(); - artisan('operations:run')->expectsOutputToContain('No pending operations.')->assertSuccessful(); + $repository = app(OperationRepository::class); + $repository->run('example', $operation, new Command); - expect(DB::connection('operations')->table('operations')->count())->toBe(1) + expect($repository->completed()->keys()->all())->toBe(['example']) + ->and(DB::connection('operations')->table('operations')->count())->toBe(1) ->and(DB::table('operations')->count())->toBe(0); $migration->down(); @@ -40,25 +41,7 @@ config(['operations.connection' => null]); }); -test('successful transactional operations commit their changes and completion together', function () { - Schema::create('examples', function (Blueprint $table) { - $table->id(); - }); - - File::ensureDirectoryExists(config('operations.path')); - File::copy( - __DIR__.'/../Fixtures/operations/successful_transaction.php', - config('operations.path').'/2026_10_02_120000_backfill.php', - ); - - artisan('operations:run')->assertSuccessful(); - - expect(DB::table('examples')->count())->toBe(1) - ->and(DB::table('operations')->count())->toBe(1) - ->and(DB::transactionLevel())->toBe(0); -}); - -test('completion history is read from the writer when the connection has a read replica', function () { +test('repositories use the configured connection and read from its writer', function () { File::ensureDirectoryExists(config('operations.path')); $writer = config('operations.path').'/writer.sqlite'; $reader = config('operations.path').'/reader.sqlite'; @@ -76,21 +59,36 @@ 'operations.connection' => 'operations', ]); - $migration = File::getRequire(__DIR__.'/../../database/migrations/2026_10_02_165413_create_operations_table.php'); - $migration->up(); + foreach (File::glob(__DIR__.'/../../database/migrations/*.php') as $path) { + File::getRequire($path)->up(); + } + + $operation = new class extends Operation + { + use HasCheckpoints; + + public array $observed = []; - File::copy( - __DIR__.'/../Fixtures/operations/record_execution.php', - config('operations.path').'/2026_10_02_120000_example.php', - ); + public function handle(Command $command): void + { + $this->observed[] = $this->checkpoints->get('cursor', 'initial'); + $this->checkpoints->put('cursor', 'next-page'); + $this->observed[] = $this->checkpoints->get('cursor'); + } + }; - app()->instance('executed', collect()); + $repository = app(OperationRepository::class); + $repository->run('example', $operation, new Command); - artisan('operations:run')->assertSuccessful(); - artisan('operations:run')->expectsOutputToContain('No pending operations.')->assertSuccessful(); + expect($operation->observed)->toBe(['initial', 'next-page']) + ->and($repository->completed()->keys()->all())->toBe(['example']) + ->and(DB::connection('operations')->table('operation_checkpoints')->useWritePdo()->count())->toBe(1) + ->and(DB::table('operations')->count())->toBe(0) + ->and(DB::table('operation_checkpoints')->count())->toBe(0); - expect(app('executed')->all())->toBe(['example']) - ->and(DB::connection('operations')->table('operations')->useWritePdo()->count())->toBe(1); + expect($repository->forget('example'))->toBeTrue() + ->and($repository->completed()->all())->toBe([]) + ->and(DB::connection('operations')->table('operation_checkpoints')->useWritePdo()->count())->toBe(0); config(['operations.connection' => null]); }); diff --git a/tests/Feature/ForgetTest.php b/tests/Feature/ForgetTest.php index bfbc916..f2b48d9 100644 --- a/tests/Feature/ForgetTest.php +++ b/tests/Feature/ForgetTest.php @@ -118,63 +118,3 @@ expect(DB::table('operations')->pluck('name')->all())->toBe(['2026_10_02_120000_example']); })->with(['missing', 'example', '2026_10_02_120000_example.php']); - -test('forgetting uses the configured operations connection', function () { - config([ - 'database.connections.operations' => [ - 'driver' => 'sqlite', - 'database' => ':memory:', - 'prefix' => '', - ], - 'operations.connection' => 'operations', - ]); - - foreach (File::glob(__DIR__.'/../../database/migrations/*.php') as $path) { - File::getRequire($path)->up(); - } - - $name = '2026_10_02_120000_example'; - DB::table('operations')->insert(['name' => $name, 'completed_at' => now()]); - DB::connection('operations')->table('operations')->insert(['name' => $name, 'completed_at' => now()]); - DB::table('operation_checkpoints')->insert(['operation' => $name, 'key' => 'last_id', 'value' => '10']); - DB::connection('operations')->table('operation_checkpoints')->insert(['operation' => $name, 'key' => 'last_id', 'value' => '20']); - - artisan('operations:forget', ['name' => $name, '--force' => true])->assertSuccessful(); - - expect(DB::connection('operations')->table('operations')->count())->toBe(0) - ->and(DB::connection('operations')->table('operation_checkpoints')->count())->toBe(0) - ->and(DB::table('operations')->pluck('name')->all())->toBe([$name]) - ->and(DB::table('operation_checkpoints')->pluck('operation')->all())->toBe([$name]); -}); - -test('forgetting clears checkpoints for completed and unfinished operations', function (bool $completed) { - $name = '2026_10_05_120000_example'; - $other = '2026_10_05_120001_other'; - - if ($completed) { - DB::table('operations')->insert(['name' => $name, 'completed_at' => now()]); - } - - DB::table('operation_checkpoints')->insert([ - ['operation' => $name, 'key' => 'last_id', 'value' => '10'], - ['operation' => $name, 'key' => 'finished', 'value' => 'true'], - ['operation' => $other, 'key' => 'last_id', 'value' => '20'], - ]); - - app()->instance('observed', collect()); - - File::ensureDirectoryExists(config('operations.path')); - File::copy( - __DIR__.'/../Fixtures/operations/read_checkpoint.php', - config('operations.path')."/{$name}.php", - ); - - artisan('operations:forget', ['name' => $name, '--force' => true])->assertSuccessful(); - - expect(DB::table('operations')->count())->toBe(0) - ->and(DB::table('operation_checkpoints')->pluck('operation')->all())->toBe([$other]); - - artisan('operations:run', ['operation' => $name])->assertSuccessful(); - - expect(app('observed')->all())->toBe([0]); -})->with(['completed' => true, 'unfinished' => false]); diff --git a/tests/Feature/OperationRepositoryTest.php b/tests/Feature/OperationRepositoryTest.php new file mode 100644 index 0000000..e2f6b61 --- /dev/null +++ b/tests/Feature/OperationRepositoryTest.php @@ -0,0 +1,233 @@ +previous = $this->checkpoints->get('cursor'); + $this->checkpoints->put('cursor', 'next-page'); + } + }; + + $first = new CheckpointRepository(DB::connection(), 'first'); + $second = new CheckpointRepository(DB::connection(), 'second'); + $first->put('cursor', 'first-page'); + $second->put('cursor', 'second-page'); + + $repository->run('first', $operation, new Command); + expect($operation->previous)->toBe('first-page'); + + $repository->run('second', $operation, new Command); + expect($operation->previous)->toBe('second-page') + ->and($first->get('cursor'))->toBe('next-page') + ->and($second->get('cursor'))->toBe('next-page') + ->and($repository->completed()->keys()->all())->toBe(['first', 'second']); +}); + +test('a failed chunk rolls back its work and checkpoint without losing earlier progress', function () { + Schema::create('examples', function (Blueprint $table) { + $table->id(); + }); + + $operation = new class extends Operation + { + use HasCheckpoints; + + public function handle(Command $command): void + { + foreach ([1, 2] as $id) { + DB::transaction(function () use ($id) { + DB::table('examples')->insert(['id' => $id]); + $this->checkpoints->put('last_id', $id); + + if ($id === 2) { + throw new RuntimeException('Chunk failed.'); + } + }); + } + } + }; + + expect(fn () => app(OperationRepository::class)->run('backfill', $operation, new Command)) + ->toThrow(RuntimeException::class, 'Chunk failed.'); + + expect(DB::table('examples')->pluck('id')->all())->toBe([1]) + ->and((new CheckpointRepository(DB::connection(), 'backfill'))->get('last_id'))->toBe(1) + ->and(DB::table('operations')->count())->toBe(0); +}); + +test('checkpoints participate in the whole operation transaction', function () { + $checkpoints = new CheckpointRepository(DB::connection(), 'backfill'); + $checkpoints->put('last_id', 10); + + $operation = new class extends Operation implements WithinTransaction + { + use HasCheckpoints; + + public function handle(Command $command): void + { + $this->checkpoints->put(['last_id' => 20, 'finished' => true]); + + throw new RuntimeException('Backfill failed.'); + } + }; + + expect(fn () => app(OperationRepository::class)->run('backfill', $operation, new Command)) + ->toThrow(RuntimeException::class, 'Backfill failed.'); + + expect($checkpoints->get('last_id'))->toBe(10) + ->and($checkpoints->get('finished'))->toBeNull() + ->and(DB::table('operation_checkpoints')->count())->toBe(1) + ->and(DB::table('operations')->count())->toBe(0); +}); + +test('a transaction rolls back database changes when an operation fails', function () { + Schema::create('examples', function (Blueprint $table) { + $table->id(); + }); + + $operation = new class extends Operation implements WithinTransaction + { + public function handle(Command $command): void + { + DB::table('examples')->insert(['id' => 1]); + + throw new RuntimeException('Backfill failed.'); + } + }; + + expect(fn () => app(OperationRepository::class)->run('backfill', $operation, new Command)) + ->toThrow(RuntimeException::class, 'Backfill failed.'); + + expect(DB::table('examples')->count())->toBe(0) + ->and(DB::table('operations')->count())->toBe(0); +}); + +test('a transaction rolls back the operation if recording completion fails', function () { + Schema::create('examples', function (Blueprint $table) { + $table->id(); + }); + + $operation = new class extends Operation implements WithinTransaction + { + public function handle(Command $command): void + { + DB::table('examples')->insert(['id' => 1]); + DB::table('operations')->insert(['name' => 'backfill', 'completed_at' => now()]); + } + }; + + expect(fn () => app(OperationRepository::class)->run('backfill', $operation, new Command)) + ->toThrow(QueryException::class); + + expect(DB::table('examples')->count())->toBe(0) + ->and(DB::table('operations')->count())->toBe(0); +}); + +test('successful transactional operations commit their changes and completion together', function () { + Schema::create('examples', function (Blueprint $table) { + $table->id(); + }); + + $operation = new class extends Operation implements WithinTransaction + { + public function handle(Command $command): void + { + DB::table('examples')->insert(['id' => 1]); + } + }; + + app(OperationRepository::class)->run('backfill', $operation, new Command); + + expect(DB::table('examples')->count())->toBe(1) + ->and(DB::table('operations')->where('name', 'backfill')->value('completed_at'))->not->toBeNull() + ->and(DB::transactionLevel())->toBe(0); +}); + +test('operations can dispatch jobs without becoming queued operations themselves', function () { + Bus::fake(); + + $operation = new class extends Operation + { + public function handle(Command $command): void + { + Bus::dispatch(new ExampleJob); + } + }; + + app(OperationRepository::class)->run('dispatch', $operation, new Command); + + Bus::assertDispatched(ExampleJob::class); + Bus::assertDispatchedTimes(ExampleJob::class, 1); + + expect(DB::table('operations')->where('name', 'dispatch')->value('completed_at'))->not->toBeNull(); +}); + +test('forgetting clears checkpoints for completed and unfinished operations', function (bool $completed) { + $checkpoints = new CheckpointRepository(DB::connection(), 'backfill'); + $other = new CheckpointRepository(DB::connection(), 'other'); + + if ($completed) { + DB::table('operations')->insert(['name' => 'backfill', 'completed_at' => now()]); + } + + $checkpoints->put(['last_id' => 10, 'finished' => true]); + $other->put('last_id', 20); + + $repository = app(OperationRepository::class); + + expect($repository->forget('backfill'))->toBeTrue() + ->and($checkpoints->get('last_id', 0))->toBe(0) + ->and($checkpoints->get('finished'))->toBeNull() + ->and($other->get('last_id'))->toBe(20) + ->and(DB::table('operations')->count())->toBe(0) + ->and($repository->forget('backfill'))->toBeFalse(); +})->with(['completed' => true, 'unfinished' => false]); + +test('forgetting uses the configured operations connection', function () { + config([ + 'database.connections.operations' => [ + 'driver' => 'sqlite', + 'database' => ':memory:', + 'prefix' => '', + ], + 'operations.connection' => 'operations', + ]); + + foreach (File::glob(__DIR__.'/../../database/migrations/*.php') as $path) { + File::getRequire($path)->up(); + } + + $name = '2026_10_02_120000_example'; + DB::table('operations')->insert(['name' => $name, 'completed_at' => now()]); + DB::connection('operations')->table('operations')->insert(['name' => $name, 'completed_at' => now()]); + DB::table('operation_checkpoints')->insert(['operation' => $name, 'key' => 'last_id', 'value' => '10']); + DB::connection('operations')->table('operation_checkpoints')->insert(['operation' => $name, 'key' => 'last_id', 'value' => '20']); + + expect(app(OperationRepository::class)->forget($name))->toBeTrue(); + + expect(DB::connection('operations')->table('operations')->count())->toBe(0) + ->and(DB::connection('operations')->table('operation_checkpoints')->count())->toBe(0) + ->and(DB::table('operations')->pluck('name')->all())->toBe([$name]) + ->and(DB::table('operation_checkpoints')->pluck('operation')->all())->toBe([$name]); +}); diff --git a/tests/Feature/RunTest.php b/tests/Feature/RunTest.php index f9aaa7a..4250e17 100644 --- a/tests/Feature/RunTest.php +++ b/tests/Feature/RunTest.php @@ -2,14 +2,9 @@ use DirectoryTree\Operations\Commands\RunCommand; use DirectoryTree\Operations\Runner; -use DirectoryTree\Operations\Tests\Fixtures\ExampleJob; -use Illuminate\Database\QueryException; -use Illuminate\Database\Schema\Blueprint; use Illuminate\Support\Facades\Artisan; -use Illuminate\Support\Facades\Bus; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\File; -use Illuminate\Support\Facades\Schema; use function Pest\Laravel\artisan; @@ -63,57 +58,6 @@ ])->and(DB::table('operations')->count())->toBe(3); }); -test('a transaction rolls back database changes when an operation fails', function (?string $operation) { - Schema::create('examples', function (Blueprint $table) { - $table->id(); - }); - - File::ensureDirectoryExists(config('operations.path')); - File::copy( - __DIR__.'/../Fixtures/operations/failing_transaction.php', - config('operations.path').'/2026_10_02_120000_backfill.php', - ); - - expect(fn () => Artisan::call('operations:run', ['operation' => $operation]))->toThrow(RuntimeException::class, 'Backfill failed.'); - - expect(DB::table('examples')->count())->toBe(0) - ->and(DB::table('operations')->count())->toBe(0); -})->with([null, '2026_10_02_120000_backfill']); - -test('a transaction rolls back the operation if recording completion fails', function (?string $operation) { - Schema::create('examples', function (Blueprint $table) { - $table->id(); - }); - - File::ensureDirectoryExists(config('operations.path')); - File::copy( - __DIR__.'/../Fixtures/operations/duplicate_completion.php', - config('operations.path').'/2026_10_02_120000_backfill.php', - ); - - expect(fn () => Artisan::call('operations:run', ['operation' => $operation]))->toThrow(QueryException::class); - - expect(DB::table('examples')->count())->toBe(0) - ->and(DB::table('operations')->count())->toBe(0); -})->with([null, '2026_10_02_120000_backfill']); - -test('operations can dispatch jobs without becoming queued operations themselves', function () { - Bus::fake(); - - File::ensureDirectoryExists(config('operations.path')); - File::copy( - __DIR__.'/../Fixtures/operations/dispatch_job.php', - config('operations.path').'/2026_10_02_120000_dispatch.php', - ); - - artisan('operations:run')->assertSuccessful(); - - Bus::assertDispatched(ExampleJob::class); - Bus::assertDispatchedTimes(ExampleJob::class, 1); - - expect(DB::table('operations')->count())->toBe(1); -}); - test('completed operation files are not loaded on subsequent runs', function (?string $operation) { File::ensureDirectoryExists(config('operations.path')); File::copy( diff --git a/tests/Fixtures/operations/checkpoint_connection.php b/tests/Fixtures/operations/checkpoint_connection.php deleted file mode 100644 index 95798e4..0000000 --- a/tests/Fixtures/operations/checkpoint_connection.php +++ /dev/null @@ -1,17 +0,0 @@ -push($this->checkpoints->get('cursor', 'initial')); - $this->checkpoints->put('cursor', 'next-page'); - app('observed')->push($this->checkpoints->get('cursor')); - } -}; diff --git a/tests/Fixtures/operations/checkpoint_method.php b/tests/Fixtures/operations/checkpoint_method.php deleted file mode 100644 index b91bc89..0000000 --- a/tests/Fixtures/operations/checkpoint_method.php +++ /dev/null @@ -1,15 +0,0 @@ -checkpoints->{app('checkpoint_method')}(...app('checkpoint_arguments')); - } -}; diff --git a/tests/Fixtures/operations/checkpoint_values.php b/tests/Fixtures/operations/checkpoint_values.php deleted file mode 100644 index ff8b14d..0000000 --- a/tests/Fixtures/operations/checkpoint_values.php +++ /dev/null @@ -1,31 +0,0 @@ -push($this->checkpoints->get('missing')); - app('observed')->push($this->checkpoints->get('missing', 42)); - - if (app('multiple')) { - $this->checkpoints->put(app('checkpoint_values')); - } else { - foreach (app('checkpoint_values') as $key => $value) { - $this->checkpoints->put($key, $value); - } - } - - $this->checkpoints->put('cursor', 'last-page'); - $this->checkpoints->put([]); - - foreach (app('checkpoint_values') as $key => $value) { - app('observed')->push($this->checkpoints->get($key, 'missing')); - } - } -}; diff --git a/tests/Fixtures/operations/dispatch_job.php b/tests/Fixtures/operations/dispatch_job.php deleted file mode 100644 index c193f7a..0000000 --- a/tests/Fixtures/operations/dispatch_job.php +++ /dev/null @@ -1,14 +0,0 @@ -insert(['id' => 1]); - DB::table('operations')->insert([ - 'name' => basename(__FILE__, '.php'), - 'completed_at' => now(), - ]); - } -}; diff --git a/tests/Fixtures/operations/failing_checkpoint_chunk.php b/tests/Fixtures/operations/failing_checkpoint_chunk.php deleted file mode 100644 index f1f8ae5..0000000 --- a/tests/Fixtures/operations/failing_checkpoint_chunk.php +++ /dev/null @@ -1,25 +0,0 @@ -insert(['id' => $id]); - $this->checkpoints->put('last_id', $id); - - if ($id === 2) { - throw new RuntimeException('Chunk failed.'); - } - }); - } - } -}; diff --git a/tests/Fixtures/operations/failing_checkpoint_transaction.php b/tests/Fixtures/operations/failing_checkpoint_transaction.php deleted file mode 100644 index c2e86db..0000000 --- a/tests/Fixtures/operations/failing_checkpoint_transaction.php +++ /dev/null @@ -1,18 +0,0 @@ -checkpoints->put(['last_id' => 20, 'finished' => true]); - - throw new RuntimeException('Backfill failed.'); - } -}; diff --git a/tests/Fixtures/operations/failing_transaction.php b/tests/Fixtures/operations/failing_transaction.php deleted file mode 100644 index 58b368a..0000000 --- a/tests/Fixtures/operations/failing_transaction.php +++ /dev/null @@ -1,16 +0,0 @@ -insert(['id' => 1]); - - throw new RuntimeException('Backfill failed.'); - } -}; diff --git a/tests/Fixtures/operations/invalid_checkpoint_value.php b/tests/Fixtures/operations/invalid_checkpoint_value.php deleted file mode 100644 index b052c4a..0000000 --- a/tests/Fixtures/operations/invalid_checkpoint_value.php +++ /dev/null @@ -1,16 +0,0 @@ -checkpoints->put('last_id', 10); - $this->checkpoints->put(['last_id' => 20, 'invalid' => NAN]); - } -}; diff --git a/tests/Fixtures/operations/read_checkpoint.php b/tests/Fixtures/operations/read_checkpoint.php deleted file mode 100644 index fad1852..0000000 --- a/tests/Fixtures/operations/read_checkpoint.php +++ /dev/null @@ -1,15 +0,0 @@ -push($this->checkpoints->get('last_id', 0)); - } -}; diff --git a/tests/Fixtures/operations/scoped_checkpoints.php b/tests/Fixtures/operations/scoped_checkpoints.php deleted file mode 100644 index 2ba084d..0000000 --- a/tests/Fixtures/operations/scoped_checkpoints.php +++ /dev/null @@ -1,17 +0,0 @@ -push($this->checkpoints->get('cursor')); - $this->checkpoints->put('cursor', basename(__FILE__, '.php')); - app('observed')->push($this->checkpoints->get('cursor')); - } -}; diff --git a/tests/Fixtures/operations/successful_transaction.php b/tests/Fixtures/operations/successful_transaction.php deleted file mode 100644 index 5b1c8b6..0000000 --- a/tests/Fixtures/operations/successful_transaction.php +++ /dev/null @@ -1,14 +0,0 @@ -insert(['id' => 1]); - } -};