diff --git a/README.md b/README.md index 9eaffa3..a50109d 100644 --- a/README.md +++ b/README.md @@ -219,9 +219,68 @@ The command asks for confirmation. Use `--force` to skip the prompt: php artisan operations:forget 2026_10_02_120000_backfill_company_names --force ``` -Forgetting only deletes the completion record. It does not undo previous effects, delete the file, or execute the operation. If the file is present, the next `operations:run` will execute it again. Make sure it is safe to repeat. +Forgetting deletes the completion record and any saved checkpoints. It does not undo previous effects, delete the file, or execute the operation. If the file is present, the next `operations:run` will execute it again from the beginning. Make sure it is safe to repeat. -You can also forget records whose files have been removed. The command fails if no completion record matches the supplied name. Failed operations do not need to be forgotten; they are already pending. +You can also forget records whose files have been removed, or clear checkpoints from an unfinished operation. The command fails if neither a completion record nor checkpoints match the supplied name. Failed operations are already pending; only forget them if you want to discard their saved progress. + +### Checkpoints + +Checkpoints are optional. Publish and run their migration only if you need to save progress between attempts: + +```bash +php artisan vendor:publish --tag=operations-checkpoints-migration +php artisan migrate +``` + +Add the `DirectoryTree\Operations\Concerns\HasCheckpoints` trait to access the operation's checkpoint repository. Use `get()` to read a value with an optional default, and `put()` to save a value or an array of values: + +```php +$cursor = $this->checkpoints->get('cursor'); +$lastId = $this->checkpoints->get('last_id', 0); + +$this->checkpoints->put('last_id', 15000); + +$this->checkpoints->put([ + 'cursor' => $nextCursor, + 'processed' => $processed, +]); +``` + +Keys are scoped to the operation's filename. Writes update the supplied keys and preserve other saved values. Values are stored as JSON and read back as scalars or arrays; use JSON-compatible values such as strings, numbers, booleans, arrays, and `null`. Saving `null` retains the key, so reading it returns `null` even when a default is provided. + +For example, save the last processed ID to resume a backfill: + +```php +use App\Models\Company; +use DirectoryTree\Operations\Concerns\HasCheckpoints; +use DirectoryTree\Operations\Operation; +use Illuminate\Console\Command; +use Illuminate\Support\Collection; + +return new class extends Operation +{ + use HasCheckpoints; + + public function handle(Command $command): void + { + Company::query() + ->where('id', '>', $this->checkpoints->get('last_id', 0)) + ->chunkById(1000, function (Collection $companies) { + foreach ($companies as $company) { + $company->update(['display_name' => $company->name]); + } + + $this->checkpoints->put('last_id', $companies->last()->id); + }); + } +}; +``` + +After a failure, `operations:run` starts `handle()` again. Your code reads the saved checkpoint to decide where to continue. Saving a checkpoint does not mark the operation complete or prevent concurrent execution. Checkpoints remain after completion until the operation is forgotten. You can also call `$this->checkpoints->forget()` to clear all of the operation's checkpoints without changing its completion record. + +In the example above, an interrupted chunk may run again. Write work that can safely repeat, or wrap each chunk's database changes and checkpoint write in a transaction on the same connection. Checkpoints use `operations.connection` and participate in its active transaction. With `WithinTransaction`, a failure rolls back all checkpoints written during that attempt along with the operation's database changes. External effects, such as API calls, are not rolled back. + +The standard `operations-migrations` tag only publishes the operations table. Running, checking status, and forgetting operations work without the checkpoint table; `operations:forget` clears checkpoints when that table exists. Calling the checkpoint repository's `get()`, `put()`, or `forget()` methods before installing its migration raises a database exception. ### Transactions @@ -317,4 +376,4 @@ return [ ]; ``` -`connection` selects the database connection for the operations table and optional transactions. `null` uses the application's default connection. It does not change the connection used by your application models. +`connection` selects the database connection for the operations and checkpoint tables, and optional transactions. `null` uses the application's default connection. It does not change the connection used by your application models. diff --git a/database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php b/database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php new file mode 100644 index 0000000..db05912 --- /dev/null +++ b/database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php @@ -0,0 +1,30 @@ +create('operation_checkpoints', function (Blueprint $table) { + $table->id(); + $table->string('operation'); + $table->string('key'); + $table->json('value'); + $table->unique(['operation', 'key']); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::connection(config('operations.connection'))->dropIfExists('operation_checkpoints'); + } +}; diff --git a/src/CheckpointRepository.php b/src/CheckpointRepository.php new file mode 100644 index 0000000..61b73a7 --- /dev/null +++ b/src/CheckpointRepository.php @@ -0,0 +1,72 @@ +connection->table('operation_checkpoints') + ->useWritePdo() + ->where('operation', $this->operation) + ->where('key', $key) + ->value('value'); + + return $value === null ? value($default) : json_decode($value, true, flags: JSON_THROW_ON_ERROR); + } + + /** + * Persist checkpoint values without replacing other keys. + * + * @param string|array $key + */ + public function put(string|array $key, mixed $value = null): void + { + $values = is_array($key) ? $key : [$key => $value]; + + $checkpoints = []; + + foreach ($values as $key => $value) { + $checkpoints[] = [ + 'operation' => $this->operation, + 'key' => $key, + 'value' => json_encode($value, JSON_THROW_ON_ERROR), + ]; + } + + $this->connection->table('operation_checkpoints') + ->upsert($checkpoints, ['operation', 'key'], ['value']); + } + + /** + * Determine whether the checkpoint table is installed. + */ + public function installed(): bool + { + return $this->connection->getSchemaBuilder() + ->hasTable('operation_checkpoints'); + } + + /** + * Delete the operation's checkpoints. + */ + public function forget(): bool + { + return $this->connection->table('operation_checkpoints') + ->where('operation', $this->operation) + ->delete() > 0; + } +} diff --git a/src/Commands/ForgetCommand.php b/src/Commands/ForgetCommand.php index f523eee..897b4df 100644 --- a/src/Commands/ForgetCommand.php +++ b/src/Commands/ForgetCommand.php @@ -21,7 +21,7 @@ class ForgetCommand extends Command * * @var string */ - protected $description = 'Delete the completion record of an operation'; + protected $description = 'Delete the completion record and checkpoints of an operation'; /** * Execute the console command. @@ -39,7 +39,7 @@ public function handle(OperationRepository $repository): int } if (! $repository->forget($name)) { - $this->components->error("No completion record found for operation [{$name}]."); + $this->components->error("No completion record or checkpoints found for operation [{$name}]."); return self::FAILURE; } diff --git a/src/Concerns/HasCheckpoints.php b/src/Concerns/HasCheckpoints.php new file mode 100644 index 0000000..df0b6f5 --- /dev/null +++ b/src/Concerns/HasCheckpoints.php @@ -0,0 +1,23 @@ +checkpoints = $checkpoints; + + return $this; + } +} diff --git a/src/OperationRepository.php b/src/OperationRepository.php index ba4f561..601c231 100644 --- a/src/OperationRepository.php +++ b/src/OperationRepository.php @@ -2,6 +2,7 @@ namespace DirectoryTree\Operations; +use DirectoryTree\Operations\Concerns\HasCheckpoints; use DirectoryTree\Operations\Contracts\WithinTransaction; use Illuminate\Console\Command; use Illuminate\Database\ConnectionInterface; @@ -29,13 +30,21 @@ public function completed(): Collection } /** - * Delete the completion record for the given operation. + * Delete the completion record and checkpoints for the given operation. */ public function forget(string $name): bool { - return $this->connection->table('operations') - ->where('name', $name) - ->delete() > 0; + return $this->connection->transaction(function () use ($name) { + $deleted = $this->connection->table('operations') + ->where('name', $name) + ->delete(); + + $checkpoints = $this->checkpoints($name); + + $forgotten = $checkpoints->installed() && $checkpoints->forget(); + + return $deleted > 0 || $forgotten; + }); } /** @@ -43,6 +52,10 @@ public function forget(string $name): bool */ public function run(string $name, Operation $operation, Command $command): void { + if (in_array(HasCheckpoints::class, class_uses_recursive($operation))) { + $operation->setCheckpoints($this->checkpoints($name)); + } + $run = function () use ($name, $operation, $command) { $operation->handle($command); @@ -58,4 +71,12 @@ public function run(string $name, Operation $operation, Command $command): void $run(); } } + + /** + * Create a checkpoint repository for the given operation. + */ + protected function checkpoints(string $name): CheckpointRepository + { + return new CheckpointRepository($this->connection, $name); + } } diff --git a/src/OperationsServiceProvider.php b/src/OperationsServiceProvider.php index 80d4021..b198dea 100644 --- a/src/OperationsServiceProvider.php +++ b/src/OperationsServiceProvider.php @@ -40,8 +40,12 @@ public function boot(): void ], 'operations-config'); $this->publishesMigrations([ - __DIR__.'/../database/migrations' => database_path('migrations'), + __DIR__.'/../database/migrations/2026_10_02_165413_create_operations_table.php' => database_path('migrations/2026_10_02_165413_create_operations_table.php'), ], 'operations-migrations'); + + $this->publishesMigrations([ + __DIR__.'/../database/migrations/2026_10_05_041651_create_operation_checkpoints_table.php' => database_path('migrations/2026_10_05_041651_create_operation_checkpoints_table.php'), + ], 'operations-checkpoints-migration'); } } } diff --git a/tests/Feature/CheckpointRepositoryTest.php b/tests/Feature/CheckpointRepositoryTest.php new file mode 100644 index 0000000..77546b2 --- /dev/null +++ b/tests/Feature/CheckpointRepositoryTest.php @@ -0,0 +1,106 @@ +installed())->toBeTrue(); + + Schema::drop('operation_checkpoints'); + + expect($checkpoints->installed())->toBeFalse(); +}); + +test('using checkpoints without their migration raises a database exception', function (string $method, array $arguments) { + Schema::drop('operation_checkpoints'); + + $checkpoints = new CheckpointRepository(DB::connection(), 'backfill'); + + expect(fn () => $checkpoints->{$method}(...$arguments)) + ->toThrow(QueryException::class, 'operation_checkpoints'); +})->with([ + 'read' => ['get', ['cursor']], + 'write' => ['put', ['cursor', 'next-page']], + 'forget' => ['forget', []], +]); + +test('checkpoints read defaults and persist JSON values while preserving other keys', function (bool $multiple) { + $checkpoints = new CheckpointRepository(DB::connection(), 'backfill'); + $values = [ + 'cursor' => 'next-page', + 'count' => 0, + 'ratio' => 1.5, + 'enabled' => false, + 'empty' => '', + 'nullable' => null, + 'ids' => [1, 2, 3], + 'metadata' => ['page' => 2, 'finished' => false], + ]; + + expect($checkpoints->get('missing'))->toBeNull() + ->and($checkpoints->get('missing', 42))->toBe(42); + + if ($multiple) { + $checkpoints->put($values); + } else { + foreach ($values as $key => $value) { + $checkpoints->put($key, $value); + } + } + + $checkpoints->put('cursor', 'last-page'); + $checkpoints->put([]); + $values['cursor'] = 'last-page'; + + foreach ($values as $key => $value) { + expect($checkpoints->get($key, 'missing'))->toBe($value); + } + + expect(DB::table('operation_checkpoints')->count())->toBe(count($values)) + ->and(DB::table('operation_checkpoints')->where('key', 'missing')->exists())->toBeFalse(); +})->with(['multiple values' => true, 'single values' => false]); + +test('checkpoint values are isolated between operations', function () { + $first = new CheckpointRepository(DB::connection(), 'first'); + $second = new CheckpointRepository(DB::connection(), 'second'); + + $first->put('cursor', 'first-page'); + + expect($second->get('cursor'))->toBeNull(); + + $second->put('cursor', 'second-page'); + + expect($first->get('cursor'))->toBe('first-page') + ->and($second->get('cursor'))->toBe('second-page'); +}); + +test('forgetting checkpoints preserves other operations and completion records', function () { + $first = new CheckpointRepository(DB::connection(), 'first'); + $second = new CheckpointRepository(DB::connection(), 'second'); + + DB::table('operations')->insert(['name' => 'first', 'completed_at' => now()]); + $first->put(['cursor' => 'first-page', 'processed' => 10]); + $second->put('cursor', 'second-page'); + + expect($first->forget())->toBeTrue() + ->and($first->forget())->toBeFalse() + ->and($first->get('cursor'))->toBeNull() + ->and($first->get('processed'))->toBeNull() + ->and($second->get('cursor'))->toBe('second-page') + ->and(DB::table('operations')->pluck('name')->all())->toBe(['first']); +}); + +test('invalid JSON values fail before any checkpoints are changed', function () { + $checkpoints = new CheckpointRepository(DB::connection(), 'backfill'); + $checkpoints->put('last_id', 10); + + expect(fn () => $checkpoints->put(['last_id' => 20, 'invalid' => NAN]))->toThrow(JsonException::class); + + expect($checkpoints->get('last_id'))->toBe(10) + ->and($checkpoints->get('invalid'))->toBeNull() + ->and(DB::table('operation_checkpoints')->count())->toBe(1); +}); diff --git a/tests/Feature/CheckpointTest.php b/tests/Feature/CheckpointTest.php new file mode 100644 index 0000000..69b880e --- /dev/null +++ b/tests/Feature/CheckpointTest.php @@ -0,0 +1,82 @@ + [ + 'driver' => 'sqlite', + 'database' => ':memory:', + 'prefix' => '', + ], + 'operations.connection' => 'operations', + ]); + + File::getRequire(__DIR__.'/../../database/migrations/2026_10_02_165413_create_operations_table.php')->up(); + } else { + Schema::drop('operation_checkpoints'); + } + + $connection = DB::connection(config('operations.connection')); + $name = '2026_10_05_120000_example'; + + File::ensureDirectoryExists(config('operations.path')); + File::copy( + __DIR__.'/../Fixtures/operations/successful_operation.php', + config('operations.path')."/{$name}.php", + ); + + artisan('operations:run')->expectsOutputToContain('Operation executed.')->assertSuccessful(); + artisan('operations:run')->expectsOutputToContain('No pending operations.')->assertSuccessful(); + + artisan('operations:status')->expectsTable( + ['Operation', 'Status', 'Completed at', 'File'], + [[$name, 'Completed', $connection->table('operations')->value('completed_at'), 'Present']], + )->assertSuccessful(); + + File::delete(config('operations.path')."/{$name}.php"); + + artisan('operations:forget', ['name' => $name, '--force' => true])->assertSuccessful(); + artisan('operations:forget', ['name' => $name, '--force' => true])->assertFailed(); + + expect($connection->table('operations')->count())->toBe(0) + ->and($connection->getSchemaBuilder()->hasTable('operation_checkpoints'))->toBeFalse(); + + config(['operations.connection' => null]); +})->with(['default connection' => false, 'separate connection' => true]); + +test('a failed operation resumes from its saved checkpoint on the next run', function () { + Schema::create('examples', function (Blueprint $table) { + $table->id(); + }); + + app()->instance('should_fail', true); + + File::ensureDirectoryExists(config('operations.path')); + File::copy( + __DIR__.'/../Fixtures/operations/resumable_backfill.php', + config('operations.path').'/2026_10_05_120000_backfill.php', + ); + + expect(fn () => artisan('operations:run')->run())->toThrow(RuntimeException::class, 'Backfill interrupted.'); + + expect(DB::table('examples')->pluck('id')->all())->toBe([1, 2]) + ->and(json_decode(DB::table('operation_checkpoints')->value('value')))->toBe(2) + ->and(DB::table('operations')->count())->toBe(0); + + app()->instance('should_fail', false); + + artisan('operations:run')->assertSuccessful(); + artisan('operations:run')->expectsOutputToContain('No pending operations.')->assertSuccessful(); + + expect(DB::table('examples')->pluck('id')->all())->toBe([1, 2, 3]) + ->and(json_decode(DB::table('operation_checkpoints')->value('value')))->toBe(3) + ->and(DB::table('operations')->pluck('name')->all())->toBe(['2026_10_05_120000_backfill']) + ->and(DB::table('operation_checkpoints')->value('operation'))->toBe('2026_10_05_120000_backfill'); +}); diff --git a/tests/Feature/ConnectionTest.php b/tests/Feature/ConnectionTest.php index 187b2d2..958dcd4 100644 --- a/tests/Feature/ConnectionTest.php +++ b/tests/Feature/ConnectionTest.php @@ -1,13 +1,14 @@ [ 'driver' => 'sqlite', @@ -20,19 +21,16 @@ $migration = File::getRequire(__DIR__.'/../../database/migrations/2026_10_02_165413_create_operations_table.php'); $migration->up(); - File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_example.php', <<<'PHP' - assertSuccessful(); - artisan('operations:run')->expectsOutputToContain('No pending operations.')->assertSuccessful(); + $repository = app(OperationRepository::class); + $repository->run('example', $operation, new Command); - expect(DB::connection('operations')->table('operations')->count())->toBe(1) + expect($repository->completed()->keys()->all())->toBe(['example']) + ->and(DB::connection('operations')->table('operations')->count())->toBe(1) ->and(DB::table('operations')->count())->toBe(0); $migration->down(); @@ -43,31 +41,7 @@ public function handle(\Illuminate\Console\Command $command): void {} config(['operations.connection' => null]); }); -test('successful transactional operations commit their changes and completion together', function () { - Schema::create('examples', function (Blueprint $table) { - $table->id(); - }); - - File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_backfill.php', <<<'PHP' - insert(['id' => 1]); - } - }; - PHP); - - artisan('operations:run')->assertSuccessful(); - - expect(DB::table('examples')->count())->toBe(1) - ->and(DB::table('operations')->count())->toBe(1) - ->and(DB::transactionLevel())->toBe(0); -}); - -test('completion history is read from the writer when the connection has a read replica', function () { +test('repositories use the configured connection and read from its writer', function () { File::ensureDirectoryExists(config('operations.path')); $writer = config('operations.path').'/writer.sqlite'; $reader = config('operations.path').'/reader.sqlite'; @@ -85,27 +59,36 @@ public function handle(\Illuminate\Console\Command $command): void 'operations.connection' => 'operations', ]); - $migration = File::getRequire(__DIR__.'/../../database/migrations/2026_10_02_165413_create_operations_table.php'); - $migration->up(); + foreach (File::glob(__DIR__.'/../../database/migrations/*.php') as $path) { + File::getRequire($path)->up(); + } + + $operation = new class extends Operation + { + use HasCheckpoints; - File::put(config('operations.path').'/2026_10_02_120000_example.php', <<<'PHP' - push('example'); + $this->observed[] = $this->checkpoints->get('cursor', 'initial'); + $this->checkpoints->put('cursor', 'next-page'); + $this->observed[] = $this->checkpoints->get('cursor'); } }; - PHP); - app()->instance('executed', collect()); + $repository = app(OperationRepository::class); + $repository->run('example', $operation, new Command); - artisan('operations:run')->assertSuccessful(); - artisan('operations:run')->expectsOutputToContain('No pending operations.')->assertSuccessful(); + expect($operation->observed)->toBe(['initial', 'next-page']) + ->and($repository->completed()->keys()->all())->toBe(['example']) + ->and(DB::connection('operations')->table('operation_checkpoints')->useWritePdo()->count())->toBe(1) + ->and(DB::table('operations')->count())->toBe(0) + ->and(DB::table('operation_checkpoints')->count())->toBe(0); - expect(app('executed')->all())->toBe(['example']) - ->and(DB::connection('operations')->table('operations')->useWritePdo()->count())->toBe(1); + expect($repository->forget('example'))->toBeTrue() + ->and($repository->completed()->all())->toBe([]) + ->and(DB::connection('operations')->table('operation_checkpoints')->useWritePdo()->count())->toBe(0); config(['operations.connection' => null]); }); diff --git a/tests/Feature/ForgetTest.php b/tests/Feature/ForgetTest.php index 8c30353..f2b48d9 100644 --- a/tests/Feature/ForgetTest.php +++ b/tests/Feature/ForgetTest.php @@ -18,7 +18,10 @@ File::ensureDirectoryExists(config('operations.path')); $path = config('operations.path')."/{$name}.php"; - File::put($path, ' $name]) ->expectsOutputToContain('Forgetting does not undo previous effects.') @@ -27,23 +30,17 @@ ->assertSuccessful(); expect(DB::table('operations')->pluck('name')->all())->toBe([$other]) - ->and(File::get($path))->toBe('and(File::get($path))->toBe(File::get(__DIR__.'/../Fixtures/operations/throw_on_load.php')); }); test('a forgotten operation runs again only when the runner is invoked', function () { $name = '2026_10_02_120000_example'; File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path')."/{$name}.php", <<<'PHP' - push('example'); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/record_execution.php', + config('operations.path')."/{$name}.php", + ); app()->instance('executed', collect()); @@ -69,13 +66,19 @@ public function handle(\Illuminate\Console\Command $command): void app()->instance('env', $environment); $name = '2026_10_02_120000_example'; DB::table('operations')->insert(['name' => $name, 'completed_at' => now()]); + DB::table('operation_checkpoints')->insert([ + 'operation' => $name, + 'key' => 'last_id', + 'value' => '10', + ]); artisan('operations:forget', ['name' => $name]) ->expectsConfirmation("Forget operation [{$name}]?", 'no') ->expectsOutputToContain('Command cancelled.') ->assertFailed(); - expect(DB::table('operations')->pluck('name')->all())->toBe([$name]); + expect(DB::table('operations')->pluck('name')->all())->toBe([$name]) + ->and(DB::table('operation_checkpoints')->pluck('operation')->all())->toBe([$name]); app()->instance('env', 'testing'); })->with(['testing', 'production']); @@ -106,35 +109,12 @@ public function handle(\Illuminate\Console\Command $command): void app()->instance('env', 'testing'); }); -test('forgetting fails when the exact completion record does not exist', function (string $name) { +test('forgetting fails when no completion record or checkpoints exist for the exact name', function (string $name) { DB::table('operations')->insert(['name' => '2026_10_02_120000_example', 'completed_at' => now()]); artisan('operations:forget', ['name' => $name, '--force' => true]) - ->expectsOutputToContain("No completion record found for operation [{$name}].") + ->expectsOutputToContain("No completion record or checkpoints found for operation [{$name}].") ->assertFailed(); expect(DB::table('operations')->pluck('name')->all())->toBe(['2026_10_02_120000_example']); })->with(['missing', 'example', '2026_10_02_120000_example.php']); - -test('forgetting uses the configured operations connection', function () { - config([ - 'database.connections.operations' => [ - 'driver' => 'sqlite', - 'database' => ':memory:', - 'prefix' => '', - ], - 'operations.connection' => 'operations', - ]); - - $migration = File::getRequire(__DIR__.'/../../database/migrations/2026_10_02_165413_create_operations_table.php'); - $migration->up(); - - $name = '2026_10_02_120000_example'; - DB::table('operations')->insert(['name' => $name, 'completed_at' => now()]); - DB::connection('operations')->table('operations')->insert(['name' => $name, 'completed_at' => now()]); - - artisan('operations:forget', ['name' => $name, '--force' => true])->assertSuccessful(); - - expect(DB::connection('operations')->table('operations')->count())->toBe(0) - ->and(DB::table('operations')->pluck('name')->all())->toBe([$name]); -}); diff --git a/tests/Feature/InstallTest.php b/tests/Feature/InstallTest.php index f5beae8..0de5940 100644 --- a/tests/Feature/InstallTest.php +++ b/tests/Feature/InstallTest.php @@ -5,7 +5,7 @@ use function Pest\Laravel\artisan; -test('the migration can be published with a current timestamp', function () { +test('the standard migration can be published without opting into checkpoints', function () { Date::setTestNow('2026-10-02 13:45:00'); artisan('vendor:publish', ['--tag' => 'operations-migrations'])->assertSuccessful(); @@ -13,7 +13,8 @@ $files = File::glob(database_path('migrations/*_create_operations_table.php')); expect($files)->toHaveCount(1) - ->and(basename($files[0]))->toStartWith('2026_10_02_1345'); + ->and(basename($files[0]))->toStartWith('2026_10_02_1345') + ->and(File::glob(database_path('migrations/*_create_operation_checkpoints_table.php')))->toBe([]); }); test('the package configuration can be published', function () { @@ -21,3 +22,19 @@ expect(File::get(config_path('operations.php')))->toBe(File::get(__DIR__.'/../../config/operations.php')); }); + +test('existing installations can publish only the checkpoint migration', function () { + Date::setTestNow('2026-10-05 12:00:00'); + + File::ensureDirectoryExists(database_path('migrations')); + $original = database_path('migrations/2026_10_02_120000_create_operations_table.php'); + File::copy(__DIR__.'/../../database/migrations/2026_10_02_165413_create_operations_table.php', $original); + + artisan('vendor:publish', ['--tag' => 'operations-checkpoints-migration'])->assertSuccessful(); + + $files = File::glob(database_path('migrations/*_create_operation_checkpoints_table.php')); + + expect(File::glob(database_path('migrations/*_create_operations_table.php')))->toBe([$original]) + ->and($files)->toHaveCount(1) + ->and(basename($files[0]))->toStartWith('2026_10_05_1200'); +}); diff --git a/tests/Feature/IsolationTest.php b/tests/Feature/IsolationTest.php index e7e92bb..237459e 100644 --- a/tests/Feature/IsolationTest.php +++ b/tests/Feature/IsolationTest.php @@ -9,11 +9,10 @@ test('isolated invocations skip pending operations when another command holds the lock', function (bool|string $isolated, int $exitCode) { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_backfill.php', <<<'PHP' - true]) ->expectsOutputToContain('Completed 1 operation(s).') diff --git a/tests/Feature/OperationRepositoryTest.php b/tests/Feature/OperationRepositoryTest.php new file mode 100644 index 0000000..e2f6b61 --- /dev/null +++ b/tests/Feature/OperationRepositoryTest.php @@ -0,0 +1,233 @@ +previous = $this->checkpoints->get('cursor'); + $this->checkpoints->put('cursor', 'next-page'); + } + }; + + $first = new CheckpointRepository(DB::connection(), 'first'); + $second = new CheckpointRepository(DB::connection(), 'second'); + $first->put('cursor', 'first-page'); + $second->put('cursor', 'second-page'); + + $repository->run('first', $operation, new Command); + expect($operation->previous)->toBe('first-page'); + + $repository->run('second', $operation, new Command); + expect($operation->previous)->toBe('second-page') + ->and($first->get('cursor'))->toBe('next-page') + ->and($second->get('cursor'))->toBe('next-page') + ->and($repository->completed()->keys()->all())->toBe(['first', 'second']); +}); + +test('a failed chunk rolls back its work and checkpoint without losing earlier progress', function () { + Schema::create('examples', function (Blueprint $table) { + $table->id(); + }); + + $operation = new class extends Operation + { + use HasCheckpoints; + + public function handle(Command $command): void + { + foreach ([1, 2] as $id) { + DB::transaction(function () use ($id) { + DB::table('examples')->insert(['id' => $id]); + $this->checkpoints->put('last_id', $id); + + if ($id === 2) { + throw new RuntimeException('Chunk failed.'); + } + }); + } + } + }; + + expect(fn () => app(OperationRepository::class)->run('backfill', $operation, new Command)) + ->toThrow(RuntimeException::class, 'Chunk failed.'); + + expect(DB::table('examples')->pluck('id')->all())->toBe([1]) + ->and((new CheckpointRepository(DB::connection(), 'backfill'))->get('last_id'))->toBe(1) + ->and(DB::table('operations')->count())->toBe(0); +}); + +test('checkpoints participate in the whole operation transaction', function () { + $checkpoints = new CheckpointRepository(DB::connection(), 'backfill'); + $checkpoints->put('last_id', 10); + + $operation = new class extends Operation implements WithinTransaction + { + use HasCheckpoints; + + public function handle(Command $command): void + { + $this->checkpoints->put(['last_id' => 20, 'finished' => true]); + + throw new RuntimeException('Backfill failed.'); + } + }; + + expect(fn () => app(OperationRepository::class)->run('backfill', $operation, new Command)) + ->toThrow(RuntimeException::class, 'Backfill failed.'); + + expect($checkpoints->get('last_id'))->toBe(10) + ->and($checkpoints->get('finished'))->toBeNull() + ->and(DB::table('operation_checkpoints')->count())->toBe(1) + ->and(DB::table('operations')->count())->toBe(0); +}); + +test('a transaction rolls back database changes when an operation fails', function () { + Schema::create('examples', function (Blueprint $table) { + $table->id(); + }); + + $operation = new class extends Operation implements WithinTransaction + { + public function handle(Command $command): void + { + DB::table('examples')->insert(['id' => 1]); + + throw new RuntimeException('Backfill failed.'); + } + }; + + expect(fn () => app(OperationRepository::class)->run('backfill', $operation, new Command)) + ->toThrow(RuntimeException::class, 'Backfill failed.'); + + expect(DB::table('examples')->count())->toBe(0) + ->and(DB::table('operations')->count())->toBe(0); +}); + +test('a transaction rolls back the operation if recording completion fails', function () { + Schema::create('examples', function (Blueprint $table) { + $table->id(); + }); + + $operation = new class extends Operation implements WithinTransaction + { + public function handle(Command $command): void + { + DB::table('examples')->insert(['id' => 1]); + DB::table('operations')->insert(['name' => 'backfill', 'completed_at' => now()]); + } + }; + + expect(fn () => app(OperationRepository::class)->run('backfill', $operation, new Command)) + ->toThrow(QueryException::class); + + expect(DB::table('examples')->count())->toBe(0) + ->and(DB::table('operations')->count())->toBe(0); +}); + +test('successful transactional operations commit their changes and completion together', function () { + Schema::create('examples', function (Blueprint $table) { + $table->id(); + }); + + $operation = new class extends Operation implements WithinTransaction + { + public function handle(Command $command): void + { + DB::table('examples')->insert(['id' => 1]); + } + }; + + app(OperationRepository::class)->run('backfill', $operation, new Command); + + expect(DB::table('examples')->count())->toBe(1) + ->and(DB::table('operations')->where('name', 'backfill')->value('completed_at'))->not->toBeNull() + ->and(DB::transactionLevel())->toBe(0); +}); + +test('operations can dispatch jobs without becoming queued operations themselves', function () { + Bus::fake(); + + $operation = new class extends Operation + { + public function handle(Command $command): void + { + Bus::dispatch(new ExampleJob); + } + }; + + app(OperationRepository::class)->run('dispatch', $operation, new Command); + + Bus::assertDispatched(ExampleJob::class); + Bus::assertDispatchedTimes(ExampleJob::class, 1); + + expect(DB::table('operations')->where('name', 'dispatch')->value('completed_at'))->not->toBeNull(); +}); + +test('forgetting clears checkpoints for completed and unfinished operations', function (bool $completed) { + $checkpoints = new CheckpointRepository(DB::connection(), 'backfill'); + $other = new CheckpointRepository(DB::connection(), 'other'); + + if ($completed) { + DB::table('operations')->insert(['name' => 'backfill', 'completed_at' => now()]); + } + + $checkpoints->put(['last_id' => 10, 'finished' => true]); + $other->put('last_id', 20); + + $repository = app(OperationRepository::class); + + expect($repository->forget('backfill'))->toBeTrue() + ->and($checkpoints->get('last_id', 0))->toBe(0) + ->and($checkpoints->get('finished'))->toBeNull() + ->and($other->get('last_id'))->toBe(20) + ->and(DB::table('operations')->count())->toBe(0) + ->and($repository->forget('backfill'))->toBeFalse(); +})->with(['completed' => true, 'unfinished' => false]); + +test('forgetting uses the configured operations connection', function () { + config([ + 'database.connections.operations' => [ + 'driver' => 'sqlite', + 'database' => ':memory:', + 'prefix' => '', + ], + 'operations.connection' => 'operations', + ]); + + foreach (File::glob(__DIR__.'/../../database/migrations/*.php') as $path) { + File::getRequire($path)->up(); + } + + $name = '2026_10_02_120000_example'; + DB::table('operations')->insert(['name' => $name, 'completed_at' => now()]); + DB::connection('operations')->table('operations')->insert(['name' => $name, 'completed_at' => now()]); + DB::table('operation_checkpoints')->insert(['operation' => $name, 'key' => 'last_id', 'value' => '10']); + DB::connection('operations')->table('operation_checkpoints')->insert(['operation' => $name, 'key' => 'last_id', 'value' => '20']); + + expect(app(OperationRepository::class)->forget($name))->toBeTrue(); + + expect(DB::connection('operations')->table('operations')->count())->toBe(0) + ->and(DB::connection('operations')->table('operation_checkpoints')->count())->toBe(0) + ->and(DB::table('operations')->pluck('name')->all())->toBe([$name]) + ->and(DB::table('operation_checkpoints')->pluck('operation')->all())->toBe([$name]); +}); diff --git a/tests/Feature/OutputTest.php b/tests/Feature/OutputTest.php index 72ce7be..a4116b6 100644 --- a/tests/Feature/OutputTest.php +++ b/tests/Feature/OutputTest.php @@ -8,26 +8,10 @@ test('operations can print messages tables and progress between their start and completion lines', function () { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_output.php', <<<'PHP' - info('Backfilling company names...'); - $command->table(['Updated'], [[2]]); - - $output = $command->getOutput(); - $output->progressStart(2); - $output->progressAdvance(2); - $output->progressFinish(); - - $command->info('Company names updated.'); - app()->instance('forced', $command->option('force')); - app()->instance('completed_during_operation', \Illuminate\Support\Facades\DB::table('operations')->count()); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/console_output.php', + config('operations.path').'/2026_10_02_120000_output.php', + ); $output = new BufferedOutput; $exitCode = Artisan::call('operations:run', ['--force' => true], $output); @@ -45,22 +29,10 @@ public function handle(\Illuminate\Console\Command $command): void test('quiet mode suppresses both operation output and runner status lines', function () { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_quiet.php', <<<'PHP' - info('Backfilling company names...'); - $command->table(['Updated'], [[1]]); - - $output = $command->getOutput(); - $output->progressStart(1); - $output->progressAdvance(); - $output->progressFinish(); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/quiet_output.php', + config('operations.path').'/2026_10_02_120000_quiet.php', + ); $output = new BufferedOutput; $exitCode = Artisan::call('operations:run', ['--quiet' => true], $output); @@ -72,26 +44,10 @@ public function handle(\Illuminate\Console\Command $command): void test('failed operations never print a successful completion line', function (string $failure, string $exception) { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_failure.php', <<<'PHP' - info('Starting the backfill...'); - - if (app('failure') === 'operation') { - throw new \RuntimeException('Backfill failed.'); - } - - // Cause the runner's completion insert to fail on the unique name. - \Illuminate\Support\Facades\DB::table('operations')->insert([ - 'name' => basename(__FILE__, '.php'), - 'completed_at' => now(), - ]); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/failing_output.php', + config('operations.path').'/2026_10_02_120000_failure.php', + ); app()->instance('failure', $failure); $output = new BufferedOutput; diff --git a/tests/Feature/RunSelectedTest.php b/tests/Feature/RunSelectedTest.php index c33aff1..c96eed0 100644 --- a/tests/Feature/RunSelectedTest.php +++ b/tests/Feature/RunSelectedTest.php @@ -13,19 +13,16 @@ File::ensureDirectoryExists(config('operations.path')); foreach (['2026_10_03_110000_unrelated', '2026_10_03_130000_unrelated'] as $name) { - File::put(config('operations.path')."/{$name}.php", 'info('Selected operation executed.'); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/selected_operation.php', + config('operations.path').'/2026_10_03_120000_selected.php', + ); artisan('operations:run', ['operation' => '2026_10_03_120000_selected']) ->expectsOutputToContain('Selected operation executed.') @@ -49,7 +46,10 @@ public function handle(\Illuminate\Console\Command $command): void File::ensureDirectoryExists(config('operations.path')); foreach (['2026_10_03_120000_backfill', '2026_10_03_130000_backfill'] as $name) { - File::put(config('operations.path')."/{$name}.php", ' artisan('operations:run', [ diff --git a/tests/Feature/RunTest.php b/tests/Feature/RunTest.php index 1350f23..4250e17 100644 --- a/tests/Feature/RunTest.php +++ b/tests/Feature/RunTest.php @@ -2,14 +2,9 @@ use DirectoryTree\Operations\Commands\RunCommand; use DirectoryTree\Operations\Runner; -use DirectoryTree\Operations\Tests\Fixtures\ExampleJob; -use Illuminate\Database\QueryException; -use Illuminate\Database\Schema\Blueprint; use Illuminate\Support\Facades\Artisan; -use Illuminate\Support\Facades\Bus; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\File; -use Illuminate\Support\Facades\Schema; use function Pest\Laravel\artisan; @@ -17,17 +12,10 @@ File::ensureDirectoryExists(config('operations.path')); foreach (['2026_10_02_120002_second', '2026_10_02_120001_first'] as $name) { - File::put(config('operations.path')."/{$name}.php", <<<'PHP' - push(basename(__FILE__, '.php')); - app('ledger_counts')->push(\Illuminate\Support\Facades\DB::table('operations')->count()); - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/record_execution_order.php', + config('operations.path')."/{$name}.php", + ); } app()->instance('executed', collect()); @@ -45,21 +33,10 @@ public function handle(\Illuminate\Console\Command $command): void File::ensureDirectoryExists(config('operations.path')); foreach (['2026_10_02_120001_first', '2026_10_02_120002_second', '2026_10_02_120003_third'] as $name) { - File::put(config('operations.path')."/{$name}.php", <<<'PHP' - push($name); - - if (str_ends_with($name, 'second') && app('should_fail')) { - throw new \RuntimeException('Backfill failed.'); - } - } - }; - PHP); + File::copy( + __DIR__.'/../Fixtures/operations/resumable_operations.php', + config('operations.path')."/{$name}.php", + ); } app()->instance('attempts', collect()); @@ -81,84 +58,12 @@ public function handle(\Illuminate\Console\Command $command): void ])->and(DB::table('operations')->count())->toBe(3); }); -test('a transaction rolls back database changes when an operation fails', function (?string $operation) { - Schema::create('examples', function (Blueprint $table) { - $table->id(); - }); - - File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_backfill.php', <<<'PHP' - insert(['id' => 1]); - - throw new \RuntimeException('Backfill failed.'); - } - }; - PHP); - - expect(fn () => Artisan::call('operations:run', ['operation' => $operation]))->toThrow(RuntimeException::class, 'Backfill failed.'); - - expect(DB::table('examples')->count())->toBe(0) - ->and(DB::table('operations')->count())->toBe(0); -})->with([null, '2026_10_02_120000_backfill']); - -test('a transaction rolls back the operation if recording completion fails', function (?string $operation) { - Schema::create('examples', function (Blueprint $table) { - $table->id(); - }); - - File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_backfill.php', <<<'PHP' - insert(['id' => 1]); - \Illuminate\Support\Facades\DB::table('operations')->insert([ - 'name' => basename(__FILE__, '.php'), - 'completed_at' => now(), - ]); - } - }; - PHP); - - expect(fn () => Artisan::call('operations:run', ['operation' => $operation]))->toThrow(QueryException::class); - - expect(DB::table('examples')->count())->toBe(0) - ->and(DB::table('operations')->count())->toBe(0); -})->with([null, '2026_10_02_120000_backfill']); - -test('operations can dispatch jobs without becoming queued operations themselves', function () { - Bus::fake(); - - File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_dispatch.php', <<<'PHP' - assertSuccessful(); - - Bus::assertDispatched(ExampleJob::class); - Bus::assertDispatchedTimes(ExampleJob::class, 1); - - expect(DB::table('operations')->count())->toBe(1); -}); - test('completed operation files are not loaded on subsequent runs', function (?string $operation) { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120000_old.php', 'insert(['name' => '2026_10_02_120000_old', 'completed_at' => now()]); @@ -168,7 +73,10 @@ public function handle(\Illuminate\Console\Command $command): void test('invalid operation files fail without being recorded', function () { File::ensureDirectoryExists(config('operations.path')); $path = config('operations.path').'/2026_10_02_120000_invalid.php'; - File::put($path, ' app(Runner::class)->run(app(RunCommand::class)))->toThrow(UnexpectedValueException::class); diff --git a/tests/Feature/StatusTest.php b/tests/Feature/StatusTest.php index 8306f74..046400c 100644 --- a/tests/Feature/StatusTest.php +++ b/tests/Feature/StatusTest.php @@ -7,7 +7,10 @@ test('status includes pending operations and completed operations whose files were removed', function () { File::ensureDirectoryExists(config('operations.path')); - File::put(config('operations.path').'/2026_10_02_120002_pending.php', 'insert([ 'name' => '2026_10_02_120001_completed', 'completed_at' => '2026-10-02 12:00:00', diff --git a/tests/Fixtures/operations/console_output.php b/tests/Fixtures/operations/console_output.php new file mode 100644 index 0000000..633a0fb --- /dev/null +++ b/tests/Fixtures/operations/console_output.php @@ -0,0 +1,23 @@ +info('Backfilling company names...'); + $command->table(['Updated'], [[2]]); + + $output = $command->getOutput(); + $output->progressStart(2); + $output->progressAdvance(2); + $output->progressFinish(); + + $command->info('Company names updated.'); + app()->instance('forced', $command->option('force')); + app()->instance('completed_during_operation', DB::table('operations')->count()); + } +}; diff --git a/tests/Fixtures/operations/empty_operation.php b/tests/Fixtures/operations/empty_operation.php new file mode 100644 index 0000000..0652e3f --- /dev/null +++ b/tests/Fixtures/operations/empty_operation.php @@ -0,0 +1,9 @@ +info('Starting the backfill...'); + + if (app('failure') === 'operation') { + throw new RuntimeException('Backfill failed.'); + } + + // Cause the runner's completion insert to fail on the unique name. + DB::table('operations')->insert([ + 'name' => basename(__FILE__, '.php'), + 'completed_at' => now(), + ]); + } +}; diff --git a/tests/Fixtures/operations/invalid_operation.php b/tests/Fixtures/operations/invalid_operation.php new file mode 100644 index 0000000..13436bc --- /dev/null +++ b/tests/Fixtures/operations/invalid_operation.php @@ -0,0 +1,3 @@ +info('Backfilling company names...'); + $command->table(['Updated'], [[1]]); + + $output = $command->getOutput(); + $output->progressStart(1); + $output->progressAdvance(); + $output->progressFinish(); + } +}; diff --git a/tests/Fixtures/operations/record_execution.php b/tests/Fixtures/operations/record_execution.php new file mode 100644 index 0000000..2971b89 --- /dev/null +++ b/tests/Fixtures/operations/record_execution.php @@ -0,0 +1,12 @@ +push('example'); + } +}; diff --git a/tests/Fixtures/operations/record_execution_order.php b/tests/Fixtures/operations/record_execution_order.php new file mode 100644 index 0000000..a66b366 --- /dev/null +++ b/tests/Fixtures/operations/record_execution_order.php @@ -0,0 +1,14 @@ +push(basename(__FILE__, '.php')); + app('ledger_counts')->push(DB::table('operations')->count()); + } +}; diff --git a/tests/Fixtures/operations/resumable_backfill.php b/tests/Fixtures/operations/resumable_backfill.php new file mode 100644 index 0000000..a9ca6c5 --- /dev/null +++ b/tests/Fixtures/operations/resumable_backfill.php @@ -0,0 +1,23 @@ +checkpoints->get('last_id', 0) + 1; $id <= 3; $id++) { + if ($id === 3 && app('should_fail')) { + throw new RuntimeException('Backfill interrupted.'); + } + + DB::table('examples')->insert(['id' => $id]); + $this->checkpoints->put('last_id', $id); + } + } +}; diff --git a/tests/Fixtures/operations/resumable_operations.php b/tests/Fixtures/operations/resumable_operations.php new file mode 100644 index 0000000..a4ad388 --- /dev/null +++ b/tests/Fixtures/operations/resumable_operations.php @@ -0,0 +1,17 @@ +push($name); + + if (str_ends_with($name, 'second') && app('should_fail')) { + throw new RuntimeException('Backfill failed.'); + } + } +}; diff --git a/tests/Fixtures/operations/selected_operation.php b/tests/Fixtures/operations/selected_operation.php new file mode 100644 index 0000000..850c784 --- /dev/null +++ b/tests/Fixtures/operations/selected_operation.php @@ -0,0 +1,12 @@ +info('Selected operation executed.'); + } +}; diff --git a/tests/Fixtures/operations/successful_operation.php b/tests/Fixtures/operations/successful_operation.php new file mode 100644 index 0000000..d38e47f --- /dev/null +++ b/tests/Fixtures/operations/successful_operation.php @@ -0,0 +1,12 @@ +info('Operation executed.'); + } +}; diff --git a/tests/Fixtures/operations/throw_on_load.php b/tests/Fixtures/operations/throw_on_load.php new file mode 100644 index 0000000..aa9a765 --- /dev/null +++ b/tests/Fixtures/operations/throw_on_load.php @@ -0,0 +1,3 @@ +