From 547ded2b674acac515683d064b63506689d72ce7 Mon Sep 17 00:00:00 2001 From: Sean Mancini Date: Tue, 15 Sep 2026 22:43:14 -0400 Subject: [PATCH] Pre create partitions ahead of time instead of a race condition --- functions.php | 77 +++++++++-- setup.php | 27 +++- tests/Security/PartitionTableLockingTest.php | 6 +- tests/Unit/PartitionAheadTest.php | 130 +++++++++++++++++++ 4 files changed, 220 insertions(+), 20 deletions(-) create mode 100644 tests/Unit/PartitionAheadTest.php diff --git a/functions.php b/functions.php index b5ff42e..53ee692 100644 --- a/functions.php +++ b/functions.php @@ -642,9 +642,10 @@ function syslog_traditional_manage() { */ function syslog_partition_manage() { $syslog_deleted = 0; + $ahead_days = syslog_partition_ahead_days(); - // Always create the partition an hour ahead of time - $time = time() + 3600; + // Always create partitions ahead of time to avoid midnight races. + $base_time = time() + 7200; /* * Only run the retention prune when the next partition is ready. @@ -652,21 +653,62 @@ function syslog_partition_manage() { * as the write-path safety net and avoid dropping old partitions * without a replacement. */ - if (syslog_partition_check('syslog', $time)) { - if (syslog_partition_create('syslog', $time)) { - $syslog_deleted = syslog_partition_remove('syslog'); - } + if (syslog_partition_ensure_ahead('syslog', $base_time, $ahead_days)) { + $syslog_deleted = syslog_partition_remove('syslog'); } - if (syslog_partition_check('syslog_removed', $time)) { - if (syslog_partition_create('syslog_removed', $time)) { - $syslog_deleted += syslog_partition_remove('syslog_removed'); - } + if (syslog_partition_ensure_ahead('syslog_removed', $base_time, $ahead_days)) { + $syslog_deleted += syslog_partition_remove('syslog_removed'); } return $syslog_deleted; } +/** + * Return the configured number of future daily partitions to maintain. + * + * @return int Days ahead to pre-create. + */ +function syslog_partition_ahead_days() { + $ahead_days = read_config_option('syslog_partition_ahead_days'); + + if ($ahead_days === '' || !is_numeric($ahead_days)) { + $ahead_days = 3; + } + + $ahead_days = (int) $ahead_days; + + if ($ahead_days < 1 || $ahead_days > 7) { + $ahead_days = 3; + } + + return $ahead_days; +} + +/** + * Ensure concrete partitions exist from the current window through the + * configured future horizon. + * + * @param string $table The table to maintain + * @param int $base_time Base timestamp for the maintenance window + * @param int $ahead_days Number of future days to maintain + * + * @return bool true when all needed partitions already exist or were created. + */ +function syslog_partition_ensure_ahead($table, $base_time, $ahead_days) { + for ($day = 0; $day <= $ahead_days; $day++) { + $time = $base_time + ($day * 86400); + + if (syslog_partition_check($table, $time)) { + if (!syslog_partition_create($table, $time)) { + return false; + } + } + } + + return true; +} + /** * Validate tables that support partition maintenance. * @@ -874,17 +916,19 @@ function syslog_partition_remove($table) { ORDER BY partition_ordinal_position', [$syslogdb_default, $table]); - $days = read_config_option('syslog_retention'); + $days = read_config_option('syslog_retention'); + $ahead_days = syslog_partition_ahead_days(); - syslog_debug("There are currently '" . sizeof($number_of_partitions) . "' Syslog Partitions, We will keep '$days' of them."); + syslog_debug("There are currently '" . sizeof($number_of_partitions) . "' Syslog Partitions, We will keep '$days' retention partition(s) plus '$ahead_days' future partition(s)."); if ($days > 0) { $user_partitions = sizeof($number_of_partitions) - 1; + $keep_partitions = (int) $days + $ahead_days; - if ($user_partitions >= $days) { + if ($user_partitions >= $keep_partitions) { $i = 0; - while ($user_partitions > $days) { + while ($user_partitions > $keep_partitions) { $oldest = $number_of_partitions[$i]; $part_name = $oldest['PARTITION_NAME']; @@ -3399,6 +3443,7 @@ function syslog_process_log($start_time, $deleted, $incoming, $removed, $xferred function syslog_init_variables() { $syslog_retention = read_config_option('syslog_retention'); $alert_retention = read_config_option('syslog_alert_retention'); + $ahead_days = read_config_option('syslog_partition_ahead_days'); if ($syslog_retention == '' || $syslog_retention < 0 || $syslog_retention > 365) { set_config_option('syslog_retention', '30'); @@ -3408,6 +3453,10 @@ function syslog_init_variables() { set_config_option('syslog_alert_retention', '30'); } + if ($ahead_days == '' || !is_numeric($ahead_days) || $ahead_days < 1 || $ahead_days > 7) { + set_config_option('syslog_partition_ahead_days', '3'); + } + if (substr(read_config_option('base_url'), 0, 4) != 'http') { if (read_config_option('force_https') == 'on') { $prefix = 'https://'; diff --git a/setup.php b/setup.php index 0e19b99..fc27913 100644 --- a/setup.php +++ b/setup.php @@ -492,7 +492,7 @@ function syslog_check_upgrade() { } } -function syslog_create_partitioned_syslog_table($engine = 'InnoDB', $days = 30) { +function syslog_create_partitioned_syslog_table($engine = 'InnoDB', $days = 30, $ahead_days = 3) { global $config, $syslogdb_default, $syslog_levels; syslog_connect(); @@ -508,6 +508,12 @@ function syslog_create_partitioned_syslog_table($engine = 'InnoDB', $days = 30) $days = 30; } + if (!is_numeric($ahead_days) || (int) $ahead_days < 1 || (int) $ahead_days > 7) { + $ahead_days = 3; + } + + $ahead_days = (int) $ahead_days; + if (stripos($engine, 'aria') !== false) { $row_format = 'ROW_FORMAT=Page'; } else { @@ -543,7 +549,7 @@ function syslog_create_partitioned_syslog_table($engine = 'InnoDB', $days = 30) * out of the equation: the boundary is always the next UTC midnight * after the labeled day. */ - for ($i = $days; $i >= -1; $i--) { + for ($i = $days; $i >= (0 - $ahead_days); $i--) { $day_epoch = $now - ($i * 86400); $boundary_epoch = (intdiv($day_epoch, 86400) + 1) * 86400; $format = gmdate('Ymd', $day_epoch); @@ -648,7 +654,7 @@ function syslog_setup_table_new($options) { // The syslog table is created partitioned; the helper also selects the // matching ROW_FORMAT for the chosen engine. - syslog_create_partitioned_syslog_table($engine, $options['days']); + syslog_create_partitioned_syslog_table($engine, $options['days'], read_config_option('syslog_partition_ahead_days')); if ($truncate) { syslog_db_execute("DROP TABLE IF EXISTS `$syslogdb_default`.`syslog_alert`"); @@ -1312,6 +1318,21 @@ function syslog_config_settings() { 'default' => '30', 'array' => $syslog_retentions ], + 'syslog_partition_ahead_days' => [ + 'friendly_name' => __('Partition Pre-create Window', 'syslog'), + 'description' => __('This is the number of future daily partitions to maintain for partitioned Syslog tables.', 'syslog'), + 'method' => 'drop_array', + 'default' => '3', + 'array' => [ + '1' => __('%d Day', 1, 'syslog'), + '2' => __('%d Days', 2, 'syslog'), + '3' => __('%d Days', 3, 'syslog'), + '4' => __('%d Days', 4, 'syslog'), + '5' => __('%d Days', 5, 'syslog'), + '6' => __('%d Days', 6, 'syslog'), + '7' => __('%d Days', 7, 'syslog') + ] + ], 'syslog_alert_retention' => [ 'friendly_name' => __('Syslog Alert Retention', 'syslog'), 'description' => __('This is the number of days to keep alert logs.', 'syslog'), diff --git a/tests/Security/PartitionTableLockingTest.php b/tests/Security/PartitionTableLockingTest.php index 743abeb..52d166d 100644 --- a/tests/Security/PartitionTableLockingTest.php +++ b/tests/Security/PartitionTableLockingTest.php @@ -222,7 +222,7 @@ throw new RuntimeException("syslog_partition_create does not log a warning when the partition expression can't be determined."); } - // ---- syslog_partition_manage must exist and drive both tables through check/create/remove ---- + // ---- syslog_partition_manage must exist and drive both tables through ensure/remove ---- $manage_start = strpos($functions, 'function syslog_partition_manage'); @@ -239,8 +239,8 @@ $manage_body = substr($functions, $manage_start, $manage_end - $manage_start); foreach (['syslog', 'syslog_removed'] as $table) { - if (!preg_match('/syslog_partition_create\s*\(\s*\'' . $table . '\'/', $manage_body)) { - throw new RuntimeException("syslog_partition_manage does not call syslog_partition_create('$table')."); + if (!preg_match('/syslog_partition_ensure_ahead\s*\(\s*\'' . $table . '\'/', $manage_body)) { + throw new RuntimeException("syslog_partition_manage does not ensure future partitions for '$table'."); } if (!preg_match('/syslog_partition_remove\s*\(\s*\'' . $table . '\'\s*\)/', $manage_body)) { diff --git a/tests/Unit/PartitionAheadTest.php b/tests/Unit/PartitionAheadTest.php new file mode 100644 index 0000000..f72a523 --- /dev/null +++ b/tests/Unit/PartitionAheadTest.php @@ -0,0 +1,130 @@ +toBe(3); + + foreach (['0', '8', '-1', 'abc'] as $invalid) { + test_override('read_config_option', function ($name) use ($invalid) { + return $name === 'syslog_partition_ahead_days' ? $invalid : ''; + }); + + expect(syslog_partition_ahead_days())->toBe(3); + } + + test_override('read_config_option', function ($name) { + return $name === 'syslog_partition_ahead_days' ? '7' : ''; + }); + + expect(syslog_partition_ahead_days())->toBe(7); +}); + +it('ensures partitions sequentially through the configured future horizon', function () { + syslog_load_plugin_source('functions.php'); + + $created = []; + $last_partition = '20260914'; + $base_time = gmmktime(22, 30, 0, 9, 15, 2026); + + $GLOBALS['syslogdb_default'] = 'syslog'; + + test_override('syslog_db_fetch_cell_prepared', function ($sql) use (&$last_partition) { + if (str_contains($sql, 'GET_LOCK') || str_contains($sql, 'RELEASE_LOCK')) { + return 1; + } + + if (str_contains($sql, 'PARTITION_NAME')) { + return 'd' . $last_partition; + } + + return ''; + }); + + test_override('syslog_db_fetch_row_prepared', function ($sql, $params = []) { + if (str_contains($sql, 'information_schema')) { + return []; + } + + if (str_contains($sql, 'SHOW CREATE TABLE')) { + return ['Create Table' => 'CREATE TABLE `syslog` (`logtime` timestamp NOT NULL) PARTITION BY RANGE (UNIX_TIMESTAMP(logtime))']; + } + + return []; + }); + + test_override('syslog_db_execute_prepared', function ($sql) use (&$created, &$last_partition) { + if (preg_match('/PARTITION (d\d{8}) VALUES LESS THAN/', $sql, $matches)) { + $created[] = $matches[1]; + $last_partition = substr($matches[1], 1); + } + + return true; + }); + + expect(syslog_partition_ensure_ahead('syslog', $base_time, 3))->toBeTrue(); + expect($created)->toBe([ + 'd20260915', + 'd20260916', + 'd20260917', + 'd20260918' + ]); +}); + +it('keeps retention plus future partitions when pruning', function () { + syslog_load_plugin_source('functions.php'); + + $GLOBALS['syslogdb_default'] = 'syslog'; + $dropped = []; + + test_override('read_config_option', function ($name) { + if ($name === 'syslog_retention') { + return '3'; + } + + if ($name === 'syslog_partition_ahead_days') { + return '3'; + } + + return ''; + }); + + test_override('syslog_db_fetch_cell_prepared', function () { + return 1; + }); + + test_override('syslog_db_fetch_assoc_prepared', function () { + return [ + ['PARTITION_NAME' => 'd20260910'], + ['PARTITION_NAME' => 'd20260911'], + ['PARTITION_NAME' => 'd20260912'], + ['PARTITION_NAME' => 'd20260913'], + ['PARTITION_NAME' => 'd20260914'], + ['PARTITION_NAME' => 'd20260915'], + ['PARTITION_NAME' => 'd20260916'], + ['PARTITION_NAME' => 'dMaxValue'] + ]; + }); + + test_override('syslog_db_execute_prepared', function ($sql) use (&$dropped) { + if (preg_match('/DROP PARTITION `([^`]+)`/', $sql, $matches)) { + $dropped[] = $matches[1]; + } + + return true; + }); + + expect(syslog_partition_remove('syslog'))->toBe(1); + expect($dropped)->toBe(['d20260910']); +});