diff --git a/CHANGELOG.md b/CHANGELOG.md index 95fd37e1733..a99b478859a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## master / unreleased * [FEATURE] Engine: Add `-querier.selector-batch-size` and `-ruler.selector-batch-size` flags to configure series batching in the Thanos promQL engine. 0 disables batching. #7763 * [CHANGE] Ingester: Formally deprecate `-blocks-storage.tsdb.max-exemplars`, scheduled for removal in v1.24.0. Use the per-tenant `max_exemplars` limit instead. The flag still works as the global fallback when `max_exemplars` is 0, but setting it now logs a warning and increments `deprecated_flags_inuse_total`. #7793 -* [CHANGE] Querier: Make query time range configurations per-tenant: `query_ingesters_within`, `query_store_after`, and `shuffle_sharding_ingesters_lookback_period`. Uses `model.Duration` instead of `time.Duration` to support serialization but has minimum unit of 1ms (nanoseconds/microseconds not supported). #7160 +* [CHANGE] Querier: Make query time range configurations per-tenant: `query_ingesters_within`, `query_store_after`, `close_idle_tsdb_timeout` and `shuffle_sharding_ingesters_lookback_period`. Uses `model.Duration` instead of `time.Duration` to support serialization but has minimum unit of 1ms (nanoseconds/microseconds not supported). #7323 #7813 * [CHANGE] Cache: Setting `-blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl` to 0 will disable the bucket-index cache. #7446 * [CHANGE] HA Tracker: Move `-distributor.ha-tracker.failover-timeout` from a global config to a per-tenant runtime config. The flag name and default value (30s) remain the same. #7481 * [FEATURE] Parquet: Support sharded parquet file conversion and querying. #7610 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 3dc3c704b21..6320f9912a0 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -2242,15 +2242,6 @@ blocks_storage: # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown [flush_blocks_on_shutdown: | default = false] - # If TSDB has not received any data for this duration, and all blocks from - # TSDB have been shipped, TSDB is closed and deleted from local disk. If set - # to positive value, this value must be greater than - # -limits.query-ingesters-within flag to make sure that TSDB is not closed - # prematurely, which could cause partial query results. 0 or negative value - # disables closing of idle TSDB. - # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout - [close_idle_tsdb_timeout: | default = 0s] - # The size of the in-memory queue used before flushing chunks to the disk. # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: | default = 0] diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 8c43bcb4653..739a0ebd37c 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -2295,15 +2295,6 @@ blocks_storage: # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown [flush_blocks_on_shutdown: | default = false] - # If TSDB has not received any data for this duration, and all blocks from - # TSDB have been shipped, TSDB is closed and deleted from local disk. If set - # to positive value, this value must be greater than - # -limits.query-ingesters-within flag to make sure that TSDB is not closed - # prematurely, which could cause partial query results. 0 or negative value - # disables closing of idle TSDB. - # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout - [close_idle_tsdb_timeout: | default = 0s] - # The size of the in-memory queue used before flushing chunks to the disk. # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: | default = 0] diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 2b11f47f82d..beee5dfcbae 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2938,15 +2938,6 @@ tsdb: # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown [flush_blocks_on_shutdown: | default = false] - # If TSDB has not received any data for this duration, and all blocks from - # TSDB have been shipped, TSDB is closed and deleted from local disk. If set - # to positive value, this value must be greater than - # -limits.query-ingesters-within flag to make sure that TSDB is not closed - # prematurely, which could cause partial query results. 0 or negative value - # disables closing of idle TSDB. - # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout - [close_idle_tsdb_timeout: | default = 0s] - # The size of the in-memory queue used before flushing chunks to the disk. # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: | default = 0] @@ -4778,10 +4769,18 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # Lookback period for shuffle sharding of ingesters. This is a per-tenant limit # that can be overridden in the runtime configuration. Should be greater than or -# equal to query-ingesters-within. +# equal to query-store-after and query-ingesters-within. # CLI flag: -limits.shuffle-sharding-ingesters-lookback-period [shuffle_sharding_ingesters_lookback_period: | default = 0s] +# If TSDB has not received any data for this duration, and all blocks from TSDB +# have been shipped, TSDB is closed and deleted from local disk. This is a +# per-tenant limit that can be overridden in the runtime configuration. Should +# be greater than or equal to -limits.query-ingesters-within to prevent +# premature TSDB closure. 0 to disable. +# CLI flag: -limits.close-idle-tsdb-timeout +[close_idle_tsdb_timeout: | default = 0s] + # The maximum number of rows that can be fetched when querying parquet storage. # Each row maps to a series in a parquet file. This limit applies before # materializing chunks. 0 to disable. diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index 658388e5ebb..36108723e19 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -10,6 +10,7 @@ import ( "reflect" "slices" "strings" + "time" "github.com/go-kit/log" "github.com/go-kit/log/level" @@ -220,7 +221,7 @@ func (c *Config) Validate(log log.Logger) error { if err := c.LimitsConfig.Validate(c.NameValidationScheme, c.Distributor.ShardByAllLabels, c.Ingester.ActiveSeriesMetricsEnabled, c.Distributor.HATrackerConfig.UpdateTimeout, c.Distributor.HATrackerConfig.UpdateTimeoutJitterMax); err != nil { return errors.Wrap(err, "invalid limits config") } - if err := c.LimitsConfig.ValidateQueryLimits("default", c.BlocksStorage.TSDB.CloseIdleTSDBTimeout); err != nil { + if err := c.LimitsConfig.ValidateQueryLimits("default", time.Duration(c.LimitsConfig.CloseIdleTSDBTimeout)); err != nil { return errors.Wrap(err, "invalid query routing config") } if err := c.ResourceMonitor.Validate(); err != nil { diff --git a/pkg/cortex/runtime_config.go b/pkg/cortex/runtime_config.go index 18d62a49f07..ecdeb5d1fb4 100644 --- a/pkg/cortex/runtime_config.go +++ b/pkg/cortex/runtime_config.go @@ -5,6 +5,7 @@ import ( "io" "net/http" "strings" + "time" "gopkg.in/yaml.v2" @@ -80,7 +81,7 @@ func (l runtimeConfigLoader) load(r io.Reader) (any, error) { if err := ul.Validate(l.cfg.NameValidationScheme, l.cfg.Distributor.ShardByAllLabels, l.cfg.Ingester.ActiveSeriesMetricsEnabled, l.cfg.Distributor.HATrackerConfig.UpdateTimeout, l.cfg.Distributor.HATrackerConfig.UpdateTimeoutJitterMax); err != nil { return nil, err } - if err := ul.ValidateQueryLimits(userID, l.cfg.BlocksStorage.TSDB.CloseIdleTSDBTimeout); err != nil { + if err := ul.ValidateQueryLimits(userID, time.Duration(ul.CloseIdleTSDBTimeout)); err != nil { return nil, err } } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 933a2e31835..40aa9b3ea71 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1016,7 +1016,9 @@ func (i *Ingester) starting(ctx context.Context) error { servs = append(servs, shippingService) } - if i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout > 0 { + // The close-idle TSDB timeout is now a per-tenant limit, so the timer always runs; + // closeAndDeleteIdleUserTSDBs decides per tenant whether any TSDB is actually closed. + { interval := i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval if interval == 0 { interval = cortex_tsdb.DefaultCloseIdleTSDBInterval @@ -3539,6 +3541,17 @@ func (i *Ingester) closeAndDeleteIdleUserTSDBs(ctx context.Context) error { return nil } + // A non-positive close-idle timeout disables idle-based closing for this tenant, + // so we skip the idle check entirely (and don't record an idleTsdbChecks result, + // since no check is performed). A tenant marked for deletion is the one exception: + // its local TSDB must still be closed and deleted regardless of the timeout. + if i.limits.CloseIdleTSDBTimeout(userID) <= 0 { + userDB, err := i.getTSDB(userID) + if err != nil || userDB == nil || !userDB.deletionMarkFound.Load() { + continue + } + } + result := i.closeAndDeleteUserTSDBIfIdle(userID) i.TSDBState.idleTsdbChecks.WithLabelValues(string(result)).Inc() @@ -3569,7 +3582,9 @@ func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string) tsdbCloseCheckRes return tsdbShippingDisabled } - if result := userDB.shouldCloseTSDB(i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout); !result.shouldClose() { + closeIdleTSDBTimeout := i.limits.CloseIdleTSDBTimeout(userID) + + if result := userDB.shouldCloseTSDB(closeIdleTSDBTimeout); !result.shouldClose() { return result } @@ -3587,7 +3602,7 @@ func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string) tsdbCloseCheckRes // Verify again, things may have changed during the checks and pushes. tenantDeleted := false - if result := userDB.shouldCloseTSDB(i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout); !result.shouldClose() { + if result := userDB.shouldCloseTSDB(closeIdleTSDBTimeout); !result.shouldClose() { // This will also change TSDB state back to active (via defer above). return result } else if result == tsdbTenantMarkedForDeletion { diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index c56e79d25b8..8bbcc813ef7 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5264,7 +5264,6 @@ func TestIngester_closeAndDeleteUserTSDBIfIdle_shouldNotCloseTSDBIfShippingIsInP func TestIngester_closingAndOpeningTsdbConcurrently(t *testing.T) { ctx := context.Background() cfg := defaultIngesterTestConfig(t) - cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 0 // Will not run the loop, but will allow us to close any TSDB fast. // Create ingester i, err := prepareIngesterWithBlocksStorage(t, cfg, prometheus.NewRegistry()) @@ -5317,7 +5316,6 @@ func TestIngester_idleCloseEmptyTSDB(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.BlocksStorageConfig.TSDB.ShipInterval = 1 * time.Minute cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Minute - cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 0 // Will not run the loop, but will allow us to close any TSDB fast. // Create ingester i, err := prepareIngesterWithBlocksStorage(t, cfg, prometheus.NewRegistry()) @@ -5365,7 +5363,6 @@ func TestIngester_ReadNotFailWhenTSDBIsBeingDeleted(t *testing.T) { t.Run(name, func(t *testing.T) { ctx := context.Background() cfg := defaultIngesterTestConfig(t) - cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 0 // Will not run the loop, but will allow us to close any TSDB fast. cfg.BlocksStorageConfig.TSDB.KeepUserTSDBOpenOnShutdown = true // Create ingester @@ -6031,13 +6028,16 @@ func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) { cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1 cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Second cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout = 1 * time.Second - cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 1 * time.Second cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval = 100 * time.Millisecond + limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true + limits.CloseIdleTSDBTimeout = model.Duration(1 * time.Second) + r := prometheus.NewRegistry() // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, cfg, r) + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", r) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) @@ -6142,6 +6142,88 @@ func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) { `), metricsToCheck...)) } +func TestIngesterCloseIdleTSDB_PerTenantTimeout(t *testing.T) { + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.BlocksStorageConfig.TSDB.ShipInterval = 1 * time.Second + cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1 + cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Second + cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout = 1 * time.Second + cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval = 100 * time.Millisecond + + limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true + // Per-tenant limit enables close-idle for this tenant + limits.CloseIdleTSDBTimeout = model.Duration(1 * time.Second) + + tenantLimits := newMockTenantLimits(map[string]*validation.Limits{userID: &limits}) + + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, "", prometheus.NewRegistry()) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + t.Cleanup(func() { + require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) + }) + + test.Poll(t, 1*time.Second, ring.ACTIVE, func() any { + return i.lifecycler.GetState() + }) + + pushSingleSampleWithMetadata(t, i) + + require.Equal(t, int64(1), i.TSDBState.seriesCount.Load()) + + // Wait until TSDB has been closed via the per-tenant timeout. + test.Poll(t, 10*time.Second, 0, func() any { + i.stoppedMtx.Lock() + defer i.stoppedMtx.Unlock() + return len(i.TSDBState.dbs) + }) + + require.Greater(t, testutil.ToFloat64(i.TSDBState.idleTsdbChecks.WithLabelValues(string(tsdbIdleClosed))), float64(0)) +} + +func TestIngesterCloseIdleTSDB_DisabledWhenZero(t *testing.T) { + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.BlocksStorageConfig.TSDB.ShipInterval = 1 * time.Second + cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1 + cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Second + cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout = 1 * time.Second + cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval = 100 * time.Millisecond + + limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true + limits.CloseIdleTSDBTimeout = 0 // Disabled + + tenantLimits := newMockTenantLimits(map[string]*validation.Limits{userID: &limits}) + + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, "", prometheus.NewRegistry()) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + t.Cleanup(func() { + require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) + }) + + test.Poll(t, 1*time.Second, ring.ACTIVE, func() any { + return i.lifecycler.GetState() + }) + + pushSingleSampleWithMetadata(t, i) + + require.Equal(t, int64(1), i.TSDBState.seriesCount.Load()) + + // Wait a bit and confirm TSDB is NOT closed (timeout is disabled). + time.Sleep(500 * time.Millisecond) + + i.stoppedMtx.Lock() + numDBs := len(i.TSDBState.dbs) + i.stoppedMtx.Unlock() + require.Equal(t, 1, numDBs) +} + func verifyCompactedHead(t *testing.T, i *Ingester, expected bool) { db, err := i.getTSDB(userID) require.NoError(t, err) @@ -7768,7 +7850,6 @@ func TestIngester_UpdateLabelSetMetrics(t *testing.T) { func TestIngesterPanicHandling(t *testing.T) { ctx := context.Background() cfg := defaultIngesterTestConfig(t) - cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 0 // Will not run the loop, but will allow us to close any TSDB fast. cfg.BlocksStorageConfig.TSDB.KeepUserTSDBOpenOnShutdown = true // Create ingester diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 3ae454d19be..91b304eb395 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -157,7 +157,6 @@ type TSDBConfig struct { WALCompressionType string `yaml:"wal_compression_type"` WALSegmentSizeBytes int `yaml:"wal_segment_size_bytes"` FlushBlocksOnShutdown bool `yaml:"flush_blocks_on_shutdown"` - CloseIdleTSDBTimeout time.Duration `yaml:"close_idle_tsdb_timeout"` // The size of the in-memory queue used before flushing chunks to the disk. HeadChunksWriteQueueSize int `yaml:"head_chunks_write_queue_size"` @@ -207,7 +206,6 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) { f.StringVar(&cfg.WALCompressionType, "blocks-storage.tsdb.wal-compression-type", "", "TSDB WAL type. Supported values are: 'snappy', 'zstd' and '' (disable compression)") f.IntVar(&cfg.WALSegmentSizeBytes, "blocks-storage.tsdb.wal-segment-size-bytes", wlog.DefaultSegmentSize, "TSDB WAL segments files max size (bytes).") f.BoolVar(&cfg.FlushBlocksOnShutdown, "blocks-storage.tsdb.flush-blocks-on-shutdown", false, "True to flush blocks to storage on shutdown. If false, incomplete blocks will be reused after restart.") - f.DurationVar(&cfg.CloseIdleTSDBTimeout, "blocks-storage.tsdb.close-idle-tsdb-timeout", 0, "If TSDB has not received any data for this duration, and all blocks from TSDB have been shipped, TSDB is closed and deleted from local disk. If set to positive value, this value must be greater than -limits.query-ingesters-within flag to make sure that TSDB is not closed prematurely, which could cause partial query results. 0 or negative value disables closing of idle TSDB.") f.IntVar(&cfg.HeadChunksWriteQueueSize, "blocks-storage.tsdb.head-chunks-write-queue-size", chunks.DefaultWriteQueueSize, "The size of the in-memory queue used before flushing chunks to the disk.") f.IntVar(&cfg.MaxExemplars, "blocks-storage.tsdb.max-exemplars", 0, "Deprecated (use the per-tenant max_exemplars limit instead) and will be removed in v1.24.0: the global fallback for the maximum number of exemplars stored in TSDB, used only when the per-tenant max_exemplars limit is 0. 0 or less means exemplars are disabled.") f.BoolVar(&cfg.MemorySnapshotOnShutdown, "blocks-storage.tsdb.memory-snapshot-on-shutdown", false, "True to enable snapshotting of in-memory TSDB data on disk when shutting down.") diff --git a/pkg/util/validation/exporter_test.go b/pkg/util/validation/exporter_test.go index 6067ed96067..a27efc26a98 100644 --- a/pkg/util/validation/exporter_test.go +++ b/pkg/util/validation/exporter_test.go @@ -48,6 +48,7 @@ func TestOverridesExporter_withConfig(t *testing.T) { cortex_overrides{limit_name="alertmanager_max_templates_count",user="tenant-a"} 0 cortex_overrides{limit_name="alertmanager_notification_rate_limit",user="tenant-a"} 0 cortex_overrides{limit_name="alertmanager_receivers_firewall_block_private_addresses",user="tenant-a"} 0 + cortex_overrides{limit_name="close_idle_tsdb_timeout",user="tenant-a"} 0 cortex_overrides{limit_name="compactor_blocks_retention_period",user="tenant-a"} 0 cortex_overrides{limit_name="compactor_partition_index_size_bytes",user="tenant-a"} 6.8719476736e+10 cortex_overrides{limit_name="compactor_partition_series_count",user="tenant-a"} 0 diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 98b8175f0aa..0a4bba580ac 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -206,6 +206,7 @@ type Limits struct { // "now - queryStoreAfter" so that most recent blocks are not queried. QueryStoreAfter model.Duration `yaml:"query_store_after" json:"query_store_after"` ShuffleShardingIngestersLookbackPeriod model.Duration `yaml:"shuffle_sharding_ingesters_lookback_period" json:"shuffle_sharding_ingesters_lookback_period"` + CloseIdleTSDBTimeout model.Duration `yaml:"close_idle_tsdb_timeout" json:"close_idle_tsdb_timeout"` // Parquet Queryable enforced limits. ParquetMaxFetchedRowCount int `yaml:"parquet_max_fetched_row_count" json:"parquet_max_fetched_row_count"` @@ -335,7 +336,10 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.Var(&l.QueryStoreAfter, "limits.query-store-after", "Minimum age of data before querying the long-term storage. Queries for data younger than this will only query ingesters. This is a per-tenant limit that can be overridden in the runtime configuration.") _ = l.ShuffleShardingIngestersLookbackPeriod.Set("0") - f.Var(&l.ShuffleShardingIngestersLookbackPeriod, "limits.shuffle-sharding-ingesters-lookback-period", "Lookback period for shuffle sharding of ingesters. This is a per-tenant limit that can be overridden in the runtime configuration. Should be greater than or equal to query-ingesters-within.") + f.Var(&l.ShuffleShardingIngestersLookbackPeriod, "limits.shuffle-sharding-ingesters-lookback-period", "Lookback period for shuffle sharding of ingesters. This is a per-tenant limit that can be overridden in the runtime configuration. Should be greater than or equal to query-store-after and query-ingesters-within.") + + _ = l.CloseIdleTSDBTimeout.Set("0") + f.Var(&l.CloseIdleTSDBTimeout, "limits.close-idle-tsdb-timeout", "If TSDB has not received any data for this duration, and all blocks from TSDB have been shipped, TSDB is closed and deleted from local disk. This is a per-tenant limit that can be overridden in the runtime configuration. Should be greater than or equal to -limits.query-ingesters-within to prevent premature TSDB closure. 0 to disable.") f.Var(&l.MaxQueryLength, "store.max-query-length", "Limit the query time range (end - start time of range query parameter and max - min of data fetched time range). This limit is enforced in the query-frontend and ruler (on the received query). 0 to disable.") f.Var(&l.MaxQueryLookback, "querier.max-query-lookback", "Limit how long back data (series and metadata) can be queried, up until duration ago. This limit is enforced in the query-frontend, querier and ruler. If the requested time range is outside the allowed range, the request will not fail but will be manipulated to only query data within the allowed time range. 0 to disable.") @@ -468,8 +472,8 @@ func (l *Limits) ValidateQueryLimits(userID string, closeIdleTSDBTimeout time.Du queryStoreAfter := time.Duration(l.QueryStoreAfter) shuffleShardingLookback := time.Duration(l.ShuffleShardingIngestersLookbackPeriod) - if queryIngestersWithin > 0 && closeIdleTSDBTimeout > 0 && queryIngestersWithin >= closeIdleTSDBTimeout { - return fmt.Errorf("tenant %s: query_ingesters_within (%s) must be less than close_idle_tsdb_timeout (%s)", + if queryIngestersWithin > 0 && closeIdleTSDBTimeout > 0 && queryIngestersWithin > closeIdleTSDBTimeout { + return fmt.Errorf("tenant %s: query_ingesters_within (%s) must be less than or equal to close_idle_tsdb_timeout (%s)", userID, queryIngestersWithin, closeIdleTSDBTimeout) } @@ -483,6 +487,11 @@ func (l *Limits) ValidateQueryLimits(userID string, closeIdleTSDBTimeout time.Du userID, shuffleShardingLookback, queryStoreAfter) } + if queryIngestersWithin > 0 && shuffleShardingLookback > 0 && shuffleShardingLookback < queryIngestersWithin { + return fmt.Errorf("tenant %s: shuffle_sharding_ingesters_lookback_period (%s) is less than query_ingesters_within (%s)", + userID, shuffleShardingLookback, queryIngestersWithin) + } + return nil } @@ -1287,6 +1296,10 @@ func (o *Overrides) ShuffleShardingIngestersLookbackPeriod(userID string) time.D return time.Duration(o.GetOverridesForUser(userID).ShuffleShardingIngestersLookbackPeriod) } +func (o *Overrides) CloseIdleTSDBTimeout(userID string) time.Duration { + return time.Duration(o.GetOverridesForUser(userID).CloseIdleTSDBTimeout) +} + // GetOverridesForUser returns the per-tenant limits with overrides. func (o *Overrides) GetOverridesForUser(userID string) *Limits { if o.tenantLimits != nil { diff --git a/pkg/util/validation/limits_test.go b/pkg/util/validation/limits_test.go index 6c5813e80ba..5b3e12fa785 100644 --- a/pkg/util/validation/limits_test.go +++ b/pkg/util/validation/limits_test.go @@ -1069,19 +1069,19 @@ func TestLimits_ValidateQueryLimits(t *testing.T) { closeIdleTSDBTimeout: 0, expectedError: "", }, - "invalid: queryIngestersWithin >= closeIdleTSDBTimeout": { + "valid: queryIngestersWithin == closeIdleTSDBTimeout": { queryIngestersWithin: 25 * time.Hour, queryStoreAfter: 24 * time.Hour, shuffleShardingIngestersLookbackPeriod: 25 * time.Hour, closeIdleTSDBTimeout: 25 * time.Hour, - expectedError: "query_ingesters_within (25h0m0s) must be less than close_idle_tsdb_timeout (25h0m0s)", + expectedError: "", }, "invalid: queryIngestersWithin > closeIdleTSDBTimeout": { queryIngestersWithin: 26 * time.Hour, queryStoreAfter: 24 * time.Hour, shuffleShardingIngestersLookbackPeriod: 26 * time.Hour, closeIdleTSDBTimeout: 25 * time.Hour, - expectedError: "query_ingesters_within (26h0m0s) must be less than close_idle_tsdb_timeout (25h0m0s)", + expectedError: "query_ingesters_within (26h0m0s) must be less than or equal to close_idle_tsdb_timeout (25h0m0s)", }, "invalid: queryStoreAfter >= queryIngestersWithin": { queryIngestersWithin: 24 * time.Hour, @@ -1104,12 +1104,12 @@ func TestLimits_ValidateQueryLimits(t *testing.T) { closeIdleTSDBTimeout: 26 * time.Hour, expectedError: "shuffle_sharding_ingesters_lookback_period (20h0m0s) is less than query_store_after (24h0m0s)", }, - "valid: shuffleShardingLookback between queryStoreAfter and queryIngestersWithin": { + "invalid: shuffleShardingLookback < queryIngestersWithin": { queryIngestersWithin: 25 * time.Hour, queryStoreAfter: 20 * time.Hour, shuffleShardingIngestersLookbackPeriod: 22 * time.Hour, closeIdleTSDBTimeout: 26 * time.Hour, - expectedError: "", + expectedError: "shuffle_sharding_ingesters_lookback_period (22h0m0s) is less than query_ingesters_within (25h0m0s)", }, "boundary: queryIngestersWithin exactly 1ms less than closeIdleTSDBTimeout": { queryIngestersWithin: 25*time.Hour - time.Millisecond, @@ -1125,10 +1125,10 @@ func TestLimits_ValidateQueryLimits(t *testing.T) { closeIdleTSDBTimeout: 26 * time.Hour, expectedError: "", }, - "boundary: shuffleShardingLookback exactly equal to queryStoreAfter": { + "boundary: shuffleShardingLookback exactly equal to queryIngestersWithin": { queryIngestersWithin: 25 * time.Hour, queryStoreAfter: 24 * time.Hour, - shuffleShardingIngestersLookbackPeriod: 24 * time.Hour, + shuffleShardingIngestersLookbackPeriod: 25 * time.Hour, closeIdleTSDBTimeout: 26 * time.Hour, expectedError: "", }, @@ -1254,16 +1254,16 @@ func TestQueryLimits_TenantOverridesValidation(t *testing.T) { tenantID: "invalid-tenant", expectedError: "query_store_after (25h0m0s) must be less than query_ingesters_within (24h0m0s)", }, - "invalid tenant: queryIngestersWithin >= closeIdleTSDBTimeout": { + "invalid tenant: queryIngestersWithin > closeIdleTSDBTimeout": { tenantLimits: map[string]*Limits{ "invalid-tenant": { - QueryIngestersWithin: model.Duration(26 * time.Hour), + QueryIngestersWithin: model.Duration(27 * time.Hour), QueryStoreAfter: model.Duration(24 * time.Hour), - ShuffleShardingIngestersLookbackPeriod: model.Duration(26 * time.Hour), + ShuffleShardingIngestersLookbackPeriod: model.Duration(27 * time.Hour), }, }, tenantID: "invalid-tenant", - expectedError: "query_ingesters_within (26h0m0s) must be less than close_idle_tsdb_timeout (26h0m0s)", + expectedError: "query_ingesters_within (27h0m0s) must be less than or equal to close_idle_tsdb_timeout (26h0m0s)", }, "invalid tenant: shuffleShardingLookback < queryStoreAfter": { tenantLimits: map[string]*Limits{ @@ -1295,3 +1295,77 @@ func TestQueryLimits_TenantOverridesValidation(t *testing.T) { }) } } + +func TestQueryLimits_PerTenantCloseIdleTSDBTimeout(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + tenantCloseIdleTSDBTimeout time.Duration + queryIngestersWithin time.Duration + expectedError string + }{ + "tenant override allows larger query_ingesters_within": { + tenantCloseIdleTSDBTimeout: 30 * time.Hour, + queryIngestersWithin: 27 * time.Hour, + expectedError: "", + }, + "tenant override allows query_ingesters_within equal to close_idle_tsdb_timeout": { + tenantCloseIdleTSDBTimeout: 28 * time.Hour, + queryIngestersWithin: 28 * time.Hour, + expectedError: "", + }, + "tenant override is validated against its own close_idle_tsdb_timeout": { + tenantCloseIdleTSDBTimeout: 28 * time.Hour, + queryIngestersWithin: 29 * time.Hour, + expectedError: "query_ingesters_within (29h0m0s) must be less than or equal to close_idle_tsdb_timeout (28h0m0s)", + }, + "zero close_idle_tsdb_timeout skips validation": { + tenantCloseIdleTSDBTimeout: 0, + queryIngestersWithin: 100 * time.Hour, + expectedError: "", + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + limits := Limits{ + QueryIngestersWithin: model.Duration(testData.queryIngestersWithin), + CloseIdleTSDBTimeout: model.Duration(testData.tenantCloseIdleTSDBTimeout), + } + + err := limits.ValidateQueryLimits("test-tenant", time.Duration(limits.CloseIdleTSDBTimeout)) + + if testData.expectedError == "" { + assert.NoError(t, err, "expected no error but got: %v", err) + } else { + assert.Error(t, err, "expected error but got none") + if err != nil { + assert.Contains(t, err.Error(), testData.expectedError, "error message mismatch") + } + } + }) + } +} + +func TestCloseIdleTSDBTimeout_OverridesAccessor(t *testing.T) { + t.Parallel() + + tenantLimits := map[string]*Limits{ + "tenant-with-override": { + CloseIdleTSDBTimeout: model.Duration(30 * time.Hour), + }, + "tenant-with-zero": { + CloseIdleTSDBTimeout: 0, + }, + } + + defaults := Limits{ + CloseIdleTSDBTimeout: 0, + } + + ov := NewOverrides(defaults, newMockTenantLimits(tenantLimits)) + + assert.Equal(t, 30*time.Hour, ov.CloseIdleTSDBTimeout("tenant-with-override")) + assert.Equal(t, time.Duration(0), ov.CloseIdleTSDBTimeout("tenant-with-zero")) + assert.Equal(t, time.Duration(0), ov.CloseIdleTSDBTimeout("unknown-tenant")) +} diff --git a/schemas/cortex-config-schema.json b/schemas/cortex-config-schema.json index 939e9134dbd..e851e0fb428 100644 --- a/schemas/cortex-config-schema.json +++ b/schemas/cortex-config-schema.json @@ -3438,13 +3438,6 @@ "type": "array", "x-cli-flag": "blocks-storage.tsdb.block-ranges-period" }, - "close_idle_tsdb_timeout": { - "default": "0s", - "description": "If TSDB has not received any data for this duration, and all blocks from TSDB have been shipped, TSDB is closed and deleted from local disk. If set to positive value, this value must be greater than -limits.query-ingesters-within flag to make sure that TSDB is not closed prematurely, which could cause partial query results. 0 or negative value disables closing of idle TSDB.", - "type": "string", - "x-cli-flag": "blocks-storage.tsdb.close-idle-tsdb-timeout", - "x-format": "duration" - }, "dir": { "default": "tsdb", "description": "Local directory to store TSDBs in the ingesters.", @@ -5619,6 +5612,13 @@ "type": "boolean", "x-cli-flag": "alertmanager.receivers-firewall-block-private-addresses" }, + "close_idle_tsdb_timeout": { + "default": "0s", + "description": "If TSDB has not received any data for this duration, and all blocks from TSDB have been shipped, TSDB is closed and deleted from local disk. This is a per-tenant limit that can be overridden in the runtime configuration. Should be greater than or equal to -limits.query-ingesters-within to prevent premature TSDB closure. 0 to disable.", + "type": "string", + "x-cli-flag": "limits.close-idle-tsdb-timeout", + "x-format": "duration" + }, "compactor_blocks_retention_period": { "default": "0s", "description": "Delete blocks containing samples older than the specified retention period. 0 to disable.", @@ -6190,7 +6190,7 @@ }, "shuffle_sharding_ingesters_lookback_period": { "default": "0s", - "description": "Lookback period for shuffle sharding of ingesters. This is a per-tenant limit that can be overridden in the runtime configuration. Should be greater than or equal to query-ingesters-within.", + "description": "Lookback period for shuffle sharding of ingesters. This is a per-tenant limit that can be overridden in the runtime configuration. Should be greater than or equal to query-store-after and query-ingesters-within.", "type": "string", "x-cli-flag": "limits.shuffle-sharding-ingesters-lookback-period", "x-format": "duration"