per tenant close_idle_tsdb_timeout

CHANGELOG.md

@@ -3,7 +3,7 @@
 ## master / unreleased
 * [FEATURE] Engine: Add `-querier.selector-batch-size` and `-ruler.selector-batch-size` flags to configure series batching in the Thanos promQL engine. 0 disables batching. #7763
 * [CHANGE] Ingester: Formally deprecate `-blocks-storage.tsdb.max-exemplars`, scheduled for removal in v1.24.0. Use the per-tenant `max_exemplars` limit instead. The flag still works as the global fallback when `max_exemplars` is 0, but setting it now logs a warning and increments `deprecated_flags_inuse_total`. #7793
-* [CHANGE] Querier: Make query time range configurations per-tenant: `query_ingesters_within`, `query_store_after`, and `shuffle_sharding_ingesters_lookback_period`. Uses `model.Duration` instead of `time.Duration` to support serialization but has minimum unit of 1ms (nanoseconds/microseconds not supported). #7160
+* [CHANGE] Querier: Make query time range configurations per-tenant: `query_ingesters_within`, `query_store_after`, `close_idle_tsdb_timeout` and `shuffle_sharding_ingesters_lookback_period`. Uses `model.Duration` instead of `time.Duration` to support serialization but has minimum unit of 1ms (nanoseconds/microseconds not supported). #7323 #7813
 * [CHANGE] Cache: Setting `-blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl` to 0 will disable the bucket-index cache. #7446
 * [CHANGE] HA Tracker: Move `-distributor.ha-tracker.failover-timeout` from a global config to a per-tenant runtime config. The flag name and default value (30s) remain the same. #7481
 * [FEATURE] Parquet: Support sharded parquet file conversion and querying. #7610

docs/blocks-storage/querier.md

@@ -2242,15 +2242,6 @@ blocks_storage:
     # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown
     [flush_blocks_on_shutdown: <boolean> | default = false]
 
-    # If TSDB has not received any data for this duration, and all blocks from
-    # TSDB have been shipped, TSDB is closed and deleted from local disk. If set
-    # to positive value, this value must be greater than
-    # -limits.query-ingesters-within flag to make sure that TSDB is not closed
-    # prematurely, which could cause partial query results. 0 or negative value
-    # disables closing of idle TSDB.
-    # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout
-    [close_idle_tsdb_timeout: <duration> | default = 0s]
-
     # The size of the in-memory queue used before flushing chunks to the disk.
     # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size
     [head_chunks_write_queue_size: <int> | default = 0]

docs/blocks-storage/store-gateway.md

@@ -2295,15 +2295,6 @@ blocks_storage:
     # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown
     [flush_blocks_on_shutdown: <boolean> | default = false]
 
-    # If TSDB has not received any data for this duration, and all blocks from
-    # TSDB have been shipped, TSDB is closed and deleted from local disk. If set
-    # to positive value, this value must be greater than
-    # -limits.query-ingesters-within flag to make sure that TSDB is not closed
-    # prematurely, which could cause partial query results. 0 or negative value
-    # disables closing of idle TSDB.
-    # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout
-    [close_idle_tsdb_timeout: <duration> | default = 0s]
-
     # The size of the in-memory queue used before flushing chunks to the disk.
     # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size
     [head_chunks_write_queue_size: <int> | default = 0]

docs/configuration/config-file-reference.md

@@ -2938,15 +2938,6 @@ tsdb:
   # CLI flag: -blocks-storage.tsdb.flush-blocks-on-shutdown
   [flush_blocks_on_shutdown: <boolean> | default = false]
 
-  # If TSDB has not received any data for this duration, and all blocks from
-  # TSDB have been shipped, TSDB is closed and deleted from local disk. If set
-  # to positive value, this value must be greater than
-  # -limits.query-ingesters-within flag to make sure that TSDB is not closed
-  # prematurely, which could cause partial query results. 0 or negative value
-  # disables closing of idle TSDB.
-  # CLI flag: -blocks-storage.tsdb.close-idle-tsdb-timeout
-  [close_idle_tsdb_timeout: <duration> | default = 0s]
-
   # The size of the in-memory queue used before flushing chunks to the disk.
   # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size
   [head_chunks_write_queue_size: <int> | default = 0]
@@ -4778,10 +4769,18 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
 
 # Lookback period for shuffle sharding of ingesters. This is a per-tenant limit
 # that can be overridden in the runtime configuration. Should be greater than or
-# equal to query-ingesters-within.
+# equal to query-store-after and query-ingesters-within.
 # CLI flag: -limits.shuffle-sharding-ingesters-lookback-period
 [shuffle_sharding_ingesters_lookback_period: <duration> | default = 0s]
 
+# If TSDB has not received any data for this duration, and all blocks from TSDB
+# have been shipped, TSDB is closed and deleted from local disk. This is a
+# per-tenant limit that can be overridden in the runtime configuration. Should
+# be greater than or equal to -limits.query-ingesters-within to prevent
+# premature TSDB closure. 0 to disable.
+# CLI flag: -limits.close-idle-tsdb-timeout
+[close_idle_tsdb_timeout: <duration> | default = 0s]
+
 # The maximum number of rows that can be fetched when querying parquet storage.
 # Each row maps to a series in a parquet file. This limit applies before
 # materializing chunks. 0 to disable.

pkg/cortex/cortex.go

@@ -10,6 +10,7 @@ import (
 	"reflect"
 	"slices"
 	"strings"
+	"time"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
@@ -220,7 +221,7 @@ func (c *Config) Validate(log log.Logger) error {
 	if err := c.LimitsConfig.Validate(c.NameValidationScheme, c.Distributor.ShardByAllLabels, c.Ingester.ActiveSeriesMetricsEnabled, c.Distributor.HATrackerConfig.UpdateTimeout, c.Distributor.HATrackerConfig.UpdateTimeoutJitterMax); err != nil {
 		return errors.Wrap(err, "invalid limits config")
 	}
-	if err := c.LimitsConfig.ValidateQueryLimits("default", c.BlocksStorage.TSDB.CloseIdleTSDBTimeout); err != nil {
+	if err := c.LimitsConfig.ValidateQueryLimits("default", time.Duration(c.LimitsConfig.CloseIdleTSDBTimeout)); err != nil {
 		return errors.Wrap(err, "invalid query routing config")
 	}
 	if err := c.ResourceMonitor.Validate(); err != nil {

pkg/cortex/runtime_config.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"net/http"
 	"strings"
+	"time"
 
 	"gopkg.in/yaml.v2"
 
@@ -80,7 +81,7 @@ func (l runtimeConfigLoader) load(r io.Reader) (any, error) {
 					if err := ul.Validate(l.cfg.NameValidationScheme, l.cfg.Distributor.ShardByAllLabels, l.cfg.Ingester.ActiveSeriesMetricsEnabled, l.cfg.Distributor.HATrackerConfig.UpdateTimeout, l.cfg.Distributor.HATrackerConfig.UpdateTimeoutJitterMax); err != nil {
 						return nil, err
 					}
-					if err := ul.ValidateQueryLimits(userID, l.cfg.BlocksStorage.TSDB.CloseIdleTSDBTimeout); err != nil {
+					if err := ul.ValidateQueryLimits(userID, time.Duration(ul.CloseIdleTSDBTimeout)); err != nil {
 						return nil, err
 					}
 				}

pkg/ingester/ingester.go

@@ -1016,7 +1016,9 @@ func (i *Ingester) starting(ctx context.Context) error {
 		servs = append(servs, shippingService)
 	}
 
-	if i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout > 0 {
+	// The close-idle TSDB timeout is now a per-tenant limit, so the timer always runs;
+	// closeAndDeleteIdleUserTSDBs decides per tenant whether any TSDB is actually closed.
+	{
 		interval := i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval
 		if interval == 0 {
 			interval = cortex_tsdb.DefaultCloseIdleTSDBInterval
@@ -3539,6 +3541,17 @@ func (i *Ingester) closeAndDeleteIdleUserTSDBs(ctx context.Context) error {
 			return nil
 		}
 
+		// A non-positive close-idle timeout disables idle-based closing for this tenant,
+		// so we skip the idle check entirely (and don't record an idleTsdbChecks result,
+		// since no check is performed). A tenant marked for deletion is the one exception:
+		// its local TSDB must still be closed and deleted regardless of the timeout.
+		if i.limits.CloseIdleTSDBTimeout(userID) <= 0 {
+			userDB, err := i.getTSDB(userID)
+			if err != nil || userDB == nil || !userDB.deletionMarkFound.Load() {
+				continue
+			}
+		}
+
 		result := i.closeAndDeleteUserTSDBIfIdle(userID)
 
 		i.TSDBState.idleTsdbChecks.WithLabelValues(string(result)).Inc()
@@ -3569,7 +3582,9 @@ func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string) tsdbCloseCheckRes
 		return tsdbShippingDisabled
 	}
 
-	if result := userDB.shouldCloseTSDB(i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout); !result.shouldClose() {
+	closeIdleTSDBTimeout := i.limits.CloseIdleTSDBTimeout(userID)
+
+	if result := userDB.shouldCloseTSDB(closeIdleTSDBTimeout); !result.shouldClose() {
 		return result
 	}
 
@@ -3587,7 +3602,7 @@ func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string) tsdbCloseCheckRes
 
 	// Verify again, things may have changed during the checks and pushes.
 	tenantDeleted := false
-	if result := userDB.shouldCloseTSDB(i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout); !result.shouldClose() {
+	if result := userDB.shouldCloseTSDB(closeIdleTSDBTimeout); !result.shouldClose() {
 		// This will also change TSDB state back to active (via defer above).
 		return result
 	} else if result == tsdbTenantMarkedForDeletion {

pkg/ingester/ingester_test.go

@@ -5264,7 +5264,6 @@ func TestIngester_closeAndDeleteUserTSDBIfIdle_shouldNotCloseTSDBIfShippingIsInP
 func TestIngester_closingAndOpeningTsdbConcurrently(t *testing.T) {
 	ctx := context.Background()
 	cfg := defaultIngesterTestConfig(t)
-	cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 0 // Will not run the loop, but will allow us to close any TSDB fast.
 
 	// Create ingester
 	i, err := prepareIngesterWithBlocksStorage(t, cfg, prometheus.NewRegistry())
@@ -5317,7 +5316,6 @@ func TestIngester_idleCloseEmptyTSDB(t *testing.T) {
 	cfg := defaultIngesterTestConfig(t)
 	cfg.BlocksStorageConfig.TSDB.ShipInterval = 1 * time.Minute
 	cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Minute
-	cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 0 // Will not run the loop, but will allow us to close any TSDB fast.
 
 	// Create ingester
 	i, err := prepareIngesterWithBlocksStorage(t, cfg, prometheus.NewRegistry())
@@ -5365,7 +5363,6 @@ func TestIngester_ReadNotFailWhenTSDBIsBeingDeleted(t *testing.T) {
 		t.Run(name, func(t *testing.T) {
 			ctx := context.Background()
 			cfg := defaultIngesterTestConfig(t)
-			cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 0 // Will not run the loop, but will allow us to close any TSDB fast.
 			cfg.BlocksStorageConfig.TSDB.KeepUserTSDBOpenOnShutdown = true
 
 			// Create ingester
@@ -6031,13 +6028,16 @@ func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) {
 	cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1
 	cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Second
 	cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout = 1 * time.Second
-	cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 1 * time.Second
 	cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval = 100 * time.Millisecond
 
+	limits := defaultLimitsTestConfig()
+	limits.EnableNativeHistograms = true
+	limits.CloseIdleTSDBTimeout = model.Duration(1 * time.Second)
+
 	r := prometheus.NewRegistry()
 
 	// Create ingester
-	i, err := prepareIngesterWithBlocksStorage(t, cfg, r)
+	i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", r)
 	require.NoError(t, err)
 
 	require.NoError(t, services.StartAndAwaitRunning(context.Background(), i))
@@ -6142,6 +6142,88 @@ func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) {
     `), metricsToCheck...))
 }
 
+func TestIngesterCloseIdleTSDB_PerTenantTimeout(t *testing.T) {
+	cfg := defaultIngesterTestConfig(t)
+	cfg.LifecyclerConfig.JoinAfter = 0
+	cfg.BlocksStorageConfig.TSDB.ShipInterval = 1 * time.Second
+	cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1
+	cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Second
+	cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout = 1 * time.Second
+	cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval = 100 * time.Millisecond
+
+	limits := defaultLimitsTestConfig()
+	limits.EnableNativeHistograms = true
+	// Per-tenant limit enables close-idle for this tenant
+	limits.CloseIdleTSDBTimeout = model.Duration(1 * time.Second)
+
+	tenantLimits := newMockTenantLimits(map[string]*validation.Limits{userID: &limits})
+
+	i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, "", prometheus.NewRegistry())
+	require.NoError(t, err)
+
+	require.NoError(t, services.StartAndAwaitRunning(context.Background(), i))
+	t.Cleanup(func() {
+		require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i))
+	})
+
+	test.Poll(t, 1*time.Second, ring.ACTIVE, func() any {
+		return i.lifecycler.GetState()
+	})
+
+	pushSingleSampleWithMetadata(t, i)
+
+	require.Equal(t, int64(1), i.TSDBState.seriesCount.Load())
+
+	// Wait until TSDB has been closed via the per-tenant timeout.
+	test.Poll(t, 10*time.Second, 0, func() any {
+		i.stoppedMtx.Lock()
+		defer i.stoppedMtx.Unlock()
+		return len(i.TSDBState.dbs)
+	})
+
+	require.Greater(t, testutil.ToFloat64(i.TSDBState.idleTsdbChecks.WithLabelValues(string(tsdbIdleClosed))), float64(0))
+}
+
+func TestIngesterCloseIdleTSDB_DisabledWhenZero(t *testing.T) {
+	cfg := defaultIngesterTestConfig(t)
+	cfg.LifecyclerConfig.JoinAfter = 0
+	cfg.BlocksStorageConfig.TSDB.ShipInterval = 1 * time.Second
+	cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1
+	cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Second
+	cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout = 1 * time.Second
+	cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval = 100 * time.Millisecond
+
+	limits := defaultLimitsTestConfig()
+	limits.EnableNativeHistograms = true
+	limits.CloseIdleTSDBTimeout = 0 // Disabled
+
+	tenantLimits := newMockTenantLimits(map[string]*validation.Limits{userID: &limits})
+
+	i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, "", prometheus.NewRegistry())
+	require.NoError(t, err)
+
+	require.NoError(t, services.StartAndAwaitRunning(context.Background(), i))
+	t.Cleanup(func() {
+		require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i))
+	})
+
+	test.Poll(t, 1*time.Second, ring.ACTIVE, func() any {
+		return i.lifecycler.GetState()
+	})
+
+	pushSingleSampleWithMetadata(t, i)
+
+	require.Equal(t, int64(1), i.TSDBState.seriesCount.Load())
+
+	// Wait a bit and confirm TSDB is NOT closed (timeout is disabled).
+	time.Sleep(500 * time.Millisecond)
+
+	i.stoppedMtx.Lock()
+	numDBs := len(i.TSDBState.dbs)
+	i.stoppedMtx.Unlock()
+	require.Equal(t, 1, numDBs)
+}
+
 func verifyCompactedHead(t *testing.T, i *Ingester, expected bool) {
 	db, err := i.getTSDB(userID)
 	require.NoError(t, err)
@@ -7768,7 +7850,6 @@ func TestIngester_UpdateLabelSetMetrics(t *testing.T) {
 func TestIngesterPanicHandling(t *testing.T) {
 	ctx := context.Background()
 	cfg := defaultIngesterTestConfig(t)
-	cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 0 // Will not run the loop, but will allow us to close any TSDB fast.
 	cfg.BlocksStorageConfig.TSDB.KeepUserTSDBOpenOnShutdown = true
 
 	// Create ingester

pkg/storage/tsdb/config.go

@@ -157,7 +157,6 @@ type TSDBConfig struct {
 	WALCompressionType        string        `yaml:"wal_compression_type"`
 	WALSegmentSizeBytes       int           `yaml:"wal_segment_size_bytes"`
 	FlushBlocksOnShutdown     bool          `yaml:"flush_blocks_on_shutdown"`
-	CloseIdleTSDBTimeout      time.Duration `yaml:"close_idle_tsdb_timeout"`
 	// The size of the in-memory queue used before flushing chunks to the disk.
 	HeadChunksWriteQueueSize int `yaml:"head_chunks_write_queue_size"`
 
@@ -207,7 +206,6 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) {
 	f.StringVar(&cfg.WALCompressionType, "blocks-storage.tsdb.wal-compression-type", "", "TSDB WAL type. Supported values are: 'snappy', 'zstd' and '' (disable compression)")
 	f.IntVar(&cfg.WALSegmentSizeBytes, "blocks-storage.tsdb.wal-segment-size-bytes", wlog.DefaultSegmentSize, "TSDB WAL segments files max size (bytes).")
 	f.BoolVar(&cfg.FlushBlocksOnShutdown, "blocks-storage.tsdb.flush-blocks-on-shutdown", false, "True to flush blocks to storage on shutdown. If false, incomplete blocks will be reused after restart.")
-	f.DurationVar(&cfg.CloseIdleTSDBTimeout, "blocks-storage.tsdb.close-idle-tsdb-timeout", 0, "If TSDB has not received any data for this duration, and all blocks from TSDB have been shipped, TSDB is closed and deleted from local disk. If set to positive value, this value must be greater than -limits.query-ingesters-within flag to make sure that TSDB is not closed prematurely, which could cause partial query results. 0 or negative value disables closing of idle TSDB.")
 	f.IntVar(&cfg.HeadChunksWriteQueueSize, "blocks-storage.tsdb.head-chunks-write-queue-size", chunks.DefaultWriteQueueSize, "The size of the in-memory queue used before flushing chunks to the disk.")
 	f.IntVar(&cfg.MaxExemplars, "blocks-storage.tsdb.max-exemplars", 0, "Deprecated (use the per-tenant max_exemplars limit instead) and will be removed in v1.24.0: the global fallback for the maximum number of exemplars stored in TSDB, used only when the per-tenant max_exemplars limit is 0. 0 or less means exemplars are disabled.")
 	f.BoolVar(&cfg.MemorySnapshotOnShutdown, "blocks-storage.tsdb.memory-snapshot-on-shutdown", false, "True to enable snapshotting of in-memory TSDB data on disk when shutting down.")

pkg/util/validation/exporter_test.go

@@ -48,6 +48,7 @@ func TestOverridesExporter_withConfig(t *testing.T) {
 		cortex_overrides{limit_name="alertmanager_max_templates_count",user="tenant-a"} 0
 		cortex_overrides{limit_name="alertmanager_notification_rate_limit",user="tenant-a"} 0
 		cortex_overrides{limit_name="alertmanager_receivers_firewall_block_private_addresses",user="tenant-a"} 0
+		cortex_overrides{limit_name="close_idle_tsdb_timeout",user="tenant-a"} 0
 		cortex_overrides{limit_name="compactor_blocks_retention_period",user="tenant-a"} 0
 		cortex_overrides{limit_name="compactor_partition_index_size_bytes",user="tenant-a"} 6.8719476736e+10
 		cortex_overrides{limit_name="compactor_partition_series_count",user="tenant-a"} 0