Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/configuration/v1-guarantees.md
Original file line number Diff line number Diff line change
Expand Up @@ -133,3 +133,4 @@ Currently experimental features are:
- Ingester: Active Series Tracker
- Per-tenant `active_series_trackers` configuration in runtime config overrides
- Counts active series matching PromQL label matchers and exposes `cortex_ingester_active_series_per_tracker` metric
- `cortex_ingester_active_metric_names` metric exposing the number of unique metric names per user in the ingester head
2 changes: 2 additions & 0 deletions pkg/ingester/ingester.go
Original file line number Diff line number Diff line change
Expand Up @@ -1152,6 +1152,7 @@ func (i *Ingester) updateActiveSeries(ctx context.Context) {
userDB.activeSeries.Purge(purgeTime)
i.metrics.activeSeriesPerUser.WithLabelValues(userID).Set(float64(userDB.activeSeries.Active()))
i.metrics.activeNHSeriesPerUser.WithLabelValues(userID).Set(float64(userDB.activeSeries.ActiveNativeHistogram()))
i.metrics.activeMetricNamesPerUser.WithLabelValues(userID).Set(float64(userDB.seriesInMetric.ActiveMetricNames()))
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: the two lines above derive from userDB.activeSeries.*, which applies the -ingester.active-series-metrics-idle-timeout sliding window (10m default). This line derives from userDB.seriesInMetric, which counts anything alive in the TSDB head — effectively up to the block range (~2h).

Since this gauge is grouped under the Active Series Tracker feature, operators will reasonably expect active_metric_names to use the same window as active_series. Two options:

  1. Move the counter to ActiveSeries (track metric-name refcounts alongside active/activeNativeHistogram). Incremental cost on series creation/purge, O(1) read. Matches the window.
  2. Keep using seriesInMetric, but tighten the Help text to "unique metric names with at least one series in the head (not windowed like active_series)" so the semantic difference is explicit.

Option 1 is cleaner; Option 2 is acceptable if the current data source is what you want.

if err := userDB.labelSetCounter.UpdateMetric(ctx, userDB, i.metrics); err != nil {
level.Warn(i.logger).Log("msg", "failed to update per labelSet metrics", "user", userID, "err", err)
}
Expand Down Expand Up @@ -3041,6 +3042,7 @@ func (i *Ingester) closeAllTSDB() {
i.metrics.memUsers.Dec()
i.metrics.activeSeriesPerUser.DeleteLabelValues(userID)
i.metrics.activeNHSeriesPerUser.DeleteLabelValues(userID)
i.metrics.activeMetricNamesPerUser.DeleteLabelValues(userID)
}(userDB)
}

Expand Down
9 changes: 9 additions & 0 deletions pkg/ingester/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ type ingesterMetrics struct {

activeSeriesPerUser *prometheus.GaugeVec
activeNHSeriesPerUser *prometheus.GaugeVec
activeMetricNamesPerUser *prometheus.GaugeVec
activeQueriedSeriesPerUser *prometheus.GaugeVec
limitsPerLabelSet *prometheus.GaugeVec
usagePerLabelSet *prometheus.GaugeVec
Expand Down Expand Up @@ -298,6 +299,12 @@ func newIngesterMetrics(r prometheus.Registerer,
Help: "Number of currently active native histogram series per user.",
}, []string{"user"}),

// Not registered automatically, but only if activeSeriesEnabled is true.
activeMetricNamesPerUser: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cortex_ingester_active_metric_names",
Help: "Number of unique metric names in the ingester head per user.",
Copy link
Copy Markdown
Member

@SungJin1212 SungJin1212 May 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: What about a TSDB head?

Suggested change
Help: "Number of unique metric names in the ingester head per user.",
Help: "Number of unique metric names in the TSDB head per user.",

}, []string{"user"}),

// Not registered automatically, but only if activeSeriesEnabled is true.
activeSeriesPerTracker: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "cortex_ingester_active_series_per_tracker",
Expand Down Expand Up @@ -349,6 +356,7 @@ func newIngesterMetrics(r prometheus.Registerer,
if activeSeriesEnabled && r != nil {
r.MustRegister(m.activeSeriesPerUser)
r.MustRegister(m.activeNHSeriesPerUser)
r.MustRegister(m.activeMetricNamesPerUser)
r.MustRegister(m.activeSeriesPerTracker)
}

Expand Down Expand Up @@ -380,6 +388,7 @@ func (m *ingesterMetrics) deletePerUserMetrics(userID string) {
m.memMetadataRemovedTotal.DeleteLabelValues(userID)
m.activeSeriesPerUser.DeleteLabelValues(userID)
m.activeNHSeriesPerUser.DeleteLabelValues(userID)
m.activeMetricNamesPerUser.DeleteLabelValues(userID)
m.activeSeriesPerTracker.DeletePartialMatch(prometheus.Labels{"user": userID})
m.activeQueriedSeriesPerUser.DeletePartialMatch(prometheus.Labels{"user": userID})
m.usagePerLabelSet.DeletePartialMatch(prometheus.Labels{"user": userID})
Expand Down
11 changes: 11 additions & 0 deletions pkg/ingester/user_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,17 @@ func (m *metricCounter) increaseSeriesForMetric(metric string) {
shard.mtx.Unlock()
}

// ActiveMetricNames returns the total number of unique metric names tracked across all shards.
func (m *metricCounter) ActiveMetricNames() int {
total := 0
for i := range m.shards {
m.shards[i].mtx.Lock()
total += len(m.shards[i].m)
m.shards[i].mtx.Unlock()
}
return total
}

type labelSetCounterEntry struct {
count int
labels labels.Labels
Expand Down
33 changes: 33 additions & 0 deletions pkg/ingester/user_state_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -378,3 +378,36 @@ func (ir *mockIndexReader) LabelNamesFor(ctx context.Context, postings index.Pos
}

func (ir *mockIndexReader) Close() error { return nil }

func TestMetricCounter_ActiveMetricNames(t *testing.T) {
limits := validation.Limits{MaxLocalSeriesPerMetric: 100}
overrides := validation.NewOverrides(limits, nil)
limiter := NewLimiter(overrides, nil, util.ShardingStrategyDefault, true, 3, false, "")
mc := newMetricCounter(limiter, nil)

// Initially zero.
assert.Equal(t, 0, mc.ActiveMetricNames())

// Add series for 3 different metrics.
mc.increaseSeriesForMetric("metric_a")
mc.increaseSeriesForMetric("metric_a")
mc.increaseSeriesForMetric("metric_b")
mc.increaseSeriesForMetric("metric_c")
assert.Equal(t, 3, mc.ActiveMetricNames())

// Remove all series for metric_b.
mc.decreaseSeriesForMetric("metric_b")
assert.Equal(t, 2, mc.ActiveMetricNames())

// Remove one series for metric_a (still has one left).
mc.decreaseSeriesForMetric("metric_a")
assert.Equal(t, 2, mc.ActiveMetricNames())

// Remove last series for metric_a.
mc.decreaseSeriesForMetric("metric_a")
assert.Equal(t, 1, mc.ActiveMetricNames())

// Remove last series for metric_c.
mc.decreaseSeriesForMetric("metric_c")
assert.Equal(t, 0, mc.ActiveMetricNames())
}
Loading