Skip to content

fix: do not use new metric instances every Go memstats collection #1792

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

fix: do not use new metric instances every Go memstats collection #1792

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
074bca8
fix: do not use new metric instances every Go memstats collection
alespour Apr 7, 2025
e21d4eb
test: fix test for Go memstats struct change
alespour Apr 7, 2025
1d9d76a
fix: type switch
alespour Apr 7, 2025
5536767
refactor: move function to where it is used
alespour Apr 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
220 changes: 65 additions & 155 deletions prometheus/go_collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,176 +24,72 @@ import (
// while eval closure works on runtime.MemStats, the struct from Go 1.17+ is
// populated using runtime/metrics. Those are the defaults we can't alter.
func goRuntimeMemStats() memStatsMetrics {
const ns = "go_memstats" // see memstatNamespace(string)
return memStatsMetrics{
{
desc: NewDesc(
memstatNamespace("alloc_bytes"),
"Number of bytes allocated in heap and currently in use. Equals to /memory/classes/heap/objects:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.Alloc) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.Alloc) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "alloc_bytes", Help: "Number of bytes allocated in heap and currently in use. Equals to /memory/classes/heap/objects:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("alloc_bytes_total"),
"Total number of bytes allocated in heap until now, even if released already. Equals to /gc/heap/allocs:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.TotalAlloc) },
valType: CounterValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.TotalAlloc) },
metric: NewCounter(CounterOpts{Namespace: ns, Name: "alloc_bytes_total", Help: "Total number of bytes allocated in heap until now, even if released already. Equals to /gc/heap/allocs:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("sys_bytes"),
"Number of bytes obtained from system. Equals to /memory/classes/total:byte.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.Sys) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.Sys) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "sys_bytes", Help: "Number of bytes obtained from system. Equals to /memory/classes/total:byte."}),
}, {
desc: NewDesc(
memstatNamespace("mallocs_total"),
// TODO(bwplotka): We could add go_memstats_heap_objects, probably useful for discovery. Let's gather more feedback, kind of a waste of bytes for everybody for compatibility reasons to keep both, and we can't really rename/remove useful metric.
"Total number of heap objects allocated, both live and gc-ed. Semantically a counter version for go_memstats_heap_objects gauge. Equals to /gc/heap/allocs:objects + /gc/heap/tiny/allocs:objects.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.Mallocs) },
valType: CounterValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.Mallocs) },
// TODO(bwplotka): We could add go_memstats_heap_objects, probably useful for discovery. Let's gather more feedback, kind of a waste of bytes for everybody for compatibility reasons to keep both, and we can't really rename/remove useful metric.
metric: NewCounter(CounterOpts{Namespace: ns, Name: "mallocs_total", Help: "Total number of heap objects allocated, both live and gc-ed. Semantically a counter version for go_memstats_heap_objects gauge. Equals to /gc/heap/allocs:objects + /gc/heap/tiny/allocs:objects."}),
}, {
desc: NewDesc(
memstatNamespace("frees_total"),
"Total number of heap objects frees. Equals to /gc/heap/frees:objects + /gc/heap/tiny/allocs:objects.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.Frees) },
valType: CounterValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.Frees) },
metric: NewCounter(CounterOpts{Namespace: ns, Name: "frees_total", Help: "Total number of heap objects frees. Equals to /gc/heap/frees:objects + /gc/heap/tiny/allocs:objects."}),
}, {
desc: NewDesc(
memstatNamespace("heap_alloc_bytes"),
"Number of heap bytes allocated and currently in use, same as go_memstats_alloc_bytes. Equals to /memory/classes/heap/objects:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapAlloc) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapAlloc) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "heap_alloc_bytes", Help: "Number of heap bytes allocated and currently in use, same as go_memstats_alloc_bytes. Equals to /memory/classes/heap/objects:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("heap_sys_bytes"),
"Number of heap bytes obtained from system. Equals to /memory/classes/heap/objects:bytes + /memory/classes/heap/unused:bytes + /memory/classes/heap/released:bytes + /memory/classes/heap/free:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapSys) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapSys) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "heap_sys_bytes", Help: "Number of heap bytes obtained from system. Equals to /memory/classes/heap/objects:bytes + /memory/classes/heap/unused:bytes + /memory/classes/heap/released:bytes + /memory/classes/heap/free:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("heap_idle_bytes"),
"Number of heap bytes waiting to be used. Equals to /memory/classes/heap/released:bytes + /memory/classes/heap/free:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapIdle) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapIdle) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "heap_idle_bytes", Help: "Number of heap bytes waiting to be used. Equals to /memory/classes/heap/released:bytes + /memory/classes/heap/free:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("heap_inuse_bytes"),
"Number of heap bytes that are in use. Equals to /memory/classes/heap/objects:bytes + /memory/classes/heap/unused:bytes",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapInuse) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapInuse) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "heap_inuse_bytes", Help: "Number of heap bytes that are in use. Equals to /memory/classes/heap/objects:bytes + /memory/classes/heap/unused:bytes"}),
}, {
desc: NewDesc(
memstatNamespace("heap_released_bytes"),
"Number of heap bytes released to OS. Equals to /memory/classes/heap/released:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapReleased) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapReleased) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "heap_released_bytes", Help: "Number of heap bytes released to OS. Equals to /memory/classes/heap/released:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("heap_objects"),
"Number of currently allocated objects. Equals to /gc/heap/objects:objects.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapObjects) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.HeapObjects) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "heap_objects", Help: "Number of currently allocated objects. Equals to /gc/heap/objects:objects."}),
}, {
desc: NewDesc(
memstatNamespace("stack_inuse_bytes"),
"Number of bytes obtained from system for stack allocator in non-CGO environments. Equals to /memory/classes/heap/stacks:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.StackInuse) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.StackInuse) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "stack_inuse_bytes", Help: "Number of bytes obtained from system for stack allocator in non-CGO environments. Equals to /memory/classes/heap/stacks:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("stack_sys_bytes"),
"Number of bytes obtained from system for stack allocator. Equals to /memory/classes/heap/stacks:bytes + /memory/classes/os-stacks:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.StackSys) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.StackSys) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "stack_sys_bytes", Help: "Number of bytes obtained from system for stack allocator. Equals to /memory/classes/heap/stacks:bytes + /memory/classes/os-stacks:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("mspan_inuse_bytes"),
"Number of bytes in use by mspan structures. Equals to /memory/classes/metadata/mspan/inuse:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.MSpanInuse) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.MSpanInuse) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "mspan_inuse_bytes", Help: "Number of bytes in use by mspan structures. Equals to /memory/classes/metadata/mspan/inuse:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("mspan_sys_bytes"),
"Number of bytes used for mspan structures obtained from system. Equals to /memory/classes/metadata/mspan/inuse:bytes + /memory/classes/metadata/mspan/free:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.MSpanSys) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.MSpanSys) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "mspan_sys_bytes", Help: "Number of bytes used for mspan structures obtained from system. Equals to /memory/classes/metadata/mspan/inuse:bytes + /memory/classes/metadata/mspan/free:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("mcache_inuse_bytes"),
"Number of bytes in use by mcache structures. Equals to /memory/classes/metadata/mcache/inuse:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.MCacheInuse) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.MCacheInuse) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "mcache_inuse_bytes", Help: "Number of bytes in use by mcache structures. Equals to /memory/classes/metadata/mcache/inuse:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("mcache_sys_bytes"),
"Number of bytes used for mcache structures obtained from system. Equals to /memory/classes/metadata/mcache/inuse:bytes + /memory/classes/metadata/mcache/free:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.MCacheSys) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.MCacheSys) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "mcache_sys_bytes", Help: "Number of bytes used for mcache structures obtained from system. Equals to /memory/classes/metadata/mcache/inuse:bytes + /memory/classes/metadata/mcache/free:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("buck_hash_sys_bytes"),
"Number of bytes used by the profiling bucket hash table. Equals to /memory/classes/profiling/buckets:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.BuckHashSys) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.BuckHashSys) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "buck_hash_sys_bytes", Help: "Number of bytes used by the profiling bucket hash table. Equals to /memory/classes/profiling/buckets:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("gc_sys_bytes"),
"Number of bytes used for garbage collection system metadata. Equals to /memory/classes/metadata/other:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.GCSys) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.GCSys) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "gc_sys_bytes", Help: "Number of bytes used for garbage collection system metadata. Equals to /memory/classes/metadata/other:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("other_sys_bytes"),
"Number of bytes used for other system allocations. Equals to /memory/classes/other:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.OtherSys) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.OtherSys) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "other_sys_bytes", Help: "Number of bytes used for other system allocations. Equals to /memory/classes/other:bytes."}),
}, {
desc: NewDesc(
memstatNamespace("next_gc_bytes"),
"Number of heap bytes when next garbage collection will take place. Equals to /gc/heap/goal:bytes.",
nil, nil,
),
eval: func(ms *runtime.MemStats) float64 { return float64(ms.NextGC) },
valType: GaugeValue,
eval: func(ms *runtime.MemStats) float64 { return float64(ms.NextGC) },
metric: NewGauge(GaugeOpts{Namespace: ns, Name: "next_gc_bytes", Help: "Number of heap bytes when next garbage collection will take place. Equals to /gc/heap/goal:bytes."}),
},
}
}
Expand Down Expand Up @@ -261,14 +157,28 @@ func (c *baseGoCollector) Collect(ch chan<- Metric) {
ch <- MustNewConstMetric(c.goInfoDesc, GaugeValue, 1)
}

func memstatNamespace(s string) string {
return "go_memstats_" + s
// memStatsMetric provide description, evaluator, runtime/metrics name, and
// value type for memstat metric.
type memStatsMetric struct {
last float64
eval func(*runtime.MemStats) float64
metric Metric
}

func (m *memStatsMetric) update(memStats *runtime.MemStats) Metric {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not concurrency-safe. Metrics collection can happen concurrently, and then all kind of weirdness will happen here.

current := m.eval(memStats)
switch value := m.metric.(type) {
case Gauge:
value.Set(current)
case Counter:
value.Add(current - m.last)
default:
panic("unexpected metric type")
}
m.last = current
return m.metric
}

// memStatsMetrics provide description, evaluator, runtime/metrics name, and
// value type for memstat metrics.
type memStatsMetrics []struct {
desc *Desc
eval func(*runtime.MemStats) float64
valType ValueType
}
type memStatsMetrics []memStatsMetric
4 changes: 4 additions & 0 deletions prometheus/go_collector_go116.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,3 +120,7 @@ func (c *goCollector) msCollect(ch chan<- Metric, ms *runtime.MemStats) {
ch <- MustNewConstMetric(i.desc, i.valType, i.eval(ms))
}
}

func memstatNamespace(s string) string {
return "go_memstats_" + s
}
4 changes: 2 additions & 2 deletions prometheus/go_collector_latest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ func attachOriginalName(desc, origName string) string {
func (c *goCollector) Describe(ch chan<- *Desc) {
c.base.Describe(ch)
for _, i := range c.msMetrics {
ch <- i.desc
ch <- i.metric.Desc()
}
for _, m := range c.rmExposedMetrics {
ch <- m.Desc()
Expand Down Expand Up @@ -364,7 +364,7 @@ func (c *goCollector) Collect(ch chan<- Metric) {
var ms runtime.MemStats
memStatsFromRM(&ms, c.sampleMap)
for _, i := range c.msMetrics {
ch <- MustNewConstMetric(i.desc, i.valType, i.eval(&ms))
ch <- i.update(&ms)
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion prometheus/go_collector_latest_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func expectedBaseMetrics() map[string]struct{} {

func addExpectedRuntimeMemStats(metrics map[string]struct{}) map[string]struct{} {
for _, m := range goRuntimeMemStats() {
metrics[m.desc.fqName] = struct{}{}
metrics[m.metric.Desc().fqName] = struct{}{}
}
return metrics
}
Expand Down
Loading