Use RWMutex in OTEL provider (#214)

Currently, the `otel.Provider` uses a `sync.Mutex` to protect against race conditions on the maps that store instances of the different metric types. The mutex blocks to ensure the existence of a metric on metric modification. 

We can avoid fully blocking on reads for already existing metrics with a `sync.RWMutex`.

Author: ypaq

go-kit/metrics/provider/otel/provider.go

@@ -40,7 +40,7 @@ type Provider struct {
 	meterProvider *metric.MeterProvider
 	viewCache     *viewCache
 
-	mu         sync.Mutex
+	mu         sync.RWMutex
 	counters   map[string]*Counter
 	gauges     map[string]*Gauge
 	histograms map[string]*Histogram

go-kit/metrics/provider/otel/types.go

@@ -45,10 +45,18 @@ func (p *Provider) NewCounter(name string) metrics.Counter {
 }
 
 func (p *Provider) newCounter(name string, labelValues ...string) metrics.Counter {
+	k := keyName(name, labelValues...)
+
+	p.mu.RLock()
+	c, ok := p.counters[k]
+	p.mu.RUnlock()
+	if ok {
+		return c
+	}
+
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
-	k := keyName(name, labelValues...)
 	m := p.meterProvider.Meter(name)
 
 	if _, ok := p.counters[k]; !ok {
@@ -82,10 +90,18 @@ func (p *Provider) NewGauge(name string) metrics.Gauge {
 }
 
 func (p *Provider) newGauge(name string, labelValues ...string) metrics.Gauge {
+	k := keyName(name, labelValues...)
+
+	p.mu.RLock()
+	g, ok := p.gauges[k]
+	p.mu.RUnlock()
+	if ok {
+		return g
+	}
+
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
-	k := keyName(name, labelValues...)
 	m := p.meterProvider.Meter(name)
 
 	attributes := makeAttributes(labelValues)
@@ -166,11 +182,19 @@ func (p *Provider) NewHistogram(name string, buckets int) metrics.Histogram {
 }
 
 func (p *Provider) newHistogram(stream sdk.Stream, labelValues ...string) metrics.Histogram {
+	name := stream.Name
+	k := keyName(name, labelValues...)
+
+	p.mu.RLock()
+	h, ok := p.histograms[k]
+	p.mu.RUnlock()
+	if ok {
+		return h
+	}
+
 	p.mu.Lock()
 	defer p.mu.Unlock()
 
-	name := stream.Name
-	k := keyName(name, labelValues...)
 	m := p.meterProvider.Meter(name)
 
 	if _, ok := p.histograms[k]; !ok {