Skip to content

Add possibility to exclude deadlocks & other Hotspot collectors #672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* Can be replaced with a simple access once JDK 1.7 compatibility is baseline.
*
*/
public class BufferPoolsExports extends Collector {
public class BufferPoolsExports extends Collector implements HotspotCollector {

private static final Logger LOGGER = Logger.getLogger(BufferPoolsExports.class.getName());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* jvm_classes_unloaded_total{} 500
* </pre>
*/
public class ClassLoadingExports extends Collector {
public class ClassLoadingExports extends Collector implements HotspotCollector {
private final ClassLoadingMXBean clBean;

public ClassLoadingExports() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package io.prometheus.client.hotspot;

import io.prometheus.client.Collector;
import io.prometheus.client.GaugeMetricFamily;

import java.lang.management.ManagementFactory;
import java.lang.management.ThreadMXBean;
import java.util.ArrayList;
import java.util.List;

/**
* Exports metrics about JVM thread deadlocks.
*
* Deadlock detection might be an expensive operation.
* Consider to avoid these metrics for performance-critical applications.
*
* <p>
* Example usage:
* <pre>
* {@code
* new DeadlockExports().register();
* }
* </pre>
* Example metrics being exported:
* <pre>
* jvm_threads_deadlocked{} 4
* jvm_threads_deadlocked_monitor{} 2
* </pre>
*/
public class DeadlockExports extends Collector implements HotspotCollector {
private final ThreadMXBean threadBean;

public DeadlockExports() {
this(ManagementFactory.getThreadMXBean());
}

public DeadlockExports(ThreadMXBean threadBean) {
this.threadBean = threadBean;
}

void addDeadlockMetrics(List<MetricFamilySamples> sampleFamilies) {
sampleFamilies.add(
new GaugeMetricFamily(
"jvm_threads_deadlocked",
"Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers",
nullSafeArrayLength(threadBean.findDeadlockedThreads())));

sampleFamilies.add(
new GaugeMetricFamily(
"jvm_threads_deadlocked_monitor",
"Cycles of JVM-threads that are in deadlock waiting to acquire object monitors",
nullSafeArrayLength(threadBean.findMonitorDeadlockedThreads())));
}

private static double nullSafeArrayLength(long[] array) {
return null == array ? 0 : array.length;
}

@Override
public List<MetricFamilySamples> collect() {
List<MetricFamilySamples> mfs = new ArrayList<MetricFamilySamples>();
addDeadlockMetrics(mfs);
return mfs;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@

import io.prometheus.client.CollectorRegistry;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
* Registers the default Hotspot collectors.
* <p>
Expand All @@ -14,6 +20,14 @@
* DefaultExports.initialize();
* }
* </pre>
* If some collector is undesirable for any reason
* (e.g. {@link DeadlockExports} for potential performance penalty)
* it may be explicitly excluded by using alternative registration approach:
* <pre>
* {@code
* DefaultExports.build().exclude(DeadlockExports.class).register();
* }
* </pre>
*/
public class DefaultExports {
private static boolean initialized = false;
Expand All @@ -34,14 +48,63 @@ public static synchronized void initialize() {
* Register the default Hotspot collectors with the given registry.
*/
public static void register(CollectorRegistry registry) {
new StandardExports().register(registry);
new MemoryPoolsExports().register(registry);
new MemoryAllocationExports().register(registry);
new BufferPoolsExports().register(registry);
new GarbageCollectorExports().register(registry);
new ThreadExports().register(registry);
new ClassLoadingExports().register(registry);
new VersionInfoExports().register(registry);
register(registry, Collections.<Class<? extends HotspotCollector>>emptySet());
}

private static void register(CollectorRegistry registry, Set<Class<? extends HotspotCollector>> exclusions) {
List<? extends HotspotCollector> collectors = Arrays.asList(
new StandardExports(),
new MemoryPoolsExports(),
new MemoryAllocationExports(),
new BufferPoolsExports(),
new GarbageCollectorExports(),
new ThreadExports(),
new DeadlockExports(),
new ClassLoadingExports(),
new VersionInfoExports());

for (HotspotCollector collector : collectors) {
if (!exclusions.contains(collector.getClass())) {
collector.register(registry);
}
}
}

/**
* Return a Builder to allow configuration of a DefaultExports.
*/
public static Builder build() {
return new Builder();
}

public static class Builder {

private final Set<Class<? extends HotspotCollector>> exclusions = new HashSet<Class<? extends HotspotCollector>>();

private Builder() {
}

/**
* Exclude provided Hotspot collector from registration.
*/
public Builder exclude(Class<? extends HotspotCollector> exclusion) {
exclusions.add(exclusion);
return this;
}

/**
* Register the default Hotspot collectors (except provided exclusions) with the default registry.
*/
public void register() {
register(CollectorRegistry.defaultRegistry);
}

/**
* Register the default Hotspot collectors (except provided exclusions) with the given registry.
*/
public void register(CollectorRegistry registry) {
DefaultExports.register(registry, exclusions);
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
* jvm_gc_collection_seconds_sum{gc="PS1"} 6.7
* </pre>
*/
public class GarbageCollectorExports extends Collector {
public class GarbageCollectorExports extends Collector implements HotspotCollector {
private final List<GarbageCollectorMXBean> garbageCollectors;

public GarbageCollectorExports() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package io.prometheus.client.hotspot;

import io.prometheus.client.Collector;
import io.prometheus.client.CollectorRegistry;

/**
* Intended mostly for exclusion of undesired collectors from {@link DefaultExports}
* (see {@link DefaultExports.Builder#exclude(Class)})
*/
public interface HotspotCollector {

<T extends Collector> T register(CollectorRegistry registry);
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
import java.util.List;
import java.util.Map;

public class MemoryAllocationExports extends Collector {
public class MemoryAllocationExports extends Collector implements HotspotCollector {
private final Counter allocatedCounter = Counter.build()
.name("jvm_memory_pool_allocated_bytes_total")
.help("Total bytes allocated in a given JVM memory pool. Only updated after GC, not continuously.")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
* jvm_memory_pool_bytes_used{pool="PS Eden Space"} 2000
* </pre>
*/
public class MemoryPoolsExports extends Collector {
public class MemoryPoolsExports extends Collector implements HotspotCollector {
private final MemoryMXBean memoryBean;
private final List<MemoryPoolMXBean> poolBeans;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
* }
* </pre>
*/
public class StandardExports extends Collector {
public class StandardExports extends Collector implements HotspotCollector {
private static final Logger LOGGER = Logger.getLogger(StandardExports.class.getName());

private final StatusReader statusReader;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
* jvm_threads_started_total{} 1200
* </pre>
*/
public class ThreadExports extends Collector {
public class ThreadExports extends Collector implements HotspotCollector {
private final ThreadMXBean threadBean;

public ThreadExports() {
Expand Down Expand Up @@ -66,18 +66,6 @@ void addThreadMetrics(List<MetricFamilySamples> sampleFamilies) {
"Started thread count of a JVM",
threadBean.getTotalStartedThreadCount()));

sampleFamilies.add(
new GaugeMetricFamily(
"jvm_threads_deadlocked",
"Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers",
nullSafeArrayLength(threadBean.findDeadlockedThreads())));

sampleFamilies.add(
new GaugeMetricFamily(
"jvm_threads_deadlocked_monitor",
"Cycles of JVM-threads that are in deadlock waiting to acquire object monitors",
nullSafeArrayLength(threadBean.findMonitorDeadlockedThreads())));

GaugeMetricFamily threadStateFamily = new GaugeMetricFamily(
"jvm_threads_state",
"Current count of threads by state",
Expand Down Expand Up @@ -114,10 +102,7 @@ private Map<Thread.State, Integer> getThreadStateCountMap() {
return threadCounts;
}

private static double nullSafeArrayLength(long[] array) {
return null == array ? 0 : array.length;
}

@Override
public List<MetricFamilySamples> collect() {
List<MetricFamilySamples> mfs = new ArrayList<MetricFamilySamples>();
addThreadMetrics(mfs);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
import io.prometheus.client.Collector;
import io.prometheus.client.Info;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

/**
Expand All @@ -22,7 +20,7 @@
* </pre>
*/

public class VersionInfoExports extends Collector {
public class VersionInfoExports extends Collector implements HotspotCollector {


public List<MetricFamilySamples> collect() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
package io.prometheus.client.hotspot;

import io.prometheus.client.CollectorRegistry;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;

import java.lang.management.ThreadMXBean;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.mockito.Mockito.when;

public class DeadlockExportsTest {

private ThreadMXBean mockThreadsBean = Mockito.mock(ThreadMXBean.class);
private CollectorRegistry registry = new CollectorRegistry();
private DeadlockExports collectorUnderTest;

private static final String[] EMPTY_LABEL = new String[0];

@Before
public void setUp() {
when(mockThreadsBean.findDeadlockedThreads()).thenReturn(new long[]{1L,2L,3L});
when(mockThreadsBean.findMonitorDeadlockedThreads()).thenReturn(new long[]{2L,3L,4L});
collectorUnderTest = new DeadlockExports(mockThreadsBean).register(registry);
}

@Test
public void testDeadlockExports() {
assertNull(
registry.getSampleValue(
"jvm_threads_current", EMPTY_LABEL, EMPTY_LABEL));
assertNull(
registry.getSampleValue(
"jvm_threads_daemon", EMPTY_LABEL, EMPTY_LABEL));
assertNull(
registry.getSampleValue(
"jvm_threads_peak", EMPTY_LABEL, EMPTY_LABEL));
assertNull(
registry.getSampleValue(
"jvm_threads_started_total", EMPTY_LABEL, EMPTY_LABEL));
assertEquals(
3L,
registry.getSampleValue(
"jvm_threads_deadlocked", EMPTY_LABEL, EMPTY_LABEL),
.0000001);
assertEquals(
3L,
registry.getSampleValue(
"jvm_threads_deadlocked_monitor", EMPTY_LABEL, EMPTY_LABEL),
.0000001);
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package io.prometheus.client.hotspot;

import io.prometheus.client.exporter.MetricsServlet;
import io.prometheus.client.hotspot.StandardExports;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@

import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
import java.util.Arrays;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.mockito.Mockito.when;

public class ThreadExportsTest {
Expand All @@ -34,8 +34,6 @@ public void setUp() {
when(mockThreadsBean.getDaemonThreadCount()).thenReturn(200);
when(mockThreadsBean.getPeakThreadCount()).thenReturn(301);
when(mockThreadsBean.getTotalStartedThreadCount()).thenReturn(503L);
when(mockThreadsBean.findDeadlockedThreads()).thenReturn(new long[]{1L,2L,3L});
when(mockThreadsBean.findMonitorDeadlockedThreads()).thenReturn(new long[]{2L,3L,4L});
when(mockThreadsBean.getAllThreadIds()).thenReturn(new long[]{3L,4L,5L});
when(mockThreadInfoBlocked.getThreadState()).thenReturn(Thread.State.BLOCKED);
when(mockThreadInfoRunnable1.getThreadState()).thenReturn(Thread.State.RUNNABLE);
Expand Down Expand Up @@ -68,16 +66,12 @@ public void testThreadPools() {
registry.getSampleValue(
"jvm_threads_started_total", EMPTY_LABEL, EMPTY_LABEL),
.0000001);
assertEquals(
3L,
assertNull(
registry.getSampleValue(
"jvm_threads_deadlocked", EMPTY_LABEL, EMPTY_LABEL),
.0000001);
assertEquals(
3L,
"jvm_threads_deadlocked", EMPTY_LABEL, EMPTY_LABEL));
assertNull(
registry.getSampleValue(
"jvm_threads_deadlocked_monitor", EMPTY_LABEL, EMPTY_LABEL),
.0000001);
"jvm_threads_deadlocked_monitor", EMPTY_LABEL, EMPTY_LABEL));

assertEquals(
1L,
Expand Down