From bcdd2e5b71c2fe47b2115c18c35f7883bfed413b Mon Sep 17 00:00:00 2001 From: Danny Thomas Date: Wed, 4 Sep 2024 12:21:58 +1000 Subject: [PATCH] Add Java Flight Recorder support (#1154) This adds Java Flight Recorder support using JFR Event Streaming to begin adding measures that are not available via other means. For starters this includes Virtual Threads and ZGC. Also moves classloading, compilation, and thread metrics from JMX to JFR when running on later JDKs. This is done internally to Jmx helper so most integrations will pick this up automatically. --------- Co-authored-by: brharrington --- spectator-ext-jvm/build.gradle | 51 +++++- .../spectator/jvm/JavaFlightRecorder.java | 53 ++++++ .../java/com/netflix/spectator/jvm/Jmx.java | 18 +- .../spectator/jvm/JavaFlightRecorder.java | 163 ++++++++++++++++++ .../JavaFlightRecorderUnsupportedTest.java | 36 ++++ .../spectator/jvm/JavaFlightRecorderTest.java | 82 +++++++++ 6 files changed, 398 insertions(+), 5 deletions(-) create mode 100644 spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/JavaFlightRecorder.java create mode 100644 spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java create mode 100644 spectator-ext-jvm/src/test/java/com/netflix/spectator/jvm/JavaFlightRecorderUnsupportedTest.java create mode 100644 spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java diff --git a/spectator-ext-jvm/build.gradle b/spectator-ext-jvm/build.gradle index e882d8741..fdebd29ed 100644 --- a/spectator-ext-jvm/build.gradle +++ b/spectator-ext-jvm/build.gradle @@ -1,12 +1,59 @@ +sourceSets { + java17 { + java { + srcDirs = ['src/main/java17'] + compileClasspath = configurations.compileClasspath + runtimeClasspath = configurations.runtimeClasspath + } + } + java17Test { + java { + srcDirs = ['src/test/java17'] + compileClasspath = jar.outputs.files + configurations.testCompileClasspath + runtimeClasspath = jar.outputs.files + runtimeClasspath + configurations.testRuntimeClasspath + } + } +} + dependencies { api project(':spectator-api') implementation 'com.typesafe:config' + testImplementation 'com.google.code.findbugs:annotations:3.0.1u2' +} + +def java17Compiler = javaToolchains.compilerFor { + languageVersion = JavaLanguageVersion.of(17) +} + +tasks.named('compileJava17Java', JavaCompile).configure { + javaCompiler = java17Compiler } -jar { +tasks.named('compileJava17TestJava', JavaCompile).configure { + javaCompiler = java17Compiler +} + +tasks.named('jar').configure { + into('META-INF/versions/17') { + from sourceSets.java17.output + } manifest { attributes( - "Automatic-Module-Name": "com.netflix.spectator.jvm" + 'Automatic-Module-Name': 'com.netflix.spectator.jvm', + 'Multi-Release': 'true' ) } } + +def testJava17 = tasks.register('testJava17', Test) { + description = "Runs tests for java17Test sourceset." + group = 'verification' + + testClassesDirs = sourceSets.java17Test.output.classesDirs + classpath = sourceSets.java17Test.runtimeClasspath + + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(17) + } +} +check.dependsOn testJava17 diff --git a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/JavaFlightRecorder.java b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/JavaFlightRecorder.java new file mode 100644 index 000000000..695d574d7 --- /dev/null +++ b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/JavaFlightRecorder.java @@ -0,0 +1,53 @@ +/* + * Copyright 2014-2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spectator.jvm; + +import com.netflix.spectator.api.Registry; + +import java.util.concurrent.Executor; + +/** + * Helpers supporting continuous monitoring with Java Flight Recorder. + */ +public final class JavaFlightRecorder { + + private JavaFlightRecorder() { + } + + /** + * Return if Java Flight Recorder continuous monitoring is supported on the current JVM. + */ + public static boolean isSupported() { + return false; + } + + /** + * Collect low-overhead Java Flight Recorder events, using the provided + * {@link java.util.concurrent.Executor} to execute a single task to collect events. + *

+ * These measures provide parity with {@link Jmx#registerStandardMXBeans} and the + * `spectator-ext-gc` module. + * + * @param registry the registry + * @param executor the executor to execute the task for streaming events + * @return an {@link AutoCloseable} allowing the underlying event stream to be closed + */ + public static AutoCloseable monitorDefaultEvents(Registry registry, Executor executor) { + throw new UnsupportedOperationException("Java Flight Recorder support is only available on Java 17 and later"); + } + +} diff --git a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java index aec28f720..6e11fa99b 100644 --- a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java +++ b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java @@ -27,6 +27,8 @@ import java.lang.management.ManagementFactory; import java.lang.management.MemoryPoolMXBean; import java.lang.management.ThreadMXBean; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; /** * Helpers for working with JMX mbeans. @@ -42,9 +44,19 @@ private Jmx() { * mbeans from the local jvm. */ public static void registerStandardMXBeans(Registry registry) { - monitorClassLoadingMXBean(registry); - monitorThreadMXBean(registry); - monitorCompilationMXBean(registry); + if (JavaFlightRecorder.isSupported()) { + Executor executor = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "spectator-jfr"); + t.setDaemon(true); + return t; + }); + JavaFlightRecorder.monitorDefaultEvents(registry, executor); + return; + } else { + monitorClassLoadingMXBean(registry); + monitorThreadMXBean(registry); + monitorCompilationMXBean(registry); + } maybeRegisterHotspotInternal(registry); for (MemoryPoolMXBean mbean : ManagementFactory.getMemoryPoolMXBeans()) { diff --git a/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java new file mode 100644 index 000000000..d92cba9ca --- /dev/null +++ b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java @@ -0,0 +1,163 @@ +/* + * Copyright 2014-2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spectator.jvm; + +import com.netflix.spectator.api.Counter; +import com.netflix.spectator.api.Gauge; +import com.netflix.spectator.api.Registry; +import com.netflix.spectator.api.Timer; +import jdk.jfr.EventSettings; +import jdk.jfr.consumer.RecordedEvent; +import jdk.jfr.consumer.RecordingStream; + +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +public class JavaFlightRecorder { + + private static final String PREFIX = "jdk."; + private static final String ClassLoadingStatistics = PREFIX + "ClassLoadingStatistics"; + private static final String CompilerStatistics = PREFIX + "CompilerStatistics"; + private static final String JavaThreadStatistics = PREFIX + "JavaThreadStatistics"; + private static final String VirtualThreadPinned = PREFIX + "VirtualThreadPinned"; + private static final String VirtualThreadSubmitFailed = PREFIX + "VirtualThreadSubmitFailed"; + private static final String YoungGarbageCollection = PREFIX + "YoungGarbageCollection"; + private static final String ZAllocationStall = PREFIX + "ZAllocationStall"; + private static final String ZYoungGarbageCollection = PREFIX + "ZYoungGarbageCollection"; + + private JavaFlightRecorder() { + } + + public static boolean isSupported() { + try { + Class.forName("jdk.jfr.consumer.RecordingStream"); + } catch (ClassNotFoundException e) { + return false; + } + return true; + } + + public static AutoCloseable monitorDefaultEvents(Registry registry, Executor executor) { + if (!isSupported()) { + throw new UnsupportedOperationException("This JVM does not support Java Flight Recorder event streaming"); + } + Objects.requireNonNull(registry); + Objects.requireNonNull(executor); + RecordingStream rs = new RecordingStream(); + collectClassLoadingStatistics(registry, rs); + collectCompilerStatistics(registry, rs); + collectThreadStatistics(registry, rs); + collectVirtualThreadEvents(registry, rs); + collectGcEvents(registry, rs); + executor.execute(rs::start); + return rs::close; + } + + private static void collectClassLoadingStatistics(Registry registry, RecordingStream rs) { + Counter classesLoaded = registry.counter("jvm.classloading.classesLoaded"); + AtomicLong prevLoadedClassCount = new AtomicLong(); + Counter classesUnloaded = registry.counter("jvm.classloading.classesUnloaded"); + AtomicLong prevUnloadedClassCount = new AtomicLong(); + consume(ClassLoadingStatistics, rs, event -> { + long loadedClassCount = event.getLong("loadedClassCount"); + loadedClassCount = loadedClassCount - prevLoadedClassCount.getAndSet(loadedClassCount); + classesLoaded.increment(loadedClassCount); + + long unloadedClassCount = event.getLong("unloadedClassCount"); + unloadedClassCount = unloadedClassCount - prevUnloadedClassCount.getAndSet(unloadedClassCount); + classesUnloaded.increment(unloadedClassCount); + }); + } + + private static void collectCompilerStatistics(Registry registry, RecordingStream rs) { + Counter compilationTime = registry.counter("jvm.compilation.compilationTime"); + AtomicLong prevTotalTimeSpent = new AtomicLong(); + consume(CompilerStatistics, rs, event -> { + long totalTimeSpent = event.getLong("totalTimeSpent"); + totalTimeSpent = totalTimeSpent - prevTotalTimeSpent.getAndAdd(totalTimeSpent); + compilationTime.add(totalTimeSpent / 1000.0); + }); + } + + private static void collectThreadStatistics(Registry registry, RecordingStream rs) { + Gauge nonDaemonThreadCount = registry.gauge("jvm.thread.threadCount", "id", "non-daemon"); + Gauge daemonThreadCount = registry.gauge("jvm.thread.threadCount", "id", "daemon"); + Counter threadsStarted = registry.counter("jvm.thread.threadsStarted"); + AtomicLong prevAccumulatedCount = new AtomicLong(); + consume(JavaThreadStatistics, rs, event -> { + long activeCount = event.getLong("activeCount"); + long daemonCount = event.getLong("daemonCount"); + long nonDaemonCount = activeCount - daemonCount; + nonDaemonThreadCount.set(nonDaemonCount); + daemonThreadCount.set(daemonCount); + long accumulatedCount = event.getLong("accumulatedCount"); + accumulatedCount = accumulatedCount - prevAccumulatedCount.getAndSet(accumulatedCount); + threadsStarted.increment(accumulatedCount); + }); + } + + private static void collectVirtualThreadEvents(Registry registry, RecordingStream rs) { + Timer pinned = registry.timer("jvm.vt.pinned"); + Counter submitFailed = registry.counter("jvm.vt.submitFailed"); + // 20ms threshold set to match default behavior + consume(VirtualThreadPinned, rs, event -> + pinned.record(event.getDuration()) + ).withThreshold(Duration.ofMillis(20)); + consume(VirtualThreadSubmitFailed, rs, event -> + submitFailed.increment() + ); + } + + private static void collectGcEvents(Registry registry, RecordingStream rs) { + // ZGC and Shenandoah are not covered by the generic event, there is + // a ZGC specific event to get coverage there, right now there doesn't + // appear to be similar data available for Shenandoah + Gauge tenuringThreshold = registry.gauge("jvm.gc.tenuringThreshold"); + Consumer tenuringThresholdFn = event -> + tenuringThreshold.set(event.getLong("tenuringThreshold")); + consume(YoungGarbageCollection, rs, tenuringThresholdFn); + consume(ZYoungGarbageCollection, rs, tenuringThresholdFn); + + consume(ZAllocationStall, rs, event -> + registry.timer("jvm.gc.allocationStall", "type", event.getString("type")) + .record(event.getDuration())); + } + + /** + * Consume a given JFR event. For full event details see the event definitions and default/profiling configuration: + *

+ * - metadata.xml + * - default.jfc + * - profile.jfc + *

+ * We avoid the default event configurations because despite their claims of "low-overhead" there are + * situtations where they can impose significant overhead to the application. + */ + private static EventSettings consume(String name, RecordingStream rs, Consumer consumer) { + // Apply sensible defaults to settings to avoid the overhead of collecting unnecessary stacktraces + // and collecting periodic events at a finer interval than we require upstream + EventSettings settings = rs.enable(name) + .withoutStackTrace() + .withThreshold(Duration.ofMillis(0)) + .withPeriod(Duration.ofSeconds(5)); + rs.onEvent(name, consumer); + return settings; + } + +} diff --git a/spectator-ext-jvm/src/test/java/com/netflix/spectator/jvm/JavaFlightRecorderUnsupportedTest.java b/spectator-ext-jvm/src/test/java/com/netflix/spectator/jvm/JavaFlightRecorderUnsupportedTest.java new file mode 100644 index 000000000..e54224db9 --- /dev/null +++ b/spectator-ext-jvm/src/test/java/com/netflix/spectator/jvm/JavaFlightRecorderUnsupportedTest.java @@ -0,0 +1,36 @@ +/* + * Copyright 2014-2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spectator.jvm; + +import com.netflix.spectator.api.NoopRegistry; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class JavaFlightRecorderUnsupportedTest { + + @Test + public void isUnsupported() { + Assertions.assertFalse(JavaFlightRecorder.isSupported()); + } + + @Test + public void monitorThrowsUOE() { + Assertions.assertThrows(UnsupportedOperationException.class, () -> + JavaFlightRecorder.monitorDefaultEvents(new NoopRegistry(), Runnable::run)); + } + +} diff --git a/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java b/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java new file mode 100644 index 000000000..7b4642202 --- /dev/null +++ b/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java @@ -0,0 +1,82 @@ +/* + * Copyright 2014-2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spectator.jvm; + +import com.netflix.spectator.api.DefaultRegistry; +import com.netflix.spectator.api.Id; +import com.netflix.spectator.api.Measurement; +import com.netflix.spectator.api.Registry; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; + +public class JavaFlightRecorderTest { + + @SuppressFBWarnings + public static volatile Object obj; + + @Test + public void isSupported() { + assertTrue(JavaFlightRecorder.isSupported()); + } + + @Test + public void checkDefaultMeasures() throws Exception { + Registry registry = new DefaultRegistry(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (var closable = JavaFlightRecorder.monitorDefaultEvents(registry, executor)) { + // allocate rapidly to trigger a GC, black holing using the approach from + // https://github.com/openjdk/jdk/blob/master/test/hotspot/jtreg/gc/testlibrary/Allocation.java + for (int i = 0; i < 100; i++) { + obj = new byte[4 * 1024 * 1024]; + obj = null; + } + Thread.sleep(6000); + } + executor.shutdownNow(); + + Map measures = registry.measurements() + .collect(Collectors.toMap(Measurement::id, m -> m)); + + Measurement classesLoaded = measures.get(Id.create("jvm.classloading.classesLoaded")); + Measurement classesUnloaded = measures.get(Id.create("jvm.classloading.classesUnloaded")); + assertNotEquals(null, classesLoaded); + assertNotEquals(null, classesUnloaded); + assertTrue(classesLoaded.value() > 3000 && classesLoaded.value() < 4000); + assertEquals(0, classesUnloaded.value()); + + Measurement compilationTime = measures.get(Id.create("jvm.compilation.compilationTime")); + assertNotEquals(null, compilationTime); + + Measurement nonDaemonThreadCount = measures.get(Id.create("jvm.thread.threadCount").withTag("id", "non-daemon")); + Measurement daemonThreadCount = measures.get(Id.create("jvm.thread.threadCount").withTag("id", "daemon")); + Measurement threadsStarted = measures.get(Id.create("jvm.thread.threadsStarted")); + assertNotEquals(null, nonDaemonThreadCount); + assertEquals(5, nonDaemonThreadCount.value()); + assertNotEquals(null, daemonThreadCount); + assertEquals(7, daemonThreadCount.value()); + assertNotEquals(null, threadsStarted); + assertEquals(12, threadsStarted.value()); + } + +}