diff --git a/spectator-ext-jvm/build.gradle b/spectator-ext-jvm/build.gradle index e882d8741..fdebd29ed 100644 --- a/spectator-ext-jvm/build.gradle +++ b/spectator-ext-jvm/build.gradle @@ -1,12 +1,59 @@ +sourceSets { + java17 { + java { + srcDirs = ['src/main/java17'] + compileClasspath = configurations.compileClasspath + runtimeClasspath = configurations.runtimeClasspath + } + } + java17Test { + java { + srcDirs = ['src/test/java17'] + compileClasspath = jar.outputs.files + configurations.testCompileClasspath + runtimeClasspath = jar.outputs.files + runtimeClasspath + configurations.testRuntimeClasspath + } + } +} + dependencies { api project(':spectator-api') implementation 'com.typesafe:config' + testImplementation 'com.google.code.findbugs:annotations:3.0.1u2' +} + +def java17Compiler = javaToolchains.compilerFor { + languageVersion = JavaLanguageVersion.of(17) +} + +tasks.named('compileJava17Java', JavaCompile).configure { + javaCompiler = java17Compiler } -jar { +tasks.named('compileJava17TestJava', JavaCompile).configure { + javaCompiler = java17Compiler +} + +tasks.named('jar').configure { + into('META-INF/versions/17') { + from sourceSets.java17.output + } manifest { attributes( - "Automatic-Module-Name": "com.netflix.spectator.jvm" + 'Automatic-Module-Name': 'com.netflix.spectator.jvm', + 'Multi-Release': 'true' ) } } + +def testJava17 = tasks.register('testJava17', Test) { + description = "Runs tests for java17Test sourceset." + group = 'verification' + + testClassesDirs = sourceSets.java17Test.output.classesDirs + classpath = sourceSets.java17Test.runtimeClasspath + + javaLauncher = javaToolchains.launcherFor { + languageVersion = JavaLanguageVersion.of(17) + } +} +check.dependsOn testJava17 diff --git a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/JavaFlightRecorder.java b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/JavaFlightRecorder.java new file mode 100644 index 000000000..695d574d7 --- /dev/null +++ b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/JavaFlightRecorder.java @@ -0,0 +1,53 @@ +/* + * Copyright 2014-2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spectator.jvm; + +import com.netflix.spectator.api.Registry; + +import java.util.concurrent.Executor; + +/** + * Helpers supporting continuous monitoring with Java Flight Recorder. + */ +public final class JavaFlightRecorder { + + private JavaFlightRecorder() { + } + + /** + * Return if Java Flight Recorder continuous monitoring is supported on the current JVM. + */ + public static boolean isSupported() { + return false; + } + + /** + * Collect low-overhead Java Flight Recorder events, using the provided + * {@link java.util.concurrent.Executor} to execute a single task to collect events. + *

+ * These measures provide parity with {@link Jmx#registerStandardMXBeans} and the + * `spectator-ext-gc` module. + * + * @param registry the registry + * @param executor the executor to execute the task for streaming events + * @return an {@link AutoCloseable} allowing the underlying event stream to be closed + */ + public static AutoCloseable monitorDefaultEvents(Registry registry, Executor executor) { + throw new UnsupportedOperationException("Java Flight Recorder support is only available on Java 17 and later"); + } + +} diff --git a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java index aec28f720..6e11fa99b 100644 --- a/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java +++ b/spectator-ext-jvm/src/main/java/com/netflix/spectator/jvm/Jmx.java @@ -27,6 +27,8 @@ import java.lang.management.ManagementFactory; import java.lang.management.MemoryPoolMXBean; import java.lang.management.ThreadMXBean; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; /** * Helpers for working with JMX mbeans. @@ -42,9 +44,19 @@ private Jmx() { * mbeans from the local jvm. */ public static void registerStandardMXBeans(Registry registry) { - monitorClassLoadingMXBean(registry); - monitorThreadMXBean(registry); - monitorCompilationMXBean(registry); + if (JavaFlightRecorder.isSupported()) { + Executor executor = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "spectator-jfr"); + t.setDaemon(true); + return t; + }); + JavaFlightRecorder.monitorDefaultEvents(registry, executor); + return; + } else { + monitorClassLoadingMXBean(registry); + monitorThreadMXBean(registry); + monitorCompilationMXBean(registry); + } maybeRegisterHotspotInternal(registry); for (MemoryPoolMXBean mbean : ManagementFactory.getMemoryPoolMXBeans()) { diff --git a/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java new file mode 100644 index 000000000..d92cba9ca --- /dev/null +++ b/spectator-ext-jvm/src/main/java17/com/netflix/spectator/jvm/JavaFlightRecorder.java @@ -0,0 +1,163 @@ +/* + * Copyright 2014-2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spectator.jvm; + +import com.netflix.spectator.api.Counter; +import com.netflix.spectator.api.Gauge; +import com.netflix.spectator.api.Registry; +import com.netflix.spectator.api.Timer; +import jdk.jfr.EventSettings; +import jdk.jfr.consumer.RecordedEvent; +import jdk.jfr.consumer.RecordingStream; + +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +public class JavaFlightRecorder { + + private static final String PREFIX = "jdk."; + private static final String ClassLoadingStatistics = PREFIX + "ClassLoadingStatistics"; + private static final String CompilerStatistics = PREFIX + "CompilerStatistics"; + private static final String JavaThreadStatistics = PREFIX + "JavaThreadStatistics"; + private static final String VirtualThreadPinned = PREFIX + "VirtualThreadPinned"; + private static final String VirtualThreadSubmitFailed = PREFIX + "VirtualThreadSubmitFailed"; + private static final String YoungGarbageCollection = PREFIX + "YoungGarbageCollection"; + private static final String ZAllocationStall = PREFIX + "ZAllocationStall"; + private static final String ZYoungGarbageCollection = PREFIX + "ZYoungGarbageCollection"; + + private JavaFlightRecorder() { + } + + public static boolean isSupported() { + try { + Class.forName("jdk.jfr.consumer.RecordingStream"); + } catch (ClassNotFoundException e) { + return false; + } + return true; + } + + public static AutoCloseable monitorDefaultEvents(Registry registry, Executor executor) { + if (!isSupported()) { + throw new UnsupportedOperationException("This JVM does not support Java Flight Recorder event streaming"); + } + Objects.requireNonNull(registry); + Objects.requireNonNull(executor); + RecordingStream rs = new RecordingStream(); + collectClassLoadingStatistics(registry, rs); + collectCompilerStatistics(registry, rs); + collectThreadStatistics(registry, rs); + collectVirtualThreadEvents(registry, rs); + collectGcEvents(registry, rs); + executor.execute(rs::start); + return rs::close; + } + + private static void collectClassLoadingStatistics(Registry registry, RecordingStream rs) { + Counter classesLoaded = registry.counter("jvm.classloading.classesLoaded"); + AtomicLong prevLoadedClassCount = new AtomicLong(); + Counter classesUnloaded = registry.counter("jvm.classloading.classesUnloaded"); + AtomicLong prevUnloadedClassCount = new AtomicLong(); + consume(ClassLoadingStatistics, rs, event -> { + long loadedClassCount = event.getLong("loadedClassCount"); + loadedClassCount = loadedClassCount - prevLoadedClassCount.getAndSet(loadedClassCount); + classesLoaded.increment(loadedClassCount); + + long unloadedClassCount = event.getLong("unloadedClassCount"); + unloadedClassCount = unloadedClassCount - prevUnloadedClassCount.getAndSet(unloadedClassCount); + classesUnloaded.increment(unloadedClassCount); + }); + } + + private static void collectCompilerStatistics(Registry registry, RecordingStream rs) { + Counter compilationTime = registry.counter("jvm.compilation.compilationTime"); + AtomicLong prevTotalTimeSpent = new AtomicLong(); + consume(CompilerStatistics, rs, event -> { + long totalTimeSpent = event.getLong("totalTimeSpent"); + totalTimeSpent = totalTimeSpent - prevTotalTimeSpent.getAndAdd(totalTimeSpent); + compilationTime.add(totalTimeSpent / 1000.0); + }); + } + + private static void collectThreadStatistics(Registry registry, RecordingStream rs) { + Gauge nonDaemonThreadCount = registry.gauge("jvm.thread.threadCount", "id", "non-daemon"); + Gauge daemonThreadCount = registry.gauge("jvm.thread.threadCount", "id", "daemon"); + Counter threadsStarted = registry.counter("jvm.thread.threadsStarted"); + AtomicLong prevAccumulatedCount = new AtomicLong(); + consume(JavaThreadStatistics, rs, event -> { + long activeCount = event.getLong("activeCount"); + long daemonCount = event.getLong("daemonCount"); + long nonDaemonCount = activeCount - daemonCount; + nonDaemonThreadCount.set(nonDaemonCount); + daemonThreadCount.set(daemonCount); + long accumulatedCount = event.getLong("accumulatedCount"); + accumulatedCount = accumulatedCount - prevAccumulatedCount.getAndSet(accumulatedCount); + threadsStarted.increment(accumulatedCount); + }); + } + + private static void collectVirtualThreadEvents(Registry registry, RecordingStream rs) { + Timer pinned = registry.timer("jvm.vt.pinned"); + Counter submitFailed = registry.counter("jvm.vt.submitFailed"); + // 20ms threshold set to match default behavior + consume(VirtualThreadPinned, rs, event -> + pinned.record(event.getDuration()) + ).withThreshold(Duration.ofMillis(20)); + consume(VirtualThreadSubmitFailed, rs, event -> + submitFailed.increment() + ); + } + + private static void collectGcEvents(Registry registry, RecordingStream rs) { + // ZGC and Shenandoah are not covered by the generic event, there is + // a ZGC specific event to get coverage there, right now there doesn't + // appear to be similar data available for Shenandoah + Gauge tenuringThreshold = registry.gauge("jvm.gc.tenuringThreshold"); + Consumer tenuringThresholdFn = event -> + tenuringThreshold.set(event.getLong("tenuringThreshold")); + consume(YoungGarbageCollection, rs, tenuringThresholdFn); + consume(ZYoungGarbageCollection, rs, tenuringThresholdFn); + + consume(ZAllocationStall, rs, event -> + registry.timer("jvm.gc.allocationStall", "type", event.getString("type")) + .record(event.getDuration())); + } + + /** + * Consume a given JFR event. For full event details see the event definitions and default/profiling configuration: + *

+ * - metadata.xml + * - default.jfc + * - profile.jfc + *

+ * We avoid the default event configurations because despite their claims of "low-overhead" there are + * situtations where they can impose significant overhead to the application. + */ + private static EventSettings consume(String name, RecordingStream rs, Consumer consumer) { + // Apply sensible defaults to settings to avoid the overhead of collecting unnecessary stacktraces + // and collecting periodic events at a finer interval than we require upstream + EventSettings settings = rs.enable(name) + .withoutStackTrace() + .withThreshold(Duration.ofMillis(0)) + .withPeriod(Duration.ofSeconds(5)); + rs.onEvent(name, consumer); + return settings; + } + +} diff --git a/spectator-ext-jvm/src/test/java/com/netflix/spectator/jvm/JavaFlightRecorderUnsupportedTest.java b/spectator-ext-jvm/src/test/java/com/netflix/spectator/jvm/JavaFlightRecorderUnsupportedTest.java new file mode 100644 index 000000000..e54224db9 --- /dev/null +++ b/spectator-ext-jvm/src/test/java/com/netflix/spectator/jvm/JavaFlightRecorderUnsupportedTest.java @@ -0,0 +1,36 @@ +/* + * Copyright 2014-2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spectator.jvm; + +import com.netflix.spectator.api.NoopRegistry; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class JavaFlightRecorderUnsupportedTest { + + @Test + public void isUnsupported() { + Assertions.assertFalse(JavaFlightRecorder.isSupported()); + } + + @Test + public void monitorThrowsUOE() { + Assertions.assertThrows(UnsupportedOperationException.class, () -> + JavaFlightRecorder.monitorDefaultEvents(new NoopRegistry(), Runnable::run)); + } + +} diff --git a/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java b/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java new file mode 100644 index 000000000..7b4642202 --- /dev/null +++ b/spectator-ext-jvm/src/test/java17/com/netflix/spectator/jvm/JavaFlightRecorderTest.java @@ -0,0 +1,82 @@ +/* + * Copyright 2014-2024 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spectator.jvm; + +import com.netflix.spectator.api.DefaultRegistry; +import com.netflix.spectator.api.Id; +import com.netflix.spectator.api.Measurement; +import com.netflix.spectator.api.Registry; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; + +public class JavaFlightRecorderTest { + + @SuppressFBWarnings + public static volatile Object obj; + + @Test + public void isSupported() { + assertTrue(JavaFlightRecorder.isSupported()); + } + + @Test + public void checkDefaultMeasures() throws Exception { + Registry registry = new DefaultRegistry(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (var closable = JavaFlightRecorder.monitorDefaultEvents(registry, executor)) { + // allocate rapidly to trigger a GC, black holing using the approach from + // https://github.com/openjdk/jdk/blob/master/test/hotspot/jtreg/gc/testlibrary/Allocation.java + for (int i = 0; i < 100; i++) { + obj = new byte[4 * 1024 * 1024]; + obj = null; + } + Thread.sleep(6000); + } + executor.shutdownNow(); + + Map measures = registry.measurements() + .collect(Collectors.toMap(Measurement::id, m -> m)); + + Measurement classesLoaded = measures.get(Id.create("jvm.classloading.classesLoaded")); + Measurement classesUnloaded = measures.get(Id.create("jvm.classloading.classesUnloaded")); + assertNotEquals(null, classesLoaded); + assertNotEquals(null, classesUnloaded); + assertTrue(classesLoaded.value() > 3000 && classesLoaded.value() < 4000); + assertEquals(0, classesUnloaded.value()); + + Measurement compilationTime = measures.get(Id.create("jvm.compilation.compilationTime")); + assertNotEquals(null, compilationTime); + + Measurement nonDaemonThreadCount = measures.get(Id.create("jvm.thread.threadCount").withTag("id", "non-daemon")); + Measurement daemonThreadCount = measures.get(Id.create("jvm.thread.threadCount").withTag("id", "daemon")); + Measurement threadsStarted = measures.get(Id.create("jvm.thread.threadsStarted")); + assertNotEquals(null, nonDaemonThreadCount); + assertEquals(5, nonDaemonThreadCount.value()); + assertNotEquals(null, daemonThreadCount); + assertEquals(7, daemonThreadCount.value()); + assertNotEquals(null, threadsStarted); + assertEquals(12, threadsStarted.value()); + } + +}