Skip to content

move to source parsing for local api comparison #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
simbo1905 merged 2 commits into from
Jul 27, 2025
Merged

move to source parsing for local api comparison #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1a7f5df
move to source parsing for local api comparison
simbo1905 Jul 27, 2025
b59b048
Verified API tracker detects parameter name differences
simbo1905 Jul 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
230 changes: 73 additions & 157 deletions json-java21-api-tracker/src/main/java/io/github/simbo1905/tracker/ApiTracker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,6 @@
import jdk.sandbox.java.util.json.JsonBoolean;

import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
Expand All @@ -22,7 +16,7 @@
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
Expand All @@ -34,7 +28,6 @@
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import javax.tools.DiagnosticCollector;
import javax.tools.JavaCompiler;
Expand All @@ -44,14 +37,12 @@
import javax.tools.ToolProvider;

import com.sun.source.tree.ClassTree;
import com.sun.source.tree.CompilationUnitTree;
import com.sun.source.tree.MethodTree;
import com.sun.source.tree.ModifiersTree;
import com.sun.source.tree.Tree;
import com.sun.source.tree.VariableTree;
import com.sun.source.util.JavacTask;
import com.sun.source.util.TreePathScanner;
import com.sun.source.util.Trees;

/// API Tracker module for comparing local and upstream JSON APIs
///
Expand All @@ -61,9 +52,16 @@
/// - Compare public APIs using compiler parsing
/// - Generate structured diff reports
///
/// Modular design supports different extraction strategies:
/// - Binary reflection for quick class introspection
/// - Source parsing for accurate parameter names and signatures
///
/// All functionality is exposed as static methods following functional programming principles
public sealed interface ApiTracker permits ApiTracker.Nothing {

/// Local source root for source-based extraction
static final String LOCAL_SOURCE_ROOT = "json-java21/src/main/java";

/// Empty enum to seal the interface - no instances allowed
enum Nothing implements ApiTracker {}

Expand All @@ -76,6 +74,33 @@ enum Nothing implements ApiTracker {}
// GitHub base URL for upstream sources
static final String GITHUB_BASE_URL = "https://raw.githubusercontent.com/openjdk/jdk-sandbox/refs/heads/json/src/java.base/share/classes/";

/// Fetches content from a URL
static String fetchFromUrl(String url) {
final var httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(10))
.build();

try {
final var request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(30))
.GET()
.build();

final var response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());

if (response.statusCode() == 200) {
return response.body();
} else if (response.statusCode() == 404) {
return "NOT_FOUND: Upstream file not found (possibly deleted or renamed)";
} else {
return "HTTP_ERROR: Status " + response.statusCode();
}
} catch (Exception e) {
return "FETCH_ERROR: " + e.getMessage();
}
}

/// Discovers all classes in the local JSON API packages
/// @return sorted set of classes from jdk.sandbox.java.util.json and jdk.sandbox.internal.util.json
static Set<Class<?>> discoverLocalJsonClasses() {
Expand Down Expand Up @@ -258,150 +283,25 @@ static String mapToUpstreamPath(String className) {
return path + ".java";
}

/// Extracts public API from a compiled class using reflection
/// @param clazz the class to extract API from
/// @return JSON representation of the class's public API
static JsonObject extractLocalApi(Class<?> clazz) {
Objects.requireNonNull(clazz, "clazz must not be null");
LOGGER.info("Extracting local API for: " + clazz.getName());

final var apiMap = new LinkedHashMap<String, JsonValue>();

// Basic class information
apiMap.put("className", JsonString.of(clazz.getSimpleName()));
apiMap.put("packageName", JsonString.of(clazz.getPackage() != null ? clazz.getPackage().getName() : ""));
apiMap.put("modifiers", extractModifiers(clazz.getModifiers()));

// Type information
apiMap.put("isInterface", JsonBoolean.of(clazz.isInterface()));
apiMap.put("isEnum", JsonBoolean.of(clazz.isEnum()));
apiMap.put("isRecord", JsonBoolean.of(clazz.isRecord()));
apiMap.put("isSealed", JsonBoolean.of(clazz.isSealed()));

// Inheritance
final var superTypes = new ArrayList<JsonValue>();
if (clazz.getSuperclass() != null && !Object.class.equals(clazz.getSuperclass())) {
superTypes.add(JsonString.of(clazz.getSuperclass().getSimpleName()));
}
Arrays.stream(clazz.getInterfaces())
.map(i -> JsonString.of(i.getSimpleName()))
.forEach(superTypes::add);
apiMap.put("extends", JsonArray.of(superTypes));

// Permitted subclasses (for sealed types)
if (clazz.isSealed()) {
final var permits = Arrays.stream(clazz.getPermittedSubclasses())
.map(c -> JsonString.of(c.getSimpleName()))
.collect(Collectors.<JsonValue>toList());
apiMap.put("permits", JsonArray.of(permits));
/// Extracts local API from source file
static JsonObject extractLocalApiFromSource(String className) {
final var path = LOCAL_SOURCE_ROOT + "/" + className.replace('.', '/') + ".java";
try {
final var sourceCode = java.nio.file.Files.readString(java.nio.file.Paths.get(path));
return extractApiFromSource(sourceCode, className);
} catch (Exception e) {
return JsonObject.of(Map.of(
"error", JsonString.of("LOCAL_FILE_NOT_FOUND: " + e.getMessage()),
"className", JsonString.of(className)
));
}

// Methods
apiMap.put("methods", extractMethods(clazz));

// Fields
apiMap.put("fields", extractFields(clazz));

// Constructors
apiMap.put("constructors", extractConstructors(clazz));

return JsonObject.of(apiMap);
}

/// Extracts modifiers as JSON array
static JsonArray extractModifiers(int modifiers) {
final var modList = new ArrayList<JsonValue>();

if (Modifier.isPublic(modifiers)) modList.add(JsonString.of("public"));
if (Modifier.isProtected(modifiers)) modList.add(JsonString.of("protected"));
if (Modifier.isPrivate(modifiers)) modList.add(JsonString.of("private"));
if (Modifier.isStatic(modifiers)) modList.add(JsonString.of("static"));
if (Modifier.isFinal(modifiers)) modList.add(JsonString.of("final"));
if (Modifier.isAbstract(modifiers)) modList.add(JsonString.of("abstract"));
if (Modifier.isNative(modifiers)) modList.add(JsonString.of("native"));
if (Modifier.isSynchronized(modifiers)) modList.add(JsonString.of("synchronized"));
if (Modifier.isTransient(modifiers)) modList.add(JsonString.of("transient"));
if (Modifier.isVolatile(modifiers)) modList.add(JsonString.of("volatile"));

return JsonArray.of(modList);
}

/// Extracts public methods
static JsonObject extractMethods(Class<?> clazz) {
final var methodsMap = new LinkedHashMap<String, JsonValue>();

Arrays.stream(clazz.getDeclaredMethods())
.filter(m -> Modifier.isPublic(m.getModifiers()))
.forEach(method -> {
final var methodInfo = new LinkedHashMap<String, JsonValue>();
methodInfo.put("modifiers", extractModifiers(method.getModifiers()));
methodInfo.put("returnType", JsonString.of(method.getReturnType().getSimpleName()));
methodInfo.put("genericReturnType", JsonString.of(method.getGenericReturnType().getTypeName()));

final var params = Arrays.stream(method.getParameters())
.map(p -> JsonString.of(p.getType().getSimpleName() + " " + p.getName()))
.collect(Collectors.<JsonValue>toList());
methodInfo.put("parameters", JsonArray.of(params));

final var exceptions = Arrays.stream(method.getExceptionTypes())
.map(e -> JsonString.of(e.getSimpleName()))
.collect(Collectors.<JsonValue>toList());
methodInfo.put("throws", JsonArray.of(exceptions));

methodsMap.put(method.getName(), JsonObject.of(methodInfo));
});

return JsonObject.of(methodsMap);
}

/// Extracts public fields
static JsonObject extractFields(Class<?> clazz) {
final var fieldsMap = new LinkedHashMap<String, JsonValue>();

Arrays.stream(clazz.getDeclaredFields())
.filter(f -> Modifier.isPublic(f.getModifiers()))
.forEach(field -> {
final var fieldInfo = new LinkedHashMap<String, JsonValue>();
fieldInfo.put("modifiers", extractModifiers(field.getModifiers()));
fieldInfo.put("type", JsonString.of(field.getType().getSimpleName()));
fieldInfo.put("genericType", JsonString.of(field.getGenericType().getTypeName()));

fieldsMap.put(field.getName(), JsonObject.of(fieldInfo));
});

return JsonObject.of(fieldsMap);
}

/// Extracts public constructors
static JsonArray extractConstructors(Class<?> clazz) {
final var constructors = Arrays.stream(clazz.getDeclaredConstructors())
.filter(c -> Modifier.isPublic(c.getModifiers()))
.map(constructor -> {
final var ctorInfo = new LinkedHashMap<String, JsonValue>();
ctorInfo.put("modifiers", extractModifiers(constructor.getModifiers()));

final var params = Arrays.stream(constructor.getParameters())
.map(p -> JsonString.of(p.getType().getSimpleName() + " " + p.getName()))
.collect(Collectors.<JsonValue>toList());
ctorInfo.put("parameters", JsonArray.of(params));

final var exceptions = Arrays.stream(constructor.getExceptionTypes())
.map(e -> JsonString.of(e.getSimpleName()))
.collect(Collectors.<JsonValue>toList());
ctorInfo.put("throws", JsonArray.of(exceptions));

return JsonObject.of(ctorInfo);
})
.collect(Collectors.<JsonValue>toList());

return JsonArray.of(constructors);
}

/// Extracts public API from upstream source code using compiler parsing
/// Extracts public API from source code using compiler parsing
/// @param sourceCode the source code to parse
/// @param className the expected class name
/// @return JSON representation of the parsed API
static JsonObject extractUpstreamApi(String sourceCode, String className) {
static JsonObject extractApiFromSource(String sourceCode, String className) {
Objects.requireNonNull(sourceCode, "sourceCode must not be null");
Objects.requireNonNull(className, "className must not be null");

Expand Down Expand Up @@ -701,7 +601,11 @@ static JsonObject compareApis(JsonObject local, JsonObject upstream) {
Objects.requireNonNull(upstream, "upstream must not be null");

final var diffMap = new LinkedHashMap<String, JsonValue>();
final var className = ((JsonString) local.members().get("className")).value();

// Extract class name safely
final var localClassName = local.members().get("className");
final var className = localClassName instanceof JsonString js ?
js.value() : "Unknown";

diffMap.put("className", JsonString.of(className));

Expand Down Expand Up @@ -980,7 +884,7 @@ static String normalizeTypeName(String typeName) {
return normalized;
}

/// Runs a full comparison of local vs upstream APIs
/// Runs source-to-source comparison for fair parameter name comparison
/// @return complete comparison report as JSON
static JsonObject runFullComparison() {
LOGGER.info("Starting full API comparison");
Expand All @@ -995,20 +899,17 @@ static JsonObject runFullComparison() {
final var localClasses = discoverLocalJsonClasses();
LOGGER.info("Found " + localClasses.size() + " local classes");

// Fetch upstream sources
final var upstreamSources = fetchUpstreamSources(localClasses);

// Extract and compare APIs
final var differences = new ArrayList<JsonValue>();
var matchingCount = 0;
var missingUpstream = 0;
var differentApi = 0;

for (final var clazz : localClasses) {
final var localApi = extractLocalApi(clazz);
final var upstreamSource = upstreamSources.get(clazz.getName());
final var upstreamApi = extractUpstreamApi(upstreamSource, clazz.getName());

final var className = clazz.getName();
final var localApi = extractLocalApiFromSource(className);
final var upstreamSource = fetchUpstreamSource(className);
final var upstreamApi = extractApiFromSource(upstreamSource, className);
final var diff = compareApis(localApi, upstreamApi);
differences.add(diff);

Expand All @@ -1032,11 +933,26 @@ static JsonObject runFullComparison() {
reportMap.put("summary", summary);
reportMap.put("differences", JsonArray.of(differences));


final var duration = Duration.between(startTime, Instant.now());
reportMap.put("durationMs", JsonNumber.of(duration.toMillis()));

LOGGER.info("Comparison completed in " + duration.toMillis() + "ms");

return JsonObject.of(reportMap);
}

/// Fetches single upstream source file
static String fetchUpstreamSource(String className) {
final var cached = FETCH_CACHE.get(className);
if (cached != null) {
return cached;
}

final var upstreamPath = mapToUpstreamPath(className);
final var url = GITHUB_BASE_URL + upstreamPath;
final var source = fetchFromUrl(url);
FETCH_CACHE.put(className, source);
return source;
}
}
22 changes: 18 additions & 4 deletions json-java21-api-tracker/src/main/java/io/github/simbo1905/tracker/ApiTrackerRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,36 @@

/// Command-line runner for the API Tracker
///
/// Usage: java io.github.simbo1905.tracker.ApiTrackerRunner [loglevel]
/// where loglevel is one of: SEVERE, WARNING, INFO, FINE, FINER, FINEST
/// Usage: java io.github.simbo1905.tracker.ApiTrackerRunner [loglevel] [mode] [sourcepath]
///
/// Arguments:
/// - loglevel: SEVERE, WARNING, INFO, FINE, FINER, FINEST (default: INFO)
/// - mode: binary|source (default: binary)
/// - binary: Compare binary reflection (local) vs source parsing (remote)
/// - source: Compare source parsing (local) vs source parsing (remote) for accurate parameter names
/// - sourcepath: Path to local source files (required for source mode)
public class ApiTrackerRunner {

public static void main(String[] args) {
// Configure logging based on command line argument
// Parse command line arguments
final var logLevel = args.length > 0 ? Level.parse(args[0].toUpperCase()) : Level.INFO;
final var mode = args.length > 1 ? args[1].toLowerCase() : "binary";
final var sourcePath = args.length > 2 ? args[2] : null;

configureLogging(logLevel);

System.out.println("=== JSON API Tracker ===");
System.out.println("Comparing local jdk.sandbox.java.util.json with upstream java.util.json");
System.out.println("Log level: " + logLevel);
System.out.println("Mode: " + mode);
if (sourcePath != null) {
System.out.println("Local source path: " + sourcePath);
}
System.out.println();

try {
// Run the full comparison
// Run comparison - now only source-to-source for fair parameter comparison
System.out.println("Running source-to-source comparison for fair parameter names");
final var report = ApiTracker.runFullComparison();

// Pretty print the report
Expand Down
Loading