move to source parsing for local api comparison (#12)

Author: Simon Massey

json-java21-api-tracker/src/main/java/io/github/simbo1905/tracker/ApiTracker.java

@@ -8,12 +8,6 @@
 import jdk.sandbox.java.util.json.JsonBoolean;
 
 import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
 import java.net.URI;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
@@ -22,7 +16,7 @@
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -34,7 +28,6 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import javax.tools.DiagnosticCollector;
 import javax.tools.JavaCompiler;
@@ -44,14 +37,12 @@
 import javax.tools.ToolProvider;
 
 import com.sun.source.tree.ClassTree;
-import com.sun.source.tree.CompilationUnitTree;
 import com.sun.source.tree.MethodTree;
 import com.sun.source.tree.ModifiersTree;
 import com.sun.source.tree.Tree;
 import com.sun.source.tree.VariableTree;
 import com.sun.source.util.JavacTask;
 import com.sun.source.util.TreePathScanner;
-import com.sun.source.util.Trees;
 
 /// API Tracker module for comparing local and upstream JSON APIs
 /// 
@@ -61,9 +52,16 @@
 /// - Compare public APIs using compiler parsing
 /// - Generate structured diff reports
 ///
+/// Modular design supports different extraction strategies:
+/// - Binary reflection for quick class introspection
+/// - Source parsing for accurate parameter names and signatures
+/// 
 /// All functionality is exposed as static methods following functional programming principles
 public sealed interface ApiTracker permits ApiTracker.Nothing {
 
+    /// Local source root for source-based extraction
+    static final String LOCAL_SOURCE_ROOT = "json-java21/src/main/java";
+    
     /// Empty enum to seal the interface - no instances allowed
     enum Nothing implements ApiTracker {}
 
@@ -76,6 +74,33 @@ enum Nothing implements ApiTracker {}
     // GitHub base URL for upstream sources
     static final String GITHUB_BASE_URL = "https://raw.githubusercontent.com/openjdk/jdk-sandbox/refs/heads/json/src/java.base/share/classes/";
 
+    /// Fetches content from a URL
+    static String fetchFromUrl(String url) {
+        final var httpClient = HttpClient.newBuilder()
+            .connectTimeout(Duration.ofSeconds(10))
+            .build();
+        
+        try {
+            final var request = HttpRequest.newBuilder()
+                .uri(URI.create(url))
+                .timeout(Duration.ofSeconds(30))
+                .GET()
+                .build();
+            
+            final var response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+            
+            if (response.statusCode() == 200) {
+                return response.body();
+            } else if (response.statusCode() == 404) {
+                return "NOT_FOUND: Upstream file not found (possibly deleted or renamed)";
+            } else {
+                return "HTTP_ERROR: Status " + response.statusCode();
+            }
+        } catch (Exception e) {
+            return "FETCH_ERROR: " + e.getMessage();
+        }
+    }
+    
     /// Discovers all classes in the local JSON API packages
     /// @return sorted set of classes from jdk.sandbox.java.util.json and jdk.sandbox.internal.util.json
     static Set<Class<?>> discoverLocalJsonClasses() {
@@ -258,150 +283,25 @@ static String mapToUpstreamPath(String className) {
         return path + ".java";
     }
 
-    /// Extracts public API from a compiled class using reflection
-    /// @param clazz the class to extract API from
-    /// @return JSON representation of the class's public API
-    static JsonObject extractLocalApi(Class<?> clazz) {
-        Objects.requireNonNull(clazz, "clazz must not be null");
-        LOGGER.info("Extracting local API for: " + clazz.getName());
-        
-        final var apiMap = new LinkedHashMap<String, JsonValue>();
-        
-        // Basic class information
-        apiMap.put("className", JsonString.of(clazz.getSimpleName()));
-        apiMap.put("packageName", JsonString.of(clazz.getPackage() != null ? clazz.getPackage().getName() : ""));
-        apiMap.put("modifiers", extractModifiers(clazz.getModifiers()));
-        
-        // Type information
-        apiMap.put("isInterface", JsonBoolean.of(clazz.isInterface()));
-        apiMap.put("isEnum", JsonBoolean.of(clazz.isEnum()));
-        apiMap.put("isRecord", JsonBoolean.of(clazz.isRecord()));
-        apiMap.put("isSealed", JsonBoolean.of(clazz.isSealed()));
-        
-        // Inheritance
-        final var superTypes = new ArrayList<JsonValue>();
-        if (clazz.getSuperclass() != null && !Object.class.equals(clazz.getSuperclass())) {
-            superTypes.add(JsonString.of(clazz.getSuperclass().getSimpleName()));
-        }
-        Arrays.stream(clazz.getInterfaces())
-            .map(i -> JsonString.of(i.getSimpleName()))
-            .forEach(superTypes::add);
-        apiMap.put("extends", JsonArray.of(superTypes));
-        
-        // Permitted subclasses (for sealed types)
-        if (clazz.isSealed()) {
-            final var permits = Arrays.stream(clazz.getPermittedSubclasses())
-                .map(c -> JsonString.of(c.getSimpleName()))
-                .collect(Collectors.<JsonValue>toList());
-            apiMap.put("permits", JsonArray.of(permits));
+    /// Extracts local API from source file
+    static JsonObject extractLocalApiFromSource(String className) {
+        final var path = LOCAL_SOURCE_ROOT + "/" + className.replace('.', '/') + ".java";
+        try {
+            final var sourceCode = java.nio.file.Files.readString(java.nio.file.Paths.get(path));
+            return extractApiFromSource(sourceCode, className);
+        } catch (Exception e) {
+            return JsonObject.of(Map.of(
+                "error", JsonString.of("LOCAL_FILE_NOT_FOUND: " + e.getMessage()),
+                "className", JsonString.of(className)
+            ));
         }
-        
-        // Methods
-        apiMap.put("methods", extractMethods(clazz));
-        
-        // Fields
-        apiMap.put("fields", extractFields(clazz));
-        
-        // Constructors
-        apiMap.put("constructors", extractConstructors(clazz));
-        
-        return JsonObject.of(apiMap);
-    }
-    
-    /// Extracts modifiers as JSON array
-    static JsonArray extractModifiers(int modifiers) {
-        final var modList = new ArrayList<JsonValue>();
-        
-        if (Modifier.isPublic(modifiers)) modList.add(JsonString.of("public"));
-        if (Modifier.isProtected(modifiers)) modList.add(JsonString.of("protected"));
-        if (Modifier.isPrivate(modifiers)) modList.add(JsonString.of("private"));
-        if (Modifier.isStatic(modifiers)) modList.add(JsonString.of("static"));
-        if (Modifier.isFinal(modifiers)) modList.add(JsonString.of("final"));
-        if (Modifier.isAbstract(modifiers)) modList.add(JsonString.of("abstract"));
-        if (Modifier.isNative(modifiers)) modList.add(JsonString.of("native"));
-        if (Modifier.isSynchronized(modifiers)) modList.add(JsonString.of("synchronized"));
-        if (Modifier.isTransient(modifiers)) modList.add(JsonString.of("transient"));
-        if (Modifier.isVolatile(modifiers)) modList.add(JsonString.of("volatile"));
-        
-        return JsonArray.of(modList);
-    }
-    
-    /// Extracts public methods
-    static JsonObject extractMethods(Class<?> clazz) {
-        final var methodsMap = new LinkedHashMap<String, JsonValue>();
-        
-        Arrays.stream(clazz.getDeclaredMethods())
-            .filter(m -> Modifier.isPublic(m.getModifiers()))
-            .forEach(method -> {
-                final var methodInfo = new LinkedHashMap<String, JsonValue>();
-                methodInfo.put("modifiers", extractModifiers(method.getModifiers()));
-                methodInfo.put("returnType", JsonString.of(method.getReturnType().getSimpleName()));
-                methodInfo.put("genericReturnType", JsonString.of(method.getGenericReturnType().getTypeName()));
-                
-                final var params = Arrays.stream(method.getParameters())
-                    .map(p -> JsonString.of(p.getType().getSimpleName() + " " + p.getName()))
-                    .collect(Collectors.<JsonValue>toList());
-                methodInfo.put("parameters", JsonArray.of(params));
-                
-                final var exceptions = Arrays.stream(method.getExceptionTypes())
-                    .map(e -> JsonString.of(e.getSimpleName()))
-                    .collect(Collectors.<JsonValue>toList());
-                methodInfo.put("throws", JsonArray.of(exceptions));
-                
-                methodsMap.put(method.getName(), JsonObject.of(methodInfo));
-            });
-        
-        return JsonObject.of(methodsMap);
-    }
-    
-    /// Extracts public fields
-    static JsonObject extractFields(Class<?> clazz) {
-        final var fieldsMap = new LinkedHashMap<String, JsonValue>();
-        
-        Arrays.stream(clazz.getDeclaredFields())
-            .filter(f -> Modifier.isPublic(f.getModifiers()))
-            .forEach(field -> {
-                final var fieldInfo = new LinkedHashMap<String, JsonValue>();
-                fieldInfo.put("modifiers", extractModifiers(field.getModifiers()));
-                fieldInfo.put("type", JsonString.of(field.getType().getSimpleName()));
-                fieldInfo.put("genericType", JsonString.of(field.getGenericType().getTypeName()));
-                
-                fieldsMap.put(field.getName(), JsonObject.of(fieldInfo));
-            });
-        
-        return JsonObject.of(fieldsMap);
     }
 
-    /// Extracts public constructors
-    static JsonArray extractConstructors(Class<?> clazz) {
-        final var constructors = Arrays.stream(clazz.getDeclaredConstructors())
-            .filter(c -> Modifier.isPublic(c.getModifiers()))
-            .map(constructor -> {
-                final var ctorInfo = new LinkedHashMap<String, JsonValue>();
-                ctorInfo.put("modifiers", extractModifiers(constructor.getModifiers()));
-                
-                final var params = Arrays.stream(constructor.getParameters())
-                    .map(p -> JsonString.of(p.getType().getSimpleName() + " " + p.getName()))
-                    .collect(Collectors.<JsonValue>toList());
-                ctorInfo.put("parameters", JsonArray.of(params));
-                
-                final var exceptions = Arrays.stream(constructor.getExceptionTypes())
-                    .map(e -> JsonString.of(e.getSimpleName()))
-                    .collect(Collectors.<JsonValue>toList());
-                ctorInfo.put("throws", JsonArray.of(exceptions));
-                
-                return JsonObject.of(ctorInfo);
-            })
-            .collect(Collectors.<JsonValue>toList());
-        
-        return JsonArray.of(constructors);
-    }
-    
-    /// Extracts public API from upstream source code using compiler parsing
+    /// Extracts public API from source code using compiler parsing
     /// @param sourceCode the source code to parse
     /// @param className the expected class name
     /// @return JSON representation of the parsed API
-    static JsonObject extractUpstreamApi(String sourceCode, String className) {
+    static JsonObject extractApiFromSource(String sourceCode, String className) {
         Objects.requireNonNull(sourceCode, "sourceCode must not be null");
         Objects.requireNonNull(className, "className must not be null");
 
@@ -701,7 +601,11 @@ static JsonObject compareApis(JsonObject local, JsonObject upstream) {
         Objects.requireNonNull(upstream, "upstream must not be null");
 
         final var diffMap = new LinkedHashMap<String, JsonValue>();
-        final var className = ((JsonString) local.members().get("className")).value();
+        
+        // Extract class name safely
+        final var localClassName = local.members().get("className");
+        final var className = localClassName instanceof JsonString js ? 
+            js.value() : "Unknown";
 
         diffMap.put("className", JsonString.of(className));
 
@@ -980,7 +884,7 @@ static String normalizeTypeName(String typeName) {
         return normalized;
     }
 
-    /// Runs a full comparison of local vs upstream APIs
+    /// Runs source-to-source comparison for fair parameter name comparison
     /// @return complete comparison report as JSON
     static JsonObject runFullComparison() {
         LOGGER.info("Starting full API comparison");
@@ -995,20 +899,17 @@ static JsonObject runFullComparison() {
         final var localClasses = discoverLocalJsonClasses();
         LOGGER.info("Found " + localClasses.size() + " local classes");
 
-        // Fetch upstream sources
-        final var upstreamSources = fetchUpstreamSources(localClasses);
-        
         // Extract and compare APIs
         final var differences = new ArrayList<JsonValue>();
         var matchingCount = 0;
         var missingUpstream = 0;
         var differentApi = 0;
 
         for (final var clazz : localClasses) {
-            final var localApi = extractLocalApi(clazz);
-            final var upstreamSource = upstreamSources.get(clazz.getName());
-            final var upstreamApi = extractUpstreamApi(upstreamSource, clazz.getName());
-            
+            final var className = clazz.getName();
+            final var localApi = extractLocalApiFromSource(className);
+            final var upstreamSource = fetchUpstreamSource(className);
+            final var upstreamApi = extractApiFromSource(upstreamSource, className);
             final var diff = compareApis(localApi, upstreamApi);
             differences.add(diff);
 
@@ -1032,11 +933,26 @@ static JsonObject runFullComparison() {
         reportMap.put("summary", summary);
         reportMap.put("differences", JsonArray.of(differences));
 
+        
         final var duration = Duration.between(startTime, Instant.now());
         reportMap.put("durationMs", JsonNumber.of(duration.toMillis()));
 
         LOGGER.info("Comparison completed in " + duration.toMillis() + "ms");
 
         return JsonObject.of(reportMap);
     }
+    
+    /// Fetches single upstream source file
+    static String fetchUpstreamSource(String className) {
+        final var cached = FETCH_CACHE.get(className);
+        if (cached != null) {
+            return cached;
+        }
+        
+        final var upstreamPath = mapToUpstreamPath(className);
+        final var url = GITHUB_BASE_URL + upstreamPath;
+        final var source = fetchFromUrl(url);
+        FETCH_CACHE.put(className, source);
+        return source;
+    }
 }

json-java21-api-tracker/src/main/java/io/github/simbo1905/tracker/ApiTrackerRunner.java

@@ -7,22 +7,36 @@
 
 /// Command-line runner for the API Tracker
 /// 
-/// Usage: java io.github.simbo1905.tracker.ApiTrackerRunner [loglevel]
-/// where loglevel is one of: SEVERE, WARNING, INFO, FINE, FINER, FINEST
+/// Usage: java io.github.simbo1905.tracker.ApiTrackerRunner [loglevel] [mode] [sourcepath]
+/// 
+/// Arguments:
+/// - loglevel: SEVERE, WARNING, INFO, FINE, FINER, FINEST (default: INFO)
+/// - mode: binary|source (default: binary)
+///   - binary: Compare binary reflection (local) vs source parsing (remote)
+///   - source: Compare source parsing (local) vs source parsing (remote) for accurate parameter names
+/// - sourcepath: Path to local source files (required for source mode)
 public class ApiTrackerRunner {
 
     public static void main(String[] args) {
-        // Configure logging based on command line argument
+        // Parse command line arguments
         final var logLevel = args.length > 0 ? Level.parse(args[0].toUpperCase()) : Level.INFO;
+        final var mode = args.length > 1 ? args[1].toLowerCase() : "binary";
+        final var sourcePath = args.length > 2 ? args[2] : null;
+        
         configureLogging(logLevel);
 
         System.out.println("=== JSON API Tracker ===");
         System.out.println("Comparing local jdk.sandbox.java.util.json with upstream java.util.json");
         System.out.println("Log level: " + logLevel);
+        System.out.println("Mode: " + mode);
+        if (sourcePath != null) {
+            System.out.println("Local source path: " + sourcePath);
+        }
         System.out.println();
 
         try {
-            // Run the full comparison
+            // Run comparison - now only source-to-source for fair parameter comparison
+            System.out.println("Running source-to-source comparison for fair parameter names");
             final var report = ApiTracker.runFullComparison();
 
             // Pretty print the report