|
| 1 | +package jdk.sandbox.compatibility; |
| 2 | + |
| 3 | +import jdk.sandbox.java.util.json.Json; |
| 4 | +import jdk.sandbox.java.util.json.JsonArray; |
| 5 | +import jdk.sandbox.java.util.json.JsonObject; |
| 6 | +import jdk.sandbox.java.util.json.JsonString; |
| 7 | +import jdk.sandbox.java.util.json.JsonNumber; |
| 8 | +import jdk.sandbox.java.util.json.JsonParseException; |
| 9 | + |
| 10 | +import java.nio.charset.MalformedInputException; |
| 11 | +import java.nio.charset.StandardCharsets; |
| 12 | +import java.nio.file.Files; |
| 13 | +import java.nio.file.Path; |
| 14 | +import java.nio.file.Paths; |
| 15 | +import java.util.ArrayList; |
| 16 | +import java.util.List; |
| 17 | +import java.util.logging.Logger; |
| 18 | + |
| 19 | +/** |
| 20 | + * Generates a conformance summary report. |
| 21 | + * Run with: mvn exec:java -pl json-compatibility-suite |
| 22 | + */ |
| 23 | +public class JsonTestSuiteSummary { |
| 24 | + |
| 25 | + private static final Logger LOGGER = Logger.getLogger(JsonTestSuiteSummary.class.getName()); |
| 26 | + private static final Path TEST_DIR = Paths.get("json-compatibility-suite/target/test-resources/JSONTestSuite-master/test_parsing"); |
| 27 | + |
| 28 | + public static void main(String[] args) throws Exception { |
| 29 | + boolean jsonOutput = args.length > 0 && "--json".equals(args[0]); |
| 30 | + JsonTestSuiteSummary summary = new JsonTestSuiteSummary(); |
| 31 | + if (jsonOutput) { |
| 32 | + summary.generateJsonReport(); |
| 33 | + } else { |
| 34 | + summary.generateConformanceReport(); |
| 35 | + } |
| 36 | + } |
| 37 | + |
| 38 | + void generateConformanceReport() throws Exception { |
| 39 | + TestResults results = runTests(); |
| 40 | + |
| 41 | + System.out.println("\n=== JSON Test Suite Conformance Report ==="); |
| 42 | + System.out.println("Repository: java.util.json backport"); |
| 43 | + System.out.printf("Test files analyzed: %d%n", results.totalFiles); |
| 44 | + System.out.printf("Files skipped (could not read): %d%n%n", results.skippedFiles); |
| 45 | + |
| 46 | + System.out.println("Valid JSON (y_ files):"); |
| 47 | + System.out.printf(" Passed: %d%n", results.yPass); |
| 48 | + System.out.printf(" Failed: %d%n", results.yFail); |
| 49 | + System.out.printf(" Success rate: %.1f%%%n%n", 100.0 * results.yPass / (results.yPass + results.yFail)); |
| 50 | + |
| 51 | + System.out.println("Invalid JSON (n_ files):"); |
| 52 | + System.out.printf(" Correctly rejected: %d%n", results.nPass); |
| 53 | + System.out.printf(" Incorrectly accepted: %d%n", results.nFail); |
| 54 | + System.out.printf(" Success rate: %.1f%%%n%n", 100.0 * results.nPass / (results.nPass + results.nFail)); |
| 55 | + |
| 56 | + System.out.println("Implementation-defined (i_ files):"); |
| 57 | + System.out.printf(" Accepted: %d%n", results.iAccept); |
| 58 | + System.out.printf(" Rejected: %d%n%n", results.iReject); |
| 59 | + |
| 60 | + double conformance = 100.0 * (results.yPass + results.nPass) / (results.yPass + results.yFail + results.nPass + results.nFail); |
| 61 | + System.out.printf("Overall Conformance: %.1f%%%n", conformance); |
| 62 | + |
| 63 | + if (!results.shouldPassButFailed.isEmpty()) { |
| 64 | + System.out.println("\n⚠️ Valid JSON that failed to parse:"); |
| 65 | + results.shouldPassButFailed.forEach(f -> System.out.println(" - " + f)); |
| 66 | + } |
| 67 | + |
| 68 | + if (!results.shouldFailButPassed.isEmpty()) { |
| 69 | + System.out.println("\n⚠️ Invalid JSON that was incorrectly accepted:"); |
| 70 | + results.shouldFailButPassed.forEach(f -> System.out.println(" - " + f)); |
| 71 | + } |
| 72 | + |
| 73 | + if (results.shouldPassButFailed.isEmpty() && results.shouldFailButPassed.isEmpty()) { |
| 74 | + System.out.println("\n✅ Perfect conformance!"); |
| 75 | + } |
| 76 | + } |
| 77 | + |
| 78 | + void generateJsonReport() throws Exception { |
| 79 | + TestResults results = runTests(); |
| 80 | + JsonObject report = createJsonReport(results); |
| 81 | + System.out.println(Json.toDisplayString(report, 2)); |
| 82 | + } |
| 83 | + |
| 84 | + private TestResults runTests() throws Exception { |
| 85 | + if (!Files.exists(TEST_DIR)) { |
| 86 | + throw new RuntimeException("Test suite not downloaded. Run: ./mvnw clean compile generate-test-resources -pl json-compatibility-suite"); |
| 87 | + } |
| 88 | + |
| 89 | + List<String> shouldPassButFailed = new ArrayList<>(); |
| 90 | + List<String> shouldFailButPassed = new ArrayList<>(); |
| 91 | + List<String> skippedFiles = new ArrayList<>(); |
| 92 | + |
| 93 | + int yPass = 0, yFail = 0; |
| 94 | + int nPass = 0, nFail = 0; |
| 95 | + int iAccept = 0, iReject = 0; |
| 96 | + |
| 97 | + var files = Files.walk(TEST_DIR) |
| 98 | + .filter(p -> p.toString().endsWith(".json")) |
| 99 | + .sorted() |
| 100 | + .toList(); |
| 101 | + |
| 102 | + for (Path file : files) { |
| 103 | + String filename = file.getFileName().toString(); |
| 104 | + String content = null; |
| 105 | + char[] charContent = null; |
| 106 | + |
| 107 | + try { |
| 108 | + content = Files.readString(file, StandardCharsets.UTF_8); |
| 109 | + charContent = content.toCharArray(); |
| 110 | + } catch (MalformedInputException e) { |
| 111 | + LOGGER.warning("UTF-8 failed for " + filename + ", using robust encoding detection"); |
| 112 | + try { |
| 113 | + byte[] rawBytes = Files.readAllBytes(file); |
| 114 | + charContent = RobustCharDecoder.decodeToChars(rawBytes, filename); |
| 115 | + } catch (Exception ex) { |
| 116 | + throw new RuntimeException("Failed to read test file " + filename + " - this is a fundamental I/O failure, not an encoding issue: " + ex.getMessage(), ex); |
| 117 | + } |
| 118 | + } |
| 119 | + |
| 120 | + // Test with char[] API (always available) |
| 121 | + boolean parseSucceeded = false; |
| 122 | + try { |
| 123 | + Json.parse(charContent); |
| 124 | + parseSucceeded = true; |
| 125 | + } catch (JsonParseException e) { |
| 126 | + parseSucceeded = false; |
| 127 | + } catch (StackOverflowError e) { |
| 128 | + LOGGER.warning("StackOverflowError on file: " + filename); |
| 129 | + parseSucceeded = false; // Treat as parse failure |
| 130 | + } |
| 131 | + |
| 132 | + // Update counters based on results |
| 133 | + if (parseSucceeded) { |
| 134 | + if (filename.startsWith("y_")) { |
| 135 | + yPass++; |
| 136 | + } else if (filename.startsWith("n_")) { |
| 137 | + nFail++; |
| 138 | + shouldFailButPassed.add(filename); |
| 139 | + } else if (filename.startsWith("i_")) { |
| 140 | + iAccept++; |
| 141 | + } |
| 142 | + } else { |
| 143 | + if (filename.startsWith("y_")) { |
| 144 | + yFail++; |
| 145 | + shouldPassButFailed.add(filename); |
| 146 | + } else if (filename.startsWith("n_")) { |
| 147 | + nPass++; |
| 148 | + } else if (filename.startsWith("i_")) { |
| 149 | + iReject++; |
| 150 | + } |
| 151 | + } |
| 152 | + } |
| 153 | + |
| 154 | + return new TestResults(files.size(), skippedFiles.size(), |
| 155 | + yPass, yFail, nPass, nFail, iAccept, iReject, |
| 156 | + shouldPassButFailed, shouldFailButPassed, skippedFiles); |
| 157 | + } |
| 158 | + |
| 159 | + private JsonObject createJsonReport(TestResults results) { |
| 160 | + double ySuccessRate = 100.0 * results.yPass / (results.yPass + results.yFail); |
| 161 | + double nSuccessRate = 100.0 * results.nPass / (results.nPass + results.nFail); |
| 162 | + double conformance = 100.0 * (results.yPass + results.nPass) / (results.yPass + results.yFail + results.nPass + results.nFail); |
| 163 | + |
| 164 | + return JsonObject.of(java.util.Map.of( |
| 165 | + "repository", JsonString.of("java.util.json backport"), |
| 166 | + "filesAnalyzed", JsonNumber.of(results.totalFiles), |
| 167 | + "filesSkipped", JsonNumber.of(results.skippedFiles), |
| 168 | + "validJson", JsonObject.of(java.util.Map.of( |
| 169 | + "passed", JsonNumber.of(results.yPass), |
| 170 | + "failed", JsonNumber.of(results.yFail), |
| 171 | + "successRate", JsonNumber.of(Math.round(ySuccessRate * 10) / 10.0) |
| 172 | + )), |
| 173 | + "invalidJson", JsonObject.of(java.util.Map.of( |
| 174 | + "correctlyRejected", JsonNumber.of(results.nPass), |
| 175 | + "incorrectlyAccepted", JsonNumber.of(results.nFail), |
| 176 | + "successRate", JsonNumber.of(Math.round(nSuccessRate * 10) / 10.0) |
| 177 | + )), |
| 178 | + "implementationDefined", JsonObject.of(java.util.Map.of( |
| 179 | + "accepted", JsonNumber.of(results.iAccept), |
| 180 | + "rejected", JsonNumber.of(results.iReject) |
| 181 | + )), |
| 182 | + "overallConformance", JsonNumber.of(Math.round(conformance * 10) / 10.0), |
| 183 | + "shouldPassButFailed", JsonArray.of(results.shouldPassButFailed.stream() |
| 184 | + .map(JsonString::of) |
| 185 | + .toList()), |
| 186 | + "shouldFailButPassed", JsonArray.of(results.shouldFailButPassed.stream() |
| 187 | + .map(JsonString::of) |
| 188 | + .toList()) |
| 189 | + )); |
| 190 | + } |
| 191 | + |
| 192 | + private record TestResults( |
| 193 | + int totalFiles, int skippedFiles, |
| 194 | + int yPass, int yFail, int nPass, int nFail, int iAccept, int iReject, |
| 195 | + List<String> shouldPassButFailed, List<String> shouldFailButPassed, List<String> skippedFiles2 |
| 196 | + ) {} |
| 197 | +} |
0 commit comments