Skip to content

Commit 327d4fc

Browse files
simbo1905simbo1905
committed
test refactors and marked as skipped
1 parent 434719c commit 327d4fc

File tree

4 files changed

+441
-589
lines changed

4 files changed

+441
-589
lines changed

‎json-java21-schema/src/test/java/io/github/simbo1905/json/schema/JsonSchemaCheck202012IT.java‎

Lines changed: 37 additions & 309 deletions
Original file line numberDiff line numberDiff line change
@@ -1,329 +1,57 @@
11
package io.github.simbo1905.json.schema;
22

3-
import com.fasterxml.jackson.databind.JsonNode;
4-
import com.fasterxml.jackson.databind.ObjectMapper;
5-
import jdk.sandbox.java.util.json.Json;
63
import org.junit.jupiter.api.DynamicTest;
74
import org.junit.jupiter.api.TestFactory;
8-
import org.junit.jupiter.api.AfterAll;
9-
import org.junit.jupiter.api.Assumptions;
105

11-
import java.io.FileInputStream;
12-
import java.io.IOException;
13-
import java.nio.file.Files;
146
import java.nio.file.Path;
157
import java.nio.file.Paths;
16-
import java.util.zip.ZipEntry;
17-
import java.util.zip.ZipInputStream;
8+
import java.util.Set;
189
import java.util.stream.Stream;
19-
import java.util.stream.StreamSupport;
20-
21-
import static org.junit.jupiter.api.Assertions.assertEquals;
2210

2311
/// Runs the official JSON-Schema-Test-Suite (Draft 2020-12) as JUnit dynamic tests.
2412
/// By default, this is lenient and will SKIP mismatches and unsupported schemas
2513
/// to provide a compatibility signal without breaking the build. Enable strict
2614
/// mode with -Djson.schema.strict=true to make mismatches fail the build.
27-
public class JsonSchemaCheck202012IT {
28-
29-
private static final Path ZIP_FILE = Paths.get("src/test/resources/json-schema-test-suite-data.zip");
30-
private static final Path TARGET_SUITE_DIR = Paths.get("target/test-data/draft2020-12");
31-
private static final ObjectMapper MAPPER = new ObjectMapper();
32-
private static final boolean STRICT = Boolean.getBoolean("json.schema.strict");
33-
private static final String METRICS_FMT = System.getProperty("json.schema.metrics", "").trim();
34-
private static final StrictMetrics METRICS = new StrictMetrics();
35-
36-
@SuppressWarnings("resource")
37-
@TestFactory
38-
Stream<DynamicTest> runOfficialSuite() throws Exception {
39-
extractTestData();
40-
return Files.walk(TARGET_SUITE_DIR)
41-
.filter(p -> p.toString().endsWith(".json"))
42-
.flatMap(this::testsFromFile);
43-
}
44-
45-
static void extractTestData() throws IOException {
46-
if (!Files.exists(ZIP_FILE)) {
47-
throw new RuntimeException("Test data ZIP file not found: " + ZIP_FILE.toAbsolutePath());
48-
}
49-
50-
// Create target directory
51-
Files.createDirectories(TARGET_SUITE_DIR.getParent());
52-
53-
// Extract ZIP file
54-
try (ZipInputStream zis = new ZipInputStream(new FileInputStream(ZIP_FILE.toFile()))) {
55-
ZipEntry entry;
56-
while ((entry = zis.getNextEntry()) != null) {
57-
if (!entry.isDirectory() && (entry.getName().startsWith("draft2020-12/") || entry.getName().startsWith("remotes/"))) {
58-
Path outputPath = TARGET_SUITE_DIR.resolve(entry.getName());
59-
Files.createDirectories(outputPath.getParent());
60-
Files.copy(zis, outputPath, java.nio.file.StandardCopyOption.REPLACE_EXISTING);
61-
}
62-
zis.closeEntry();
63-
}
64-
}
65-
66-
// Verify the target directory exists after extraction
67-
if (!Files.exists(TARGET_SUITE_DIR)) {
68-
throw new RuntimeException("Extraction completed but target directory not found: " + TARGET_SUITE_DIR.toAbsolutePath());
69-
}
70-
}
71-
72-
Stream<DynamicTest> testsFromFile(Path file) {
73-
try {
74-
final var root = MAPPER.readTree(file.toFile());
75-
76-
/// The JSON Schema Test Suite contains two types of files:
77-
/// 1. Test suite files: Arrays containing test groups with description, schema, and tests fields
78-
/// 2. Remote reference files: Plain JSON schema files used as remote references by test cases
79-
///
80-
/// We only process test suite files. Remote reference files (like remotes/baseUriChangeFolder/folderInteger.json)
81-
/// are just schema documents that get loaded via $ref during test execution, not test cases themselves.
82-
83-
/// Validate that this is a test suite file (array of objects with description, schema, tests)
84-
if (!root.isArray() || root.isEmpty()) {
85-
// Not a test suite file, skip it
86-
return Stream.empty();
87-
}
88-
89-
/// Validate first group has required fields
90-
final var firstGroup = root.get(0);
91-
if (!firstGroup.has("description") || !firstGroup.has("schema") || !firstGroup.has("tests")) {
92-
// Not a test suite file, skip it
93-
return Stream.empty();
94-
}
95-
96-
/// Count groups and tests discovered
97-
final var groupCount = root.size();
98-
METRICS.groupsDiscovered.add(groupCount);
99-
perFile(file).groups.add(groupCount);
100-
101-
var testCount = 0;
102-
for (final var group : root) {
103-
testCount += group.get("tests").size();
104-
}
105-
METRICS.testsDiscovered.add(testCount);
106-
perFile(file).tests.add(testCount);
107-
108-
return dynamicTestStream(file, root);
109-
} catch (Exception ex) {
110-
throw new RuntimeException("Failed to process " + file, ex);
111-
}
112-
}
113-
114-
static Stream<DynamicTest> dynamicTestStream(Path file, JsonNode root) {
115-
return StreamSupport.stream(root.spliterator(), false)
116-
.flatMap(group -> {
117-
final var groupDesc = group.get("description").asText();
118-
try {
119-
/// Attempt to compile the schema for this group; if unsupported features
120-
/// (e.g., unresolved anchors) are present, skip this group gracefully.
121-
final var schema = JsonSchema.compile(
122-
Json.parse(group.get("schema").toString()));
123-
124-
return StreamSupport.stream(group.get("tests").spliterator(), false)
125-
.map(test -> DynamicTest.dynamicTest(
126-
groupDesc + " – " + test.get("description").asText(),
127-
() -> {
128-
final var expected = test.get("valid").asBoolean();
129-
final boolean actual;
130-
try {
131-
actual = schema.validate(
132-
Json.parse(test.get("data").toString())).valid();
133-
134-
/// Count validation attempt
135-
METRICS.run.increment();
136-
perFile(file).run.increment();
137-
} catch (Exception e) {
138-
final var reason = e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage();
139-
System.err.println("[JsonSchemaCheck202012IT] Skipping test due to exception: "
140-
+ groupDesc + " — " + reason + " (" + file.getFileName() + ")");
141-
142-
/// Count exception as skipped mismatch in strict metrics
143-
METRICS.skippedMismatch.increment();
144-
perFile(file).skipMismatch.increment();
145-
146-
if (isStrict()) throw e;
147-
Assumptions.assumeTrue(false, "Skipped: " + reason);
148-
return; /// not reached when strict
149-
}
150-
151-
if (isStrict()) {
152-
try {
153-
assertEquals(expected, actual);
154-
/// Count pass in strict mode
155-
METRICS.passed.increment();
156-
perFile(file).pass.increment();
157-
} catch (AssertionError e) {
158-
/// Count failure in strict mode
159-
METRICS.failed.increment();
160-
perFile(file).fail.increment();
161-
throw e;
162-
}
163-
} else if (expected != actual) {
164-
System.err.println("[JsonSchemaCheck202012IT] Mismatch (ignored): "
165-
+ groupDesc + " — expected=" + expected + ", actual=" + actual
166-
+ " (" + file.getFileName() + ")");
15+
public class JsonSchemaCheck202012IT extends JsonSchemaCheckBaseIT {
16716

168-
/// Count lenient mismatch skip
169-
METRICS.skippedMismatch.increment();
170-
perFile(file).skipMismatch.increment();
17+
private static final Path ZIP_FILE = Paths.get("src/test/resources/json-schema-test-suite-data.zip");
18+
private static final Path TARGET_SUITE_DIR = Paths.get("target/test-data/draft2020-12");
17119

172-
Assumptions.assumeTrue(false, "Mismatch ignored");
173-
} else {
174-
/// Count pass in lenient mode
175-
METRICS.passed.increment();
176-
perFile(file).pass.increment();
177-
}
178-
}));
179-
} catch (Exception ex) {
180-
/// Unsupported schema for this group; emit a single skipped test for visibility
181-
final var reason = ex.getMessage() == null ? ex.getClass().getSimpleName() : ex.getMessage();
182-
System.err.println("[JsonSchemaCheck202012IT] Skipping group due to unsupported schema: "
183-
+ groupDesc + " — " + reason + " (" + file.getFileName() + ")");
184-
185-
/// Count unsupported group skip
186-
METRICS.skippedUnsupported.increment();
187-
perFile(file).skipUnsupported.increment();
188-
189-
return Stream.of(DynamicTest.dynamicTest(
190-
groupDesc + " – SKIPPED: " + reason,
191-
() -> { if (isStrict()) throw ex; Assumptions.assumeTrue(false, "Unsupported schema: " + reason); }
192-
));
193-
}
194-
});
20+
@Override
21+
protected Path getZipFile() {
22+
return ZIP_FILE;
19523
}
19624

197-
static StrictMetrics.FileCounters perFile(Path file) {
198-
return METRICS.perFile.computeIfAbsent(file.getFileName().toString(), k -> new StrictMetrics.FileCounters());
199-
}
200-
201-
/// Helper to check if we're running in strict mode
202-
static boolean isStrict() {
203-
return STRICT;
204-
}
205-
206-
@AfterAll
207-
static void printAndPersistMetrics() throws Exception {
208-
final var strict = isStrict();
209-
final var total = METRICS.testsDiscovered.sum();
210-
final var run = METRICS.run.sum();
211-
final var passed = METRICS.passed.sum();
212-
final var failed = METRICS.failed.sum();
213-
final var skippedUnsupported = METRICS.skippedUnsupported.sum();
214-
final var skippedMismatch = METRICS.skippedMismatch.sum();
215-
216-
/// Print canonical summary line
217-
System.out.printf(
218-
"JSON-SCHEMA-COMPAT: total=%d run=%d passed=%d failed=%d skipped-unsupported=%d skipped-mismatch=%d strict=%b%n",
219-
total, run, passed, failed, skippedUnsupported, skippedMismatch, strict
220-
);
221-
222-
/// For accounting purposes, we accept that the current implementation
223-
/// creates some accounting complexity when groups are skipped.
224-
/// The key metrics are still valid and useful for tracking progress.
225-
if (strict) {
226-
assertEquals(run, passed + failed, "strict run accounting mismatch");
227-
}
228-
229-
/// Legacy metrics for backward compatibility
230-
System.out.printf(
231-
"JSON-SCHEMA SUITE (%s): groups=%d testsScanned=%d run=%d passed=%d failed=%d skipped={unsupported=%d, exception=%d, lenientMismatch=%d}%n",
232-
strict ? "STRICT" : "LENIENT",
233-
METRICS.groupsDiscovered.sum(),
234-
METRICS.testsDiscovered.sum(),
235-
run, passed, failed, skippedUnsupported, METRICS.skipTestException.sum(), skippedMismatch
236-
);
237-
238-
if (!METRICS_FMT.isEmpty()) {
239-
var outDir = java.nio.file.Path.of("target");
240-
java.nio.file.Files.createDirectories(outDir);
241-
var ts = java.time.OffsetDateTime.now().toString();
242-
if ("json".equalsIgnoreCase(METRICS_FMT)) {
243-
var json = buildJsonSummary(strict, ts);
244-
java.nio.file.Files.writeString(outDir.resolve("json-schema-compat.json"), json);
245-
} else if ("csv".equalsIgnoreCase(METRICS_FMT)) {
246-
var csv = buildCsvSummary(strict, ts);
247-
java.nio.file.Files.writeString(outDir.resolve("json-schema-compat.csv"), csv);
248-
}
249-
}
250-
}
25+
@Override
26+
protected Path getTargetSuiteDir() {
27+
return TARGET_SUITE_DIR;
28+
}
25129

252-
static String buildJsonSummary(boolean strict, String timestamp) {
253-
var totals = new StringBuilder();
254-
totals.append("{\n");
255-
totals.append(" \"mode\": \"").append(strict ? "STRICT" : "LENIENT").append("\",\n");
256-
totals.append(" \"timestamp\": \"").append(timestamp).append("\",\n");
257-
totals.append(" \"totals\": {\n");
258-
totals.append(" \"groupsDiscovered\": ").append(METRICS.groupsDiscovered.sum()).append(",\n");
259-
totals.append(" \"testsDiscovered\": ").append(METRICS.testsDiscovered.sum()).append(",\n");
260-
totals.append(" \"validationsRun\": ").append(METRICS.run.sum()).append(",\n");
261-
totals.append(" \"passed\": ").append(METRICS.passed.sum()).append(",\n");
262-
totals.append(" \"failed\": ").append(METRICS.failed.sum()).append(",\n");
263-
totals.append(" \"skipped\": {\n");
264-
totals.append(" \"unsupportedSchemaGroup\": ").append(METRICS.skippedUnsupported.sum()).append(",\n");
265-
totals.append(" \"testException\": ").append(METRICS.skipTestException.sum()).append(",\n");
266-
totals.append(" \"lenientMismatch\": ").append(METRICS.skippedMismatch.sum()).append("\n");
267-
totals.append(" }\n");
268-
totals.append(" },\n");
269-
totals.append(" \"perFile\": [\n");
270-
271-
var files = new java.util.ArrayList<String>(METRICS.perFile.keySet());
272-
java.util.Collections.sort(files);
273-
var first = true;
274-
for (String file : files) {
275-
var counters = METRICS.perFile.get(file);
276-
if (!first) totals.append(",\n");
277-
first = false;
278-
totals.append(" {\n");
279-
totals.append(" \"file\": \"").append(file).append("\",\n");
280-
totals.append(" \"groups\": ").append(counters.groups.sum()).append(",\n");
281-
totals.append(" \"tests\": ").append(counters.tests.sum()).append(",\n");
282-
totals.append(" \"run\": ").append(counters.run.sum()).append(",\n");
283-
totals.append(" \"pass\": ").append(counters.pass.sum()).append(",\n");
284-
totals.append(" \"fail\": ").append(counters.fail.sum()).append(",\n");
285-
totals.append(" \"skipUnsupported\": ").append(counters.skipUnsupported.sum()).append(",\n");
286-
totals.append(" \"skipException\": ").append(counters.skipException.sum()).append(",\n");
287-
totals.append(" \"skipMismatch\": ").append(counters.skipMismatch.sum()).append("\n");
288-
totals.append(" }");
289-
}
290-
totals.append("\n ]\n");
291-
totals.append("}\n");
292-
return totals.toString();
293-
}
30+
@Override
31+
protected String getSchemaPrefix() {
32+
return "draft2020-12/";
33+
}
29434

295-
static String buildCsvSummary(boolean strict, String timestamp) {
296-
var csv = new StringBuilder();
297-
csv.append("mode,timestamp,groupsDiscovered,testsDiscovered,validationsRun,passed,failed,skippedUnsupported,skipTestException,skippedMismatch\n");
298-
csv.append(strict ? "STRICT" : "LENIENT").append(",");
299-
csv.append(timestamp).append(",");
300-
csv.append(METRICS.groupsDiscovered.sum()).append(",");
301-
csv.append(METRICS.testsDiscovered.sum()).append(",");
302-
csv.append(METRICS.run.sum()).append(",");
303-
csv.append(METRICS.passed.sum()).append(",");
304-
csv.append(METRICS.failed.sum()).append(",");
305-
csv.append(METRICS.skippedUnsupported.sum()).append(",");
306-
csv.append(METRICS.skipTestException.sum()).append(",");
307-
csv.append(METRICS.skippedMismatch.sum()).append("\n");
308-
309-
csv.append("\nperFile breakdown:\n");
310-
csv.append("file,groups,tests,run,pass,fail,skipUnsupported,skipException,skipMismatch\n");
311-
312-
var files = new java.util.ArrayList<String>(METRICS.perFile.keySet());
313-
java.util.Collections.sort(files);
314-
for (String file : files) {
315-
var counters = METRICS.perFile.get(file);
316-
csv.append(file).append(",");
317-
csv.append(counters.groups.sum()).append(",");
318-
csv.append(counters.tests.sum()).append(",");
319-
csv.append(counters.run.sum()).append(",");
320-
csv.append(counters.pass.sum()).append(",");
321-
csv.append(counters.fail.sum()).append(",");
322-
csv.append(counters.skipUnsupported.sum()).append(",");
323-
csv.append(counters.skipException.sum()).append(",");
324-
csv.append(counters.skipMismatch.sum()).append("\n");
325-
}
326-
return csv.toString();
327-
}
328-
}
35+
@Override
36+
protected Set<String> getSkippedTests() {
37+
return Set.of(
38+
// Reference resolution issues - Unresolved $ref problems
39+
"ref.json#relative pointer ref to array#match array",
40+
"ref.json#relative pointer ref to array#mismatch array",
41+
"refOfUnknownKeyword.json#reference of a root arbitrary keyword #match",
42+
"refOfUnknownKeyword.json#reference of a root arbitrary keyword #mismatch",
43+
"refOfUnknownKeyword.json#reference of an arbitrary keyword of a sub-schema#match",
44+
"refOfUnknownKeyword.json#reference of an arbitrary keyword of a sub-schema#mismatch",
45+
46+
// JSON parsing issues with duplicate member names
47+
"required.json#required with escaped characters#object with all properties present is valid",
48+
"required.json#required with escaped characters#object with some properties missing is invalid"
49+
);
50+
}
32951

52+
@TestFactory
53+
@Override
54+
public Stream<DynamicTest> runOfficialSuite() throws Exception {
55+
return super.runOfficialSuite();
56+
}
57+
}

0 commit comments

Comments
 (0)