Skip to content

Commit 69d692a

Browse files
committed
docs: update schema compatibility metrics (strict headline + overall); align README test commands with wrapper; include updated schema engine changes for remote refs, logging, and compile session
1 parent c093d27 commit 69d692a

File tree

4 files changed

+24
-28
lines changed

4 files changed

+24
-28
lines changed

README.md

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -104,27 +104,28 @@ var result = schema.validate(
104104
// result.valid() => true
105105
```
106106

107-
Compatibility: runs the official 2020‑12 JSON Schema Test Suite on `verify`; **measured compatibility is 64.6%** (1,177 of 1,822 tests pass) with comprehensive metrics reporting.
107+
Compatibility: runs the official 2020‑12 JSON Schema Test Suite on `verify`; **strict compatibility is 61.6%** (1024 of 1,663 validations). [Overall including all discovered tests: 56.2% (1024 of 1,822)].
108108

109109
### JSON Schema Test Suite Metrics
110110

111111
The validator now provides defensible compatibility statistics:
112112

113113
```bash
114114
# Run with console metrics (default)
115-
mvn verify -pl json-java21-schema
115+
./mvn-test-no-boilerplate.sh -pl json-java21-schema
116116

117-
# Export detailed JSON metrics
118-
mvn verify -pl json-java21-schema -Djson.schema.metrics=json
117+
# Export detailed JSON metrics
118+
./mvn-test-no-boilerplate.sh -pl json-java21-schema -Djson.schema.metrics=json
119119

120120
# Export CSV metrics for analysis
121-
mvn verify -pl json-java21-schema -Djson.schema.metrics=csv
121+
./mvn-test-no-boilerplate.sh -pl json-java21-schema -Djson.schema.metrics=csv
122122
```
123123

124124
**Current measured compatibility**:
125-
- **Overall**: 64.6% (1,177 of 1,822 tests pass)
126-
- **Test coverage**: 420 test groups, 1,657 validation attempts
127-
- **Skip breakdown**: 70 unsupported schema groups, 2 test exceptions, 480 lenient mismatches
125+
- **Strict (headline)**: 61.6% (1024 of 1,663 validations)
126+
- **Overall (incl. out‑of‑scope)**: 56.2% (1024 of 1,822 discovered tests)
127+
- **Test coverage**: 420 test groups, 1,663 validation attempts
128+
- **Skip breakdown**: 65 unsupported schema groups, 0 test exceptions, 647 lenient mismatches
128129

129130
## Building
130131

json-java21-schema/README.md

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,20 @@ Compatibility and verify
2323

2424
- The module runs the official JSON Schema Test Suite during Maven verify.
2525
- Default mode is lenient: unsupported groups/tests are skipped to avoid build breaks while still logging.
26-
- Strict mode: enable with -Djson.schema.strict=true to enforce full assertions.
27-
- **Measured compatibility**: 54.4% (992 of 1,822 tests pass in lenient mode)
28-
- **Test coverage**: 420 test groups, 1,628 validation attempts, 73 unsupported schema groups, 0 test exceptions, 638 lenient mismatches
26+
- Strict mode: enable with `-Djson.schema.strict=true` to enforce full assertions.
27+
- Measured compatibility (headline strictness): 61.6% (1024 of 1,663 validations)
28+
- Overall including all discovered tests: 56.2% (1024 of 1,822)
29+
- Test coverage: 420 test groups, 1,663 validation attempts, 65 unsupported schema groups, 0 test exceptions, 647 lenient mismatches
2930
- Detailed metrics available via `-Djson.schema.metrics=json|csv`
3031

3132
How to run
3233

3334
```bash
3435
# Run unit + integration tests (includes official suite)
35-
mvn -pl json-java21-schema -am verify
36+
./mvn-test-no-boilerplate.sh -pl json-java21-schema
3637

3738
# Strict mode
38-
mvn -Djson.schema.strict=true -pl json-java21-schema -am verify
39+
./mvn-test-no-boilerplate.sh -pl json-java21-schema -Djson.schema.strict=true
3940
```
4041

4142
OpenRPC validation

json-java21-schema/src/main/java/io/github/simbo1905/json/schema/JsonSchema.java

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ enum Nothing implements JsonSchema {
9595

9696
@Override
9797
public ValidationResult validateAt(String path, JsonValue json, Deque<ValidationFrame> stack) {
98-
LOG.severe(() -> "ERROR: Nothing enum validateAt called - this should never happen");
98+
LOG.severe(() -> "ERROR: SCHEMA: Nothing.validateAt invoked");
9999
throw new UnsupportedOperationException("Nothing enum should not be used for validation");
100100
}
101101
}
@@ -158,7 +158,7 @@ interface RemoteFetcher {
158158

159159
static RemoteFetcher disallowed() {
160160
return (uri, policy) -> {
161-
LOG.severe(() -> "ERROR: Remote fetching disabled but requested for URI: " + uri);
161+
LOG.severe(() -> "ERROR: FETCH: " + uri + " - policy POLICY_DENIED");
162162
throw new RemoteResolutionException(
163163
Objects.requireNonNull(uri, "uri"),
164164
RemoteResolutionException.Reason.POLICY_DENIED,
@@ -316,7 +316,7 @@ static JsonSchema compile(JsonValue schemaJson, Options options, CompileOptions
316316
Objects.requireNonNull(schemaJson, "schemaJson");
317317
Objects.requireNonNull(options, "options");
318318
Objects.requireNonNull(compileOptions, "compileOptions");
319-
LOG.info(() -> "json-schema.compile start doc=" + java.net.URI.create("urn:inmemory:root") + " options=" + options.summary());
319+
LOG.fine(() -> "json-schema.compile start doc=" + java.net.URI.create("urn:inmemory:root") + " options=" + options.summary());
320320
LOG.fine(() -> "compile: Starting schema compilation with full options, schema type: " + schemaJson.getClass().getSimpleName() +
321321
", options.assertFormats=" + options.assertFormats() + ", compileOptions.remoteFetcher=" + compileOptions.remoteFetcher().getClass().getSimpleName());
322322
LOG.fine(() -> "compile: fetch policy allowedSchemes=" + compileOptions.fetchPolicy().allowedSchemes());
@@ -373,7 +373,7 @@ static JsonSchema compile(JsonValue schemaJson, Options options, CompileOptions
373373
}
374374
}
375375

376-
LOG.info(() -> "json-schema.compile done roots=" + rootCount);
376+
LOG.fine(() -> "json-schema.compile done roots=" + rootCount);
377377
return result;
378378
}
379379

@@ -700,7 +700,7 @@ static CompiledRegistry freezeRoots(Map<java.net.URI, CompiledRoot> built, java.
700700
}
701701
if (entryRoot == null) {
702702
// As a last resort, pick the first element to avoid NPE, but log an error
703-
LOG.severe(() -> "ERROR: Primary root URI not found in compiled roots: " + primaryUri);
703+
LOG.severe(() -> "ERROR: SCHEMA: primary root not found doc=" + primaryUri);
704704
entryRoot = built.values().iterator().next();
705705
}
706706
final java.net.URI primaryResolved = entryRoot.docUri();
@@ -728,7 +728,7 @@ static CompiledRegistry freezeRoots(Map<java.net.URI, CompiledRoot> built, java.
728728
/// @return ValidationResult with success/failure information
729729
default ValidationResult validate(JsonValue json) {
730730
Objects.requireNonNull(json, "json");
731-
LOG.info(() -> "json-schema.validate start frames=0 doc=unknown");
731+
LOG.fine(() -> "json-schema.validate start frames=0 doc=unknown");
732732
List<ValidationError> errors = new ArrayList<>();
733733
Deque<ValidationFrame> stack = new ArrayDeque<>();
734734
Set<ValidationKey> visited = new HashSet<>();
@@ -1805,7 +1805,7 @@ static CompilationBundle compileBundle(JsonValue schemaJson, Options options, Co
18051805
// Create compilation bundle
18061806
CompiledRoot entryRoot = compiled.get(entryUri);
18071807
if (entryRoot == null) {
1808-
LOG.severe(() -> "ERROR: Entry root must exist but was null for URI: " + entryUri);
1808+
LOG.severe(() -> "ERROR: SCHEMA: entry root null doc=" + entryUri);
18091809
}
18101810
assert entryRoot != null : "Entry root must exist";
18111811
List<CompiledRoot> allRoots = List.copyOf(compiled.values());

json-java21-schema/src/main/java/io/github/simbo1905/json/schema/VirtualThreadHttpFetcher.java

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -174,16 +174,10 @@ JsonValue fetchSchemaJson(java.net.URI docUri) {
174174

175175
return result.document();
176176
} catch (JsonSchema.RemoteResolutionException e) {
177-
LOG.finest(() -> "fetchSchemaJson: caught RemoteResolutionException object=" + e + ", uri=" + e.uri() + ", reason=" + e.reason() + ", message='" + e.getMessage() + "'");
178-
if (e.reason() == JsonSchema.RemoteResolutionException.Reason.NOT_FOUND) {
179-
LOG.warning(() -> "fetchSchemaJson: non-200 response for uri=" + docUri);
180-
} else if (e.reason() == JsonSchema.RemoteResolutionException.Reason.NETWORK_ERROR) {
181-
LOG.severe(() -> "ERROR: fetchSchemaJson network error for uri=" + docUri + ": " + e.getMessage());
182-
}
177+
// Already logged by the fetch path; rethrow
183178
throw e;
184179
} catch (Exception e) {
185-
LOG.finest(() -> "fetchSchemaJson: caught unexpected exception object=" + e + ", class=" + e.getClass().getSimpleName() + ", message='" + e.getMessage() + "'");
186-
LOG.severe(() -> "ERROR: fetchSchemaJson unexpected error for uri=" + docUri + ": " + e.getMessage());
180+
LOG.severe(() -> "ERROR: FETCH: " + docUri + " - unexpected NETWORK_ERROR");
187181
throw new JsonSchema.RemoteResolutionException(docUri, JsonSchema.RemoteResolutionException.Reason.NETWORK_ERROR, "Failed to fetch schema", e);
188182
}
189183
}

0 commit comments

Comments
 (0)