diff --git a/.vscode/launch.json b/.vscode/launch.json index 125ce23ed84..bf4b90bc521 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -26,8 +26,16 @@ { "type": "java", "request": "attach", - "name": "Attach by Process ID", - "processId": "${command:PickJavaProcess}" + "name": "Attach cldr-apps by Process ID", + "processId": "${command:PickJavaProcess}", + "projectName": "cldr-apps" + }, + { + "type": "java", + "request": "attach", + "name": "Attach cldr-code by Process ID", + "processId": "${command:PickJavaProcess}", + "projectName": "cldr-code" } ] -} \ No newline at end of file +} diff --git a/tools/cldr-apps/js/src/esm/cldrProgress.js b/tools/cldr-apps/js/src/esm/cldrProgress.js index 099101fdf8b..9eed591eb22 100644 --- a/tools/cldr-apps/js/src/esm/cldrProgress.js +++ b/tools/cldr-apps/js/src/esm/cldrProgress.js @@ -19,7 +19,7 @@ import { notification } from "ant-design-vue"; const USE_NEW_PROGRESS_WIDGET = true; const CAN_GET_VOTER_PROGRESS = true; -const CAN_GET_LOCALE_PROGRESS = false; // For now +const CAN_GET_LOCALE_PROGRESS = false; let progressWrapper = null; diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/util/SandboxLocales.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/util/SandboxLocales.java index 28f679297f7..f1a726845e9 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/util/SandboxLocales.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/util/SandboxLocales.java @@ -89,8 +89,8 @@ public File getAnnotationsDir() { * @author srl * */ - private static class ScratchXMLSource extends XMLSource { - ScratchXMLSource(String id) { + public static class ScratchXMLSource extends XMLSource { + public ScratchXMLSource(String id) { super.setLocaleID(id); super.setInitialComment("GENERATED FILE\n" + "Note- This is a sandbox (scratch) locale.\n"+ @@ -98,6 +98,7 @@ private static class ScratchXMLSource extends XMLSource { + " check it in to the CLDR source repository.\n\nThis notice generated by SandboxLocales.java"); } Map valueMap = CldrUtility.newConcurrentHashMap(); + Map fullPathMap = CldrUtility.newConcurrentHashMap(); private Comments comments = new Comments(); @Override @@ -108,7 +109,11 @@ public XMLSource freeze() { @Override public void putFullPathAtDPath(String distinguishingXPath, String fullxpath) { - throw new RuntimeException("not implemented"); + if (distinguishingXPath.equals(fullxpath)) { + fullPathMap.remove(distinguishingXPath); + } else { + fullPathMap.put(distinguishingXPath, fullxpath); + } } @Override @@ -118,7 +123,7 @@ public void putValueAtDPath(String distinguishingXPath, String value) { @Override public void removeValueAtDPath(String distinguishingXPath) { - throw new RuntimeException("not implemented"); + valueMap.remove(distinguishingXPath); } @Override @@ -128,7 +133,12 @@ public String getValueAtDPath(String path) { @Override public String getFullPathAtDPath(String path) { - throw new RuntimeException("not implemented"); + final String fullxpath = fullPathMap.get(path); + if (fullxpath == null) { + return path; + } else { + return fullxpath; + } } @Override @@ -149,7 +159,14 @@ public Iterator iterator() { @Override public void getPathsWithValue(String valueToMatch, String pathPrefix, Set result) { - throw new RuntimeException("not implemented"); + for(final String xpath : this) { + if (!xpath.startsWith(pathPrefix)) { + continue; + } + if (valueToMatch.equals(getValueAtDPath(xpath))) { + result.add(xpath); + } + } } } @@ -162,4 +179,4 @@ public Factory getFactory(File commonMain) { File dirs[] = { getMainDir(), getAnnotationsDir(), commonMain }; return SimpleFactory.make(dirs, ".*"); } -} \ No newline at end of file +} diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java index baa03ce9cc8..8f6cf9dc5dc 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/STFactory.java @@ -637,39 +637,6 @@ public final Stamp getStamp() { return stamp; } - /** - * Get the Status for the given CLDRFile, path, and value. - * - * @param cldrFile - * @param path - * @param value - * @return the Status - */ - private Status getStatus(CLDRFile cldrFile, String path, final String value) { - String fullXPath = cldrFile.getFullXPath(path); - if (fullXPath == null) { - fullXPath = path; - } - XPathParts xpp = XPathParts.getFrozenInstance(fullXPath); - String draft = xpp.getAttributeValue(-1, LDMLConstants.DRAFT); - Status status = draft == null ? Status.approved : VoteResolver.Status.fromString(draft); - - /* - * Reset to missing if the value is inherited from root or code-fallback, unless the XML actually - * contains INHERITANCE_MARKER. Pass false for skipInheritanceMarker so that status will not be - * missing for explicit INHERITANCE_MARKER. Reference: https://unicode.org/cldr/trac/ticket/11857 - */ - final String srcid = cldrFile.getSourceLocaleIdExtended(path, null, false /* skipInheritanceMarker */); - if (srcid.equals(XMLSource.CODE_FALLBACK_ID)) { - status = Status.missing; - } else if (srcid.equals("root")) { - if (!srcid.equals(diskFile.getLocaleID())) { - status = Status.missing; - } - } - return status; - } - /** * * @param user @@ -960,7 +927,7 @@ private VoteResolver getResolverInternal(PerXPathData perXPathData, Stri // set current Trunk (baseline) value (if present) final String currentValue = diskData.getValueAtDPath(path); - final Status currentStatus = getStatus(diskFile, path, currentValue); + final Status currentStatus = calculateStatus(diskFile, diskFile, path); if (ERRORS_ALLOWED_IN_VETTING || vc.canUseValue(currentValue)) { r.setBaseline(currentValue, currentStatus); r.add(currentValue); @@ -2344,4 +2311,37 @@ public static final String getLastVoteTable() { final String dbName = DBUtils.Table.VOTE_VALUE.forVersion(SurveyMain.getLastVoteVersion(), false).toString(); return dbName; } + + /** + * Utility function to calculate CLDRFile.Status + * @param cldrFile the CLDR File to use + * @param diskFile the 'baseline' file to use + * @param path xpath + * @param value current string value + * @return + */ + public static final Status calculateStatus(CLDRFile cldrFile, CLDRFile diskFile, String path) { + String fullXPath = cldrFile.getFullXPath(path); + if (fullXPath == null) { + fullXPath = path; + } + XPathParts xpp = XPathParts.getFrozenInstance(fullXPath); + String draft = xpp.getAttributeValue(-1, LDMLConstants.DRAFT); + Status status = draft == null ? Status.approved : VoteResolver.Status.fromString(draft); + + /* + * Reset to missing if the value is inherited from root or code-fallback, unless the XML actually + * contains INHERITANCE_MARKER. Pass false for skipInheritanceMarker so that status will not be + * missing for explicit INHERITANCE_MARKER. Reference: https://unicode.org/cldr/trac/ticket/11857 + */ + final String srcid = cldrFile.getSourceLocaleIdExtended(path, null, false /* skipInheritanceMarker */); + if (srcid.equals(XMLSource.CODE_FALLBACK_ID)) { + status = Status.missing; + } else if (srcid.equals("root")) { + if (!srcid.equals(diskFile.getLocaleID())) { + status = Status.missing; + } + } + return status; + } } diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/UserRegistry.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/UserRegistry.java index 16b5180fbb9..1684b27b3fd 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/UserRegistry.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/UserRegistry.java @@ -985,7 +985,10 @@ public UserRegistry.User getEmptyUser() { static SurveyMain sm = null; // static for static checking of defaultContent - private UserRegistry() { + /** + * Public for tests + */ + public UserRegistry() { } // ------- special things for "list" mode: diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/api/LocaleCompletion.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/api/LocaleCompletion.java index 2cea2220a9a..7561ea8d3bf 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/api/LocaleCompletion.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/api/LocaleCompletion.java @@ -1,18 +1,18 @@ package org.unicode.cldr.web.api; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.logging.Logger; import javax.ws.rs.GET; -import javax.ws.rs.HeaderParam; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.Status; + +import com.ibm.icu.dev.util.ElapsedTimer; import org.eclipse.microprofile.openapi.annotations.Operation; import org.eclipse.microprofile.openapi.annotations.media.Content; @@ -20,16 +20,33 @@ import org.eclipse.microprofile.openapi.annotations.responses.APIResponse; import org.eclipse.microprofile.openapi.annotations.responses.APIResponses; import org.eclipse.microprofile.openapi.annotations.tags.Tag; +import org.unicode.cldr.test.CheckCLDR; +import org.unicode.cldr.test.CoverageLevel2; +import org.unicode.cldr.test.SubmissionLocales; +import org.unicode.cldr.test.CheckCLDR.CheckStatus; +import org.unicode.cldr.test.CheckCLDR.InputMethod; +import org.unicode.cldr.test.CheckCLDR.StatusAction; +import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; +import org.unicode.cldr.test.TestCache.TestResultBundle; +import org.unicode.cldr.util.CLDRConfig; import org.unicode.cldr.util.CLDRFile; import org.unicode.cldr.util.CLDRLocale; import org.unicode.cldr.util.Level; import org.unicode.cldr.util.PathHeader; -import org.unicode.cldr.util.PathHeader.Factory; import org.unicode.cldr.util.StandardCodes; import org.unicode.cldr.util.SupplementalDataInfo; +import org.unicode.cldr.util.VoteResolver; +import org.unicode.cldr.util.CLDRFile.DraftStatus; +import org.unicode.cldr.util.CLDRFile.Status; +import org.unicode.cldr.util.CLDRInfo.CandidateInfo; +import org.unicode.cldr.util.CLDRInfo.PathValueInfo; +import org.unicode.cldr.util.PathHeader.SurveyToolStatus; +import org.unicode.cldr.web.BallotBox; import org.unicode.cldr.web.CookieSession; -import org.unicode.cldr.web.DBUtils; +import org.unicode.cldr.web.STFactory; +import org.unicode.cldr.web.SurveyLog; import org.unicode.cldr.web.SurveyMain; +import org.unicode.cldr.web.UserRegistry; /** * "A locale has complete coverage when there are no Missing values, no Provisional values, and no Errors @@ -47,9 +64,7 @@ @Path("/completion") @Tag(name = "completion", description = "APIs for voting completion statistics") public class LocaleCompletion { - private final SurveyMain sm = CookieSession.sm; - private final SupplementalDataInfo supplementalDataInfo = sm.getSupplementalDataInfo(); - private final Factory pathHeaderFactory = PathHeader.getFactory(); + private static final Logger logger = SurveyLog.forClass(LocaleCompletion.class); @GET @Path("/locale/{locale}") @@ -64,12 +79,12 @@ public class LocaleCompletion { description = "Voting completion statistics for the requesting user and the given locale and coverage level", content = @Content(mediaType = "application/json", schema = @Schema(implementation = LocaleCompletionResponse.class))), - // @APIResponse( - // responseCode = "401", - // description = "Authorization required, send a valid session id"), @APIResponse( responseCode = "404", description = "Locale not found"), + @APIResponse( + responseCode = "503", + description = "Not ready yet"), @APIResponse( responseCode = "500", description = "Internal Server Error", @@ -77,59 +92,141 @@ public class LocaleCompletion { schema = @Schema(implementation = STError.class))), }) public Response getLocaleCompletion( - @PathParam("locale") @Schema(required = true, description = "Locale ID", example = "aa") String localeId - // Should not require session header. - // @HeaderParam(Auth.SESSION_HEADER) String session - ) { - // final CookieSession mySession = Auth.getSession(session); - // if (mySession == null || mySession.user == null) { - // return Response.status(Status.UNAUTHORIZED).build(); - // } - // Level coverageLevel = Level.fromString(level); + @PathParam("locale") @Schema(required = true, description = "Locale ID", example = "aa") String localeId) { CLDRLocale cldrLocale = CLDRLocale.getInstance(localeId); - // if (coverageLevel == Level.UNDETERMINED || cldrLocale == null) { - // return Response.status(Status.NOT_FOUND).build(); - // } - return get(/* mySession.user.id,*/ cldrLocale /*, coverageLevel*/); + return getLocaleCompletion(cldrLocale); } - private Response get(/*int userId, */CLDRLocale cldrLocale/*, Level coverageLevel*/) { - final boolean MEASURE_PERFORMANCE = false; - final long firstTime, secondTime; - if (MEASURE_PERFORMANCE) { - firstTime = System.currentTimeMillis(); + /** + * Fetch locale completion for the specified locale. + * Note: Static and package-private for testability. + * @param cldrLocale + * @return Response + */ + static Response getLocaleCompletion(CLDRLocale cldrLocale) { + if (SurveyMain.isBusted() || !SurveyMain.wasInitCalled() || !SurveyMain.triedToStartUp()) { + return STError.surveyNotQuiteReady(); + } + + final STFactory stFactory = CookieSession.sm.getSTFactory(); + final LocaleCompletionResponse lcr = getLocaleCompletion(cldrLocale, stFactory); + return Response.ok(lcr).build(); + } + + static final class LocaleCompletionHelper { + PathHeader.Factory phf = null; + + LocaleCompletionHelper() { + phf = PathHeader.getFactory(CLDRConfig.getInstance().getEnglish()); } + + static LocaleCompletionHelper INSTANCE = new LocaleCompletionHelper(); + } + + static LocaleCompletionResponse getLocaleCompletion(final CLDRLocale cldrLocale, final STFactory stFactory) { final String localeId = cldrLocale.toString(); // normalized final Level level = StandardCodes.make().getTargetCoverageLevel(localeId); - int voted = 5; - int total = 9; - // try { - // voted = getVotedPathCount(userId, localeId, coverageLevel); - // } catch (SQLException se) { - // return new STError(se).build(); - // } - // if (MEASURE_PERFORMANCE) { - // System.out.println("voting completion: voted = " + voted); - // secondTime = System.currentTimeMillis(); - // System.out.println("voting completion: time elapsed for voted (ms) = " + (secondTime - firstTime)); - // } - // int total = getTotalPathCount(localeId, coverageLevel); - // if (MEASURE_PERFORMANCE) { - // System.out.println("voting completion: total = " + total + " at coverage " + coverageLevel); - // System.out.println("voting completion: time elapsed for total (ms) = " + (System.currentTimeMillis() - secondTime)); - // } - return Response.ok(new LocaleCompletionResponse(voted, total, level.name())).build(); + final CheckCLDR.Options options = new CheckCLDR.Options(cldrLocale, SurveyMain.getTestPhase(), + level.toString(), + null); + ElapsedTimer et = new ElapsedTimer("LocaleCompletion:" + options.toString()); + logger.info("Starting LocaleCompletion for " + options.toString()); + final TestResultBundle checkCldr = stFactory.getTestResult(cldrLocale, options); + final CLDRFile file = stFactory.make(localeId, true); + final CLDRFile baselineFile = stFactory.getDiskFile(cldrLocale); + + LocaleCompletionResponse lcr = new LocaleCompletionResponse(level); + + final SupplementalDataInfo sdi = SupplementalDataInfo.getInstance(stFactory.getSupplementalDirectory()); + final CoverageLevel2 covLeveller = CoverageLevel2.getInstance(sdi, localeId); + + final List results = new ArrayList<>(); + for (final String xpath : file) { + final Level pathLevel = covLeveller.getLevel(xpath); + final String fullPath = file.getFullXPath(xpath); + + final PathHeader ph = LocaleCompletionHelper.INSTANCE.phf.fromPath(xpath); + SurveyToolStatus surveyToolStatus = ph.getSurveyToolStatus(); + if (surveyToolStatus == SurveyToolStatus.DEPRECATED || surveyToolStatus == SurveyToolStatus.HIDE) { + continue; // not visible + } + + checkCldr.check(fullPath, results, file.getStringValue(xpath)); + + boolean hasError = CheckStatus.hasError(results); + + // TODO: copy and paste from CheckCLDR.getShowRowAction - CLDR-15230 + if (CheckCLDR.LIMITED_SUBMISSION && !SubmissionLocales.allowEvenIfLimited( + localeId, + xpath, + hasError, + STFactory.calculateStatus(file, baselineFile, xpath) == VoteResolver.Status.missing)) { + continue; // not allowed through by SubmissionLocales. + } + + if (hasError) { + if (results.size() == 1 && results.get(0).getSubtype() == Subtype.coverageLevel) { + lcr.addMissing(); + } else { + lcr.addError(); + } + } else { + if (pathLevel.getLevel() < level.getLevel()) { + final CLDRFile.DraftStatus status = CLDRFile.DraftStatus.forXpath(fullPath); + if (status == DraftStatus.provisional || status == DraftStatus.unconfirmed) { + lcr.addProvisional(); + } else { + lcr.addOk(); + } + } // else: out of coverage level, do not count the path. + } + } + logger.info(et.toString()); + return lcr; } - public class LocaleCompletionResponse { - public int votes; - public int total; - public String level; + public static class LocaleCompletionResponse { + public int votes = 0; + public int total = 0; + public int error = 0; + public int missing = 0; + public int provisional = 0; + final public String level; + + LocaleCompletionResponse(Level l) { + level = l.name(); + } + + /** + * Add a count for an Error path + */ + void addError() { + error++; + total++; + } + + /** + * Add a count for a Missing path + */ + void addMissing() { + missing++; + total++; + } + + /** + * Add a count for a Provisional path + */ + void addProvisional() { + provisional++; + total++; + } - public LocaleCompletionResponse(int votes, int total, String level) { - this.votes = votes; - this.total = total; - this.level = level; + /** + * Add a count for an OK path + */ + void addOk() { + votes++; // OK count + total++; } } } diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/api/STError.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/api/STError.java index c0a214247dc..b971527d810 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/api/STError.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/api/STError.java @@ -66,4 +66,10 @@ public void setMessage(String message) { public Response build() { return Response.serverError().entity(this).build(); } + + public static Response surveyNotQuiteReady() { + return Response.status(503, "Try again later") + .entity("{\"error\": \"SurveyTool is not ready to handle API requests, try again later\"}") + .build(); + } } diff --git a/tools/cldr-apps/src/test/java/org/unicode/cldr/web/api/TestLocaleCompletion.java b/tools/cldr-apps/src/test/java/org/unicode/cldr/web/api/TestLocaleCompletion.java new file mode 100644 index 00000000000..b937464b8bb --- /dev/null +++ b/tools/cldr-apps/src/test/java/org/unicode/cldr/web/api/TestLocaleCompletion.java @@ -0,0 +1,319 @@ +package org.unicode.cldr.web.api; + +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.sql.SQLException; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.junit.jupiter.api.Test; +import org.opentest4j.MultipleFailuresError; +import org.unicode.cldr.test.CheckCLDR; +import org.unicode.cldr.test.TestCache; +import org.unicode.cldr.test.TestCache.TestResultBundle; +import org.unicode.cldr.util.CLDRConfig; +import org.unicode.cldr.util.CLDRFile; +import org.unicode.cldr.util.CLDRFile.DraftStatus; +import org.unicode.cldr.util.CLDRLocale; +import org.unicode.cldr.util.CLDRPaths; +import org.unicode.cldr.util.Factory; +import org.unicode.cldr.util.Level; +import org.unicode.cldr.util.LocaleSet; +import org.unicode.cldr.util.Organization; +import org.unicode.cldr.util.SandboxLocales; +import org.unicode.cldr.util.VoteResolver; +import org.unicode.cldr.util.XMLSource; +import org.unicode.cldr.util.VoteResolver.VoterInfo; +import org.unicode.cldr.web.STFactory; +import org.unicode.cldr.web.SurveyMain; +import org.unicode.cldr.web.UserRegistry; +import org.unicode.cldr.web.XPathTable; +import org.unicode.cldr.web.api.LocaleCompletion.LocaleCompletionResponse; + +public class TestLocaleCompletion { + /** + * Locale to use. We will start with a blank slate. + */ + private static final String TEST_LOCALE = "es"; + /** + * Sanity check amount: Assume at least these many paths are present. + */ + private int lOCALE_SIZE = 5; + + @Test + void testLocaleCompletion() throws SQLException, IOException { + SurveyMain sm = getSurveyMain(); + final TestCache tc = new TestCache(); + final CLDRConfig config = CLDRConfig.getInstance(); + final CLDRFile english = config.getEnglish(); + final CLDRFile spanish = config.getCLDRFile("es", false); + CheckCLDR.setDisplayInformation(english); + + final CLDRLocale locale = CLDRLocale.getInstance(TEST_LOCALE); + // final CLDRFile mulFile = sandFactory.make(mul.getBaseName(), false); + final SandboxLocales.ScratchXMLSource localeSource = new SandboxLocales.ScratchXMLSource(locale.getBaseName()); + final CLDRFile localeFile = new CLDRFile(localeSource); + final SandboxLocales.ScratchXMLSource rootSource = new SandboxLocales.ScratchXMLSource("root"); + final CLDRFile rootFile = new CLDRFile(rootSource); + final Factory sandboxFactory = getFactory(english, localeSource, localeFile, rootSource, rootFile); + tc.setFactory(sandboxFactory, "(?!.*(CheckCoverage).*).*"); + + STFactory stf = getSTFactory(sm, tc, sandboxFactory); + + preloadLocaleFile(spanish, localeFile); + + testCompletePaths(tc, locale, localeFile, stf); + + testIncompletePaths(tc, locale, localeFile, stf); + } + + /** + * Test with less than 100% completion + * @param tc + * @param locale + * @param localeFile + * @param stf + * @throws MultipleFailuresError + */ + private void testIncompletePaths(final TestCache tc, final CLDRLocale locale, final CLDRFile localeFile, STFactory stf) throws MultipleFailuresError { + // ** Step two. + setIncompletePaths(localeFile); + tc.invalidateAllCached(); + final int EXPECT_ERROR = 2; + final int EXPECT_MISSING = 1; + final int EXPECT_PROVISIONAL = 0; // 2 when not in limited + + assertTrue(CheckCLDR.LIMITED_SUBMISSION, "Error Error. This test becomes invalid if LIMITED_SUBMISSION (or SubmissionLocales) changes."); + + { + LocaleCompletionResponse completion = LocaleCompletion.getLocaleCompletion(locale, stf); + + assertNotNull(completion); + assertAll("tests on completion - not quite 100%", + () -> assertEquals(Level.MODERN.name(), completion.level, "completion level"), + () -> assertTrue(completion.votes > lOCALE_SIZE, "completion votes should be bigger but was " + completion.votes), + () -> assertTrue(completion.votes < completion.total, + "completion votes should be less than completion total but was " + completion.votes + "/" + completion.total), + () -> assertEquals(completion.total - (EXPECT_ERROR + EXPECT_MISSING + EXPECT_PROVISIONAL), completion.votes, + "completion votes should be less than completion total but was " + completion.votes + "/" + completion.total), + () -> assertTrue(completion.total > lOCALE_SIZE, "completion total should be bigger but was " + completion.total), + () -> assertEquals(EXPECT_ERROR, completion.error, "error count was " + completion.error), + () -> assertEquals(EXPECT_MISSING, completion.missing, "missing count was " + completion.missing), + () -> assertEquals(EXPECT_PROVISIONAL, completion.provisional, "provisional count was " + completion.provisional)); + } + } + + /** + * Add some paths that would lead to less than 100% completion + * @param localeFile + */ + private void setIncompletePaths(final CLDRFile localeFile) { + // Make some changes, and we should have a couple of errors. + // "Expected <100%" + // Mutate + localeFile.add("//ldml/characters/exemplarCharacters", "[]"); + localeFile.add("//ldml/numbers/symbols[@numberSystem=\"latn\"]/group", + ",,,"); // err; expected 1, not 3 + // rootFile.add("//ldml/localeDisplayNames/languages/language[@type=\""+TEST_LOCALE+"\"]", + // TEST_LOCALE); // for fallback + localeFile.remove("//ldml/localeDisplayNames/languages/language[@type=\"" + TEST_LOCALE + "\"]"); + localeFile.add("//ldml/dates/timeZoneNames/zone[@type=\"Pacific/Kanton\"]/exemplarCity[@draft=\"provisional\"]", "cabaca"); + localeFile.add("//ldml/dates/timeZoneNames/zone[@type=\"Asia/Qyzylorda\"]/exemplarCity[@draft=\"unconfirmed\"]", "bacaba"); + } + + /** + * Test with 100% completion + * @param tc + * @param locale + * @param localeFile + * @param stf + * @throws MultipleFailuresError + */ + private void testCompletePaths(final TestCache tc, final CLDRLocale locale, final CLDRFile localeFile, STFactory stf) throws MultipleFailuresError { + // ** Step one. Setup the locale + // Expected 100% + setCompletePaths(localeFile); + + tc.invalidateAllCached(); + + { + LocaleCompletionResponse completion = LocaleCompletion.getLocaleCompletion(locale, stf); + + assertNotNull(completion); + assertAll("tests on completio - 100%", + () -> assertEquals(Level.MODERN.name(), completion.level, "completion level"), + () -> assertTrue(completion.votes > lOCALE_SIZE, "completion votes should be bigger but was " + completion.votes), + () -> assertEquals(completion.total, completion.votes, + "completion votes should be equal completion total but was " + completion.votes + "/" + completion.total), + () -> assertTrue(completion.total > lOCALE_SIZE, "completion total should be bigger but was " + completion.total), + () -> assertEquals(0, completion.error), + () -> assertEquals(0, completion.missing), + () -> assertEquals(0, completion.provisional)); + } + } + + /** + * Add some paths that would give 100% locale completion + * @param localeFile + */ + private void setCompletePaths(final CLDRFile localeFile) { + localeFile.add("//ldml/characters/exemplarCharacters", "[abc]"); + localeFile.add("//ldml/numbers/symbols[@numberSystem=\"latn\"]/group", + ","); // ok + localeFile.add("//ldml/localeDisplayNames/languages/language[@type=\"" + TEST_LOCALE + "\"]", + "baccacab"); // for fallback (need to use only 'a,b,c') + + localeFile.add("//ldml/numbers/currencies/currency[@type=\"XTS\"]/displayName[@draft=\"provisional\"]", + "Test Currency (comprehensive level, should not have an effect on completion)"); + localeFile.add("//ldml/numbers/currencies/currency[@type=\"XUA\"]/displayName[@draft=\"unconfirmed\"]", + "Other Test Currency (comprehensive level, should not have an effect on completion)"); + } + + /** + * Load some initial paths (with exclusions) from a disk file + * @param spanish + * @param localeFile + */ + private void preloadLocaleFile(final CLDRFile spanish, final CLDRFile localeFile) { + // Preload some stuff + for (final String xpath : spanish) { + // Skip some problematic paths. + if (xpath.startsWith("//ldml/characters/parseLenients") || + xpath.startsWith("//ldml/numbers/currency") || + xpath.startsWith("//ldml/numbers/scientific") || + xpath.startsWith("//ldml/numbers/decimal") || + xpath.startsWith("//ldml/numbers/percent")) { + continue; + } + localeFile.add(xpath, spanish.getStringValue(xpath)); + } + } + + + /** + * New up a STFactory + * @param sm + * @param tc + * @param sandboxFactory + * @return + */ + private STFactory getSTFactory(SurveyMain sm, final TestCache tc, final Factory sandboxFactory) { + STFactory stf = new STFactory(sm) { + @Override + public TestResultBundle getTestResult(CLDRLocale loc, CheckCLDR.Options options) { + return tc.getBundle(options); + } + + @Override + public CLDRFile make(String locale, boolean resolved) { + return sandboxFactory.make(locale, resolved); // pass through + } + }; + return stf; + } + + /** + * New up a Factory + * @param english + * @param localeSource + * @param localeFile + * @param rootSource + * @param rootFile + * @return + */ + private Factory getFactory(final CLDRFile english, final SandboxLocales.ScratchXMLSource localeSource, final CLDRFile localeFile, + final SandboxLocales.ScratchXMLSource rootSource, final CLDRFile rootFile) { + final Factory sandFactory = new Factory() { + + @Override + public File getSupplementalDirectory() { + return english.getSupplementalDirectory(); + } + + @Override + public File[] getSourceDirectories() { + File[] f = { new File(CLDRPaths.MAIN_DIRECTORY) }; + return f; + } + + @Override + protected CLDRFile handleMake(String localeID, boolean resolved, DraftStatus madeWithMinimalDraftStatus) { + if (resolved) { + if (localeID.equals(TEST_LOCALE)) { + List l = new LinkedList<>(); + l.add(localeSource); + l.add(rootSource); + XMLSource.ResolvingSource rs = new XMLSource.ResolvingSource(l); + return new CLDRFile(rs); + } else if (localeID.equals("root")) { + List l = new LinkedList<>(); + l.add(rootSource); + XMLSource.ResolvingSource rs = new XMLSource.ResolvingSource(l); + return new CLDRFile(rs); + } + throw new RuntimeException("dont know how to make " + localeID + " r=" + resolved); + } else if (localeID.equals(TEST_LOCALE)) { + return localeFile; + } else if (localeID.equals("root")) { + return rootFile; + } else { + throw new RuntimeException("dont know how to make " + localeID + " r=" + resolved); + } + } + + @Override + public DraftStatus getMinimalDraftStatus() { + return DraftStatus.unconfirmed; + } + + @Override + protected Set handleGetAvailable() { + return Collections.singleton(TEST_LOCALE); + } + + @Override + public List getSourceDirectoriesForLocale(String localeName) { + return Collections.singletonList(new File(CLDRPaths.MAIN_DIRECTORY)); + } + }; + return sandFactory; + } + + /** + * New up a SurveyMain + * @return + */ + private SurveyMain getSurveyMain() { + SurveyMain sm = new SurveyMain(); + sm.xpt = new XPathTable(); + sm.reg = new UserRegistry() { + final VoterInfo user0 = new VoterInfo(Organization.guest, VoteResolver.Level.tc, "Test User", new LocaleSet(true)); + + @Override + public synchronized Map getVoterToInfo() { + Map m = new TreeMap(); + m.put(0, user0); + return m; + } + + @Override + public VoterInfo getVoterToInfo(int userid) { + if (userid == 0) { + return user0; + } else { + return null; + } + } + }; + return sm; + } +} diff --git a/tools/cldr-code/src/main/java/org/unicode/cldr/test/TestCache.java b/tools/cldr-code/src/main/java/org/unicode/cldr/test/TestCache.java index 3f874de3cd0..389d383e82c 100644 --- a/tools/cldr-code/src/main/java/org/unicode/cldr/test/TestCache.java +++ b/tools/cldr-code/src/main/java/org/unicode/cldr/test/TestCache.java @@ -310,4 +310,12 @@ private static void updateExampleGeneratorCache(String xpath, CLDRLocale locale) eg.updateCache(xpath); } } + + /** + * For tests. Invalidate cache. + */ + public void invalidateAllCached() { + testResultCache.invalidateAll(); + exampleGeneratorCache.invalidateAll(); + } }