Merge pull request #78 from ilo-token/master

0.6.0

Author: neverRare

CHANGELOG.md

@@ -10,11 +10,35 @@ NOTE: Before publishing:
 <!--
 <details>
 <summary>On development changelog</summary>
+
+## (On development)
+
 The latest on-development version can be accessed by building the source code.
 On this on-development version, things can be broken.
 </details>
 -->
 
+## 0.6.0
+
+Released 22 April 2025
+
+ilo Token can now translate prepositions. This includes preposition before "la"
+such as "tawa mi la sina pona" and prepositional predicate such as "mi kepeken
+ilo".
+
+- Implement translation of preposition.
+- Implement "nanpa X la" translation.
+- Context clauses can now be translated into adverbs e.g. "pona la mi moku" will
+  be translated as "nicely, I ate".
+- Fix "nanpa" translation: Before it translated into just "position" instead of
+  e.g. "position 1".
+- When telo misikeke is enabled, it won't mark words defined in the custom
+  dictionary.
+- Improve error messages for bugs. A detailed instruction for bug reporting is
+  now shown.
+- Improve custom dictionary error messages. There is now a dedicated space for
+  displaying errors.
+
 ## 0.5.3
 
 Released 2 April 2025

CONTRIBUTING.md

@@ -5,21 +5,13 @@ about the pages to get you started. Remember you can always ask for help in the
 [discussion](#Discussion) or privately through my email:
 [[email protected]](mailto:[email protected])
 
-## [Wiki](https://github.com/ilo-token/ilo-token.github.io/wiki)
-
-The wiki is a place full of useful information for contributors from how the
-code works to guidelines for editing the dictionary!
-
-It's not perfect however. If you spot a mistake, please open an issue.
-
 ## [Issue](https://github.com/ilo-token/ilo-token.github.io/issues)
 
-The issue page is intended for tracking the development of ilo Token as well as
-its wiki. You may open an issue for:
+The issue page is intended for tracking the development of ilo Token. You may
+open an issue for:
 
 - Bug report
 - Feature request
-- Fix suggestion to the wiki
 
 Please remember to search first before opening an issue, it might already exist!
 Also check the [changelog](./CHANGELOG.md) and open the on-development
@@ -51,8 +43,6 @@ Please search first before opening a new page! Duplicate pages are unnecessary.
 Before forking and editing, please claim an issue first or open an issue then
 claim it. After that, you can start away. This is necessary to avoid wasted
 duplicate efforts.
-[The wiki](https://github.com/ilo-token/ilo-token.github.io/wiki) contains
-useful information for contributors.
 
 You **don't** have to open a new issue this if:
 
@@ -71,7 +61,7 @@ More things to remember:
   `// This code is browser/Deno only`.
 
 (The following aren't strict rules. It's perfectly fine to not follow any of
-these, we can adapt.)
+these for any reason e.g. you just want to fix typo.)
 
 - Ensure all the files are formatted: Run `deno fmt`.
 - Make sure you don't accidentally make more tests fail: Run

README.md

@@ -13,7 +13,7 @@ interpretation of the text. No machine learning involved.
 
 After using `git clone`, you'll need [Deno](https://deno.com/) for this project.
 Then you'll need to install the dependencies before running any command:
-`deno install`.
+`deno install --frozen`.
 
 After this, you can start away by running `deno task start` then open the link
 provided by the stdout to open a local copy of ilo Token. The server will

build/build.ts

@@ -1,16 +1,22 @@
 // This code is Deno only
 
 import * as ESBuild from "esbuild";
-import { OPTIONS } from "./config.ts";
 import * as Dictionary from "../dictionary/build.ts";
+import { OPTIONS } from "./config.ts";
 
 const BUILD_OPTIONS: ESBuild.BuildOptions = {
   ...OPTIONS,
   minify: true,
   define: { LIVE_RELOAD: "false" },
 };
-
 if (import.meta.main) {
-  await Dictionary.build();
-  await ESBuild.build(BUILD_OPTIONS);
+  const start = performance.now();
+  if (await Dictionary.build()) {
+    await ESBuild.build(BUILD_OPTIONS);
+    const end = performance.now();
+    // deno-lint-ignore no-console
+    console.log(`Total time took: ${Math.floor(end - start)}ms`);
+  } else {
+    Deno.exitCode = 1;
+  }
 }

build/dev.ts

@@ -3,26 +3,33 @@
 import { assert } from "@std/assert/assert";
 import { exists } from "@std/fs/exists";
 import { BuildOptions, context } from "esbuild";
-import { AsyncDisposableStack } from "../misc/async_disposable_stack.ts";
 import { OPTIONS } from "./config.ts";
 
+const DICTIONARY = new URL("../dictionary/dictionary.ts", import.meta.url);
+
 const BUILD_OPTIONS: BuildOptions = {
   ...OPTIONS,
   minify: false,
   define: { LIVE_RELOAD: "true" },
 };
 async function watchMain(): Promise<AsyncDisposable> {
   await using stack = new AsyncDisposableStack();
-  const buildContext = await context(BUILD_OPTIONS);
-  stack.defer(async () => await buildContext.dispose());
-  buildContext.watch();
-  buildContext.serve({ servedir: "./dist/" });
-  return stack.move();
+  const buildContext = stack.use({
+    context: await context(BUILD_OPTIONS),
+    async [Symbol.asyncDispose](): Promise<void> {
+      await this.context.dispose();
+    },
+  });
+  buildContext.context.watch();
+  buildContext.context.serve({ servedir: "./dist/" });
+  stack.move();
+  return buildContext;
 }
 async function watchDictionary(): Promise<number> {
   const command = new Deno.Command(Deno.execPath(), {
     args: [
       "run",
+      "-E=NO_COLOR",
       "-R=./dictionary/dictionary",
       "-W=./dictionary/dictionary.ts",
       "--no-prompt",
@@ -43,17 +50,17 @@ async function watchDictionary(): Promise<number> {
   return status.code;
 }
 if (import.meta.main) {
-  let statusCode: number;
-  {
-    if (
-      !await exists(new URL("../dictionary/dictionary.ts", import.meta.url))
-    ) {
-      const Dictionary = await import("../dictionary/build.ts");
-      await Dictionary.build();
+  if (!await exists(DICTIONARY)) {
+    const Dictionary = await import("../dictionary/build.ts");
+    if (!await Dictionary.build()) {
+      await Dictionary.buildWithDictionary(new Map());
+      // deno-lint-ignore no-console
+      console.error(
+        "Dictionary failed to build. Empty dictionary is used instead. Please fix it.",
+      );
     }
-    const statusCodePromise = watchDictionary();
-    await using _ = await watchMain();
-    statusCode = await statusCodePromise;
   }
-  Deno.exit(statusCode);
+  const statusCodePromise = watchDictionary();
+  await using _ = await watchMain();
+  Deno.exitCode = await statusCodePromise;
 }

deno.json

@@ -18,11 +18,11 @@
     },
     "update": {
       "description": "Updates dependencies and telo misikeke",
-      "command": "deno outdated --update && deno run -N=api.linku.la,gitlab.com -W=./telo_misikeke/linku_data.json,./telo_misikeke/Parser.js,./telo_misikeke/rules.js --no-prompt --frozen --cached-only ./telo_misikeke/update.ts"
+      "command": "deno outdated --update && deno run -N=gitlab.com -W=./telo_misikeke/linku_data.json,./telo_misikeke/Parser.js,./telo_misikeke/rules.js --no-prompt --frozen --cached-only ./telo_misikeke/update.ts"
     },
     "build-dictionary": {
       "description": "Builds the dictionary at `./dictionary/dictionary.ts`",
-      "command": "deno run -R=./dictionary/dictionary -W=./dictionary/dictionary.ts --no-prompt --frozen --cached-only ./dictionary/build.ts"
+      "command": "deno run -E=NO_COLOR -R=./dictionary/dictionary -W=./dictionary/dictionary.ts --no-prompt --frozen --cached-only ./dictionary/build.ts"
     }
   },
   "fmt": {
@@ -55,7 +55,6 @@
         "no-self-compare",
         "no-sparse-arrays",
         "no-sync-fn-in-async-fn",
-        "no-throw-literal",
         "no-useless-rename",
         "prefer-ascii",
         "single-var-declarator"
@@ -69,7 +68,7 @@
     "@std/async": "jsr:@std/async@^1.0.12",
     "@std/cache": "jsr:@std/cache@^0.2.0",
     "@std/collections": "jsr:@std/collections@^1.0.10",
-    "@std/fs": "jsr:@std/fs@^1.0.15",
+    "@std/fs": "jsr:@std/fs@^1.0.16",
     "@std/html": "jsr:@std/html@^1.0.3",
     "@std/random": "jsr:@std/random@^0.1.0",
     "@std/regexp": "jsr:@std/regexp@^1.0.1",

deno.lock

@@ -1,16 +1,20 @@
 // This code is Deno only
 
-import { parseDictionary } from "./parser.ts";
+// deno-lint-ignore-file no-console
+
+import { ArrayResultError } from "../src/array_result.ts";
+import { PositionedError } from "../src/parser/parser_lib.ts";
+import { dictionaryParser } from "./parser.ts";
+import { Dictionary } from "./type.ts";
 
 const SOURCE = new URL("./dictionary", import.meta.url);
 const DESTINATION = new URL("./dictionary.ts", import.meta.url);
 
-export async function build(): Promise<void> {
-  // deno-lint-ignore no-console
-  console.log("Building dictionary...");
-  const text = await Deno.readTextFile(SOURCE);
+export async function buildWithDictionary(
+  dictionary: Dictionary,
+): Promise<void> {
   const json = JSON.stringify(
-    Object.fromEntries(parseDictionary(text)),
+    Object.fromEntries(dictionary),
     undefined,
     2,
   );
@@ -22,9 +26,93 @@ import { Dictionary } from "./type.ts";
 export const dictionary: Dictionary = new Map(Object.entries(${json}));
 `;
   await Deno.writeTextFile(DESTINATION, code);
-  // deno-lint-ignore no-console
-  console.log("Building dictionary done");
+}
+export async function build(): Promise<boolean> {
+  console.log("Building dictionary...");
+  const start = performance.now();
+  const text = await Deno.readTextFile(SOURCE);
+  const startDictionary = performance.now();
+  const result = dictionaryParser.parse(text);
+  const endDictionary = performance.now();
+  let dictionary: Dictionary;
+  if (!result.isError()) {
+    dictionary = result.array[0];
+  } else {
+    displayError(text, result.errors);
+    return false;
+  }
+  await buildWithDictionary(dictionary);
+  const end = performance.now();
+  const total = Math.floor(end - start);
+  const parsing = Math.floor(endDictionary - startDictionary);
+  console.log(
+    `Building dictionary done in ${total}ms (parsing dictionary took ${parsing}ms)`,
+  );
+  return true;
+}
+function displayError(
+  source: string,
+  errors: ReadonlyArray<ArrayResultError>,
+): void {
+  let color: boolean;
+  try {
+    color = Deno.env.get("NO_COLOR") !== "1";
+  } catch (error) {
+    if (error instanceof Deno.errors.NotCapable) {
+      color = true;
+    } else {
+      throw error;
+    }
+  }
+  const red = color ? "color: red" : "";
+  const sourceStyle = color ? "color: blue" : "";
+  for (const error of errors) {
+    console.error(`%cError%c: ${error.message}`, red, "");
+    if (error instanceof PositionedError && error.position != null) {
+      const { position, length } = error.position;
+      const end = position + length;
+      // The only instance returning -1 is useful
+      const startLine = source.lastIndexOf("\n", position) + 1;
+      let currentLine = startLine;
+      let currentPosition = position;
+
+      while (true) {
+        const index = source.indexOf("\n", currentLine);
+        const nextLine = index === -1 ? source.length : index + 1;
+        const line = source.slice(currentLine, nextLine).trimEnd();
+        console.error(line);
+        let relativeStart = Math.min(
+          currentPosition - currentLine,
+          line.length,
+        );
+        let relativeEnd = Math.min(end - currentLine, line.length);
+        if (relativeEnd - relativeStart === 0) {
+          if (relativeStart !== 0) {
+            relativeStart--;
+          }
+          if (relativeEnd !== line.length) {
+            relativeEnd++;
+          }
+        }
+        console.error(
+          `${" ".repeat(relativeStart)}%c${
+            "^".repeat(relativeEnd - relativeStart)
+          }`,
+          red,
+        );
+        if (end <= nextLine) {
+          break;
+        } else {
+          currentLine = currentPosition = nextLine;
+        }
+      }
+      const line = source.slice(0, startLine).match(/\n/g)?.length ?? 1;
+      const column = position - startLine + 1;
+      console.error(`    at %c${SOURCE}:${line}:${column}`, sourceStyle);
+      console.error();
+    }
+  }
 }
 if (import.meta.main) {
-  await build();
+  Deno.exitCode = await build() ? 0 : 1;
 }