feat(sdk): 1882 complete google datastore store integration and tests (#3313)

* feat(sdk): 1882 complete google datastore store integration and tests

Signed-off-by: David Adams <[email protected]>

* improve(sdk): stricter gstore tests, minor updates

Signed-off-by: David Adams <[email protected]>

Author: David A

packages/sdk/package.json

@@ -38,12 +38,14 @@
   ],
   "dependencies": {
     "@uma/core": "^2.7.0",
-    "ethers": "^5.4.2"
+    "ethers": "^5.4.2",
+    "highland": "^2.13.5"
   },
   "devDependencies": {
+    "@types/highland": "^2.12.13",
     "@ethersproject/abi": "^5.4.0",
-    "@ethersproject/contracts": "^5.4.0",
     "@ethersproject/abstract-provider": "^5.4.0",
+    "@ethersproject/contracts": "^5.4.0",
     "@ethersproject/providers": "^5.4.2",
     "@size-limit/preset-small-lib": "^4.10.2",
     "size-limit": "^4.10.2",

packages/sdk/src/stores/google-datastore/store.e2e.ts

@@ -3,36 +3,142 @@ import Store from ".";
 
 import { Datastore } from "@google-cloud/datastore";
 
-// TODO: fix e2e tests, we need to skip this for now, e2e tests are broken
-describe.skip("google-store", function () {
+type Price = {
+  id?: string;
+  timestamp: number;
+  identifier: string;
+  price: number;
+};
+function makeId(data: Price) {
+  return [data.identifier, data.timestamp.toString().padStart(4, "0")].join("!");
+}
+function sortPrices(a: Price, b: Price) {
+  return makeId(a) <= makeId(b) ? -1 : 1;
+}
+const prices: Price[] = [
+  { timestamp: 10, identifier: "a", price: 100 },
+  { timestamp: 11, identifier: "a", price: 99 },
+  { timestamp: 9, identifier: "a", price: 98 },
+  { timestamp: 2, identifier: "a", price: 82 },
+  { timestamp: 13, identifier: "a", price: 101 },
+
+  { timestamp: 10, identifier: "b", price: 1000 },
+  { timestamp: 11, identifier: "b", price: 990 },
+  { timestamp: 9, identifier: "b", price: 980 },
+  { timestamp: 2, identifier: "b", price: 820 },
+  { timestamp: 13, identifier: "b", price: 1001 },
+];
+const sortedPrices = [...prices].sort(sortPrices);
+
+describe("google-store", function () {
   let store: any;
   let datastore: Datastore;
   test("init", function () {
     datastore = new Datastore();
-    store = Store("testing", datastore);
+    store = Store("testing-prices", datastore);
     assert(store);
   });
-  test("delete", async function () {
+  test("clear", async function () {
     try {
-      await store.delete("test");
+      await store.clear();
     } catch (err) {
       // do nothing
     }
   });
-  test("set", async function () {
-    await store.set("test", { testing: true });
+  test("add prices", async () => {
+    for (const price of prices) {
+      await store.set(makeId(price), price);
+    }
+  });
+  test("entries", async function () {
+    const result = await store.entries();
+    assert.equal(result.length, prices.length);
+  });
+  test("keys", async function () {
+    const result = await store.keys();
+    assert.equal(result.length, prices.length);
+    result.forEach((id: string, i: number) => {
+      assert.equal(id, makeId(sortedPrices[i]));
+    });
+  });
+  test("values", async function () {
+    const result = await store.values();
+    assert.equal(result.length, prices.length);
+    result.forEach((price: Price, i: number) => {
+      assert.equal(price.identifier, sortedPrices[i].identifier);
+      assert.equal(price.timestamp, sortedPrices[i].timestamp);
+      assert.equal(price.price, sortedPrices[i].price);
+    });
   });
   test("get", async function () {
-    const result = await store.get("test");
-    assert.deepEqual(result, { testing: true });
+    const result = await store.get(makeId(prices[0]));
+    assert.deepEqual(result, prices[0]);
   });
   test("has", async function () {
-    const result = await store.has("test");
+    let result = await store.has(makeId(prices[0]));
     assert.equal(result, true);
+
+    result = await store.has("dne");
+    assert.equal(result, false);
+  });
+  // not supported
+  test("size", async function () {
+    try {
+      await store.size();
+    } catch (err) {
+      assert.ok(err);
+    }
+  });
+  test("between a", async () => {
+    const result = await store.between("a", "b");
+    const answer = sortedPrices.filter((x) => x.identifier === "a");
+    assert.equal(result.length, 5);
+    result.forEach((price: Price, i: number) => {
+      assert.equal(price.identifier, answer[i].identifier);
+      assert.equal(price.timestamp, answer[i].timestamp);
+      assert.equal(price.price, answer[i].price);
+    });
+  });
+  test("between b", async () => {
+    const result = await store.between("b", "b~");
+    const answer = sortedPrices.filter((x) => x.identifier === "b");
+    assert.equal(result.length, 5);
+    result.forEach((price: Price, i: number) => {
+      assert.equal(price.identifier, answer[i].identifier);
+      assert.equal(price.timestamp, answer[i].timestamp);
+      assert.equal(price.price, answer[i].price);
+    });
+  });
+  test("slice a", async () => {
+    const len = 4;
+    const result = await store.slice("a", len);
+    assert.equal(result.length, len);
+
+    const answer = sortedPrices.filter((x) => x.identifier === "a").slice(0, len);
+    result.forEach((price: Price, i: number) => {
+      assert.equal(price.identifier, answer[i].identifier);
+      assert.equal(price.timestamp, answer[i].timestamp);
+      assert.equal(price.price, answer[i].price);
+    });
+  });
+  test("slice b", async () => {
+    const len = 3;
+    const result = await store.slice("b", len);
+    assert.equal(result.length, len);
+
+    const answer = sortedPrices.filter((x) => x.identifier === "b").slice(0, len);
+    result.forEach((price: Price, i: number) => {
+      assert.equal(price.identifier, answer[i].identifier);
+      assert.equal(price.timestamp, answer[i].timestamp);
+      assert.equal(price.price, answer[i].price);
+    });
   });
   test("delete", async function () {
-    await store.delete("test");
-    const result = await store.has("test");
+    await store.delete(makeId(prices[0]));
+    const result = await store.has(makeId(prices[0]));
     assert.equal(result, false);
   });
+  test("clear", async function () {
+    await store.clear();
+  });
 });

packages/sdk/src/stores/google-datastore/store.ts

@@ -1,15 +1,18 @@
+import { exists } from "../../utils";
 import type { SortedStore } from "..";
-import type { Datastore, Key } from "@google-cloud/datastore";
+import { Datastore, Key } from "@google-cloud/datastore";
+import highland from "highland";
 
 // TODO: this is a work in progress which will be completed during datastore integration.
 export default function <D>(kind: string, store: Datastore): SortedStore<string, D> {
   function makeKey(id: string): Key {
     return store.key([kind, id]);
   }
+  // return value or undefined if it doesnt exist
   async function get(id: string) {
     try {
-      const [first] = await store.get(makeKey(id));
-      return first;
+      const [result] = await store.get(makeKey(id));
+      return result;
     } catch (err) {
       return;
     }
@@ -22,37 +25,59 @@ export default function <D>(kind: string, store: Datastore): SortedStore<string,
     });
   }
   async function has(id: string) {
-    try {
-      await get(id);
-      return true;
-    } catch (err) {
-      return false;
-    }
+    // horribly ineficient to actually query data to see if it exists, but cannot find a better way
+    return exists(await get(id));
   }
   async function del(id: string) {
     await store.delete(makeKey(id));
   }
   async function entries() {
-    return [];
+    const [results] = await store.createQuery(kind).run();
+    return results.map((result) => {
+      return [result[store.KEY].name, result];
+    }) as [string, D][];
   }
   async function values() {
-    return [];
+    const [results] = await store.createQuery(kind).run();
+    return results;
   }
+  // all this does is map over full entries to return the id of the entry. have not found a better way to query this.
   async function keys() {
-    return [];
+    const [results] = await store.createQuery(kind).select("__key__").run();
+    return results.map((result) => {
+      return result[store.KEY].name;
+    }) as string[];
   }
-  async function size() {
-    return 0;
+  // theres no way to really do built into the store client. Google recommends managing a size entry yourself.
+  async function size(): Promise<number> {
+    throw new Error("size not supported in google store");
   }
-  // TODO: implement these functions
+  // this actually queries all values, then batches them to delete N at a time. This is a horrible way
+  // to do this, but I could not find a better way to "drop" all kinds from the table.
   async function clear() {
-    throw new Error("clear is not implemented");
+    await highland(store.createQuery(kind).runStream())
+      .map((val: any) => {
+        return val[store.KEY];
+      })
+      .batch(500)
+      .map(async (keys) => {
+        return store.delete(keys);
+      })
+      .flatMap(highland)
+      .collect()
+      .toPromise(Promise);
   }
-  async function slice() {
-    return [];
+  async function slice(id: string, length: number) {
+    const [result] = await store.createQuery(kind).filter("__key__", ">=", makeKey(id)).limit(length).run();
+    return result;
   }
-  async function between() {
-    return [];
+  async function between(a: string, b: string) {
+    const [result] = await store
+      .createQuery(kind)
+      .filter("__key__", ">=", makeKey(a))
+      .filter("__key__", "<", makeKey(b))
+      .run();
+    return result;
   }
 
   return {

yarn.lock

@@ -6298,6 +6298,13 @@
   dependencies:
     "@types/node" "*"
 
+"@types/highland@^2.12.13":
+  version "2.12.13"
+  resolved "https://registry.yarnpkg.com/@types/highland/-/highland-2.12.13.tgz#ec41635d9b4b7305c6ea41b7e4a37c6598cccfd5"
+  integrity sha512-mKESwg+EDt+wJTxl8uIiuwP6Vc0jWoo9XdpT6fLYutw56I63obI+/FTIed3WHXxAjIzkjJOgx4ESS4JuGwFpQw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/hoist-non-react-statics@^3.0.1":
   version "3.3.1"
   resolved "https://registry.yarnpkg.com/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.1.tgz#1124aafe5118cb591977aeb1ceaaed1070eb039f"