Fix target/source keymapping (#402)

thomcom · trxcllnt · web-flow · commit 5f3833b4ac71 · 2022-06-28T15:49:25.000-05:00
* Insane errors are percolating. * Fix #397 * Forgotten files, ugh. * Typos in schema. * Update modules/demo/api-server/util/gpu_cache.js Co-authored-by: Paul Taylor <paul.e.taylor@me.com> * Update modules/demo/api-server/util/gpu_cache.js Co-authored-by: Paul Taylor <paul.e.taylor@me.com> Co-authored-by: Paul Taylor <paul.e.taylor@me.com>
diff --git a/modules/demo/api-server/index.js b/modules/demo/api-server/index.js
@@ -29,5 +29,5 @@ const env = {
 };
 
 spawnSync(process.execPath,
-          [fastify, 'start', '-l', 'info', '-P', '-w', 'app.js'],
+          [fastify, 'start', '-l', 'info', '-P', '-p', '3010', '-w', 'app.js'],
           {env, cwd: __dirname, stdio: 'inherit'});
diff --git a/modules/demo/api-server/routes/graphology/index.js b/modules/demo/api-server/routes/graphology/index.js
@@ -304,11 +304,16 @@ module.exports = async function(fastify, opts) {
         let tiled = Series.sequence({type: new Float32, init: 0.0, size: (6 * edges.numRows)});
         let base_offset = Series.sequence({type: new Int32, init: 0.0, size: edges.numRows}).mul(3);
         //
-        // Duplicatin the sigma.j createNormalizationFunction here because there's no other way
-        // to let the Graph object compute it.
+        // Duplicatin the sigma.js createNormalizationFunction here because this is the best way
+        // to let the Graph object compute it on GPU.
         //
-        let source         = edges.get('source');
-        let target         = edges.get('target');
+        // Remap the indices in the key table to their real targets. See
+        // https://github.com/rapidsai/node/issue/397
+        let keys           = df.get('key');
+        let source_map     = edges.get('source');
+        let source         = keys.gather(source_map, false);
+        let target_map     = edges.get('target');
+        let target         = keys.gather(target_map, false);
         let x              = df.get('x');
         let y              = df.get('y');
         const [xMin, xMax] = x.minmax();
@@ -318,10 +323,10 @@ module.exports = async function(fastify, opts) {
         const dY           = (yMax + yMin) / 2.0;
         x                  = x.add(-1.0 * dX).mul(1.0 / ratio).add(0.5);
         y                  = y.add(-1.0 * dY).mul(1.0 / ratio).add(0.5);
-        const source_xmap  = x.gather(source.cast(new Int32));
-        const source_ymap  = y.gather(source.cast(new Int32));
-        const target_xmap  = x.gather(target.cast(new Int32));
-        const target_ymap  = y.gather(target.cast(new Int32));
+        const source_xmap  = x.gather(source, false);
+        const source_ymap  = y.gather(source, false);
+        const target_xmap  = x.gather(target, false);
+        const target_ymap  = y.gather(target, false);
         const color        = Series.new(['#999'])
                         .hexToIntegers(new Int32)
                         .bitwiseOr(0xff000000)
diff --git a/modules/demo/api-server/test/fixtures.js b/modules/demo/api-server/test/fixtures.js
@@ -0,0 +1,134 @@
+// Copyright (c) 2022, NVIDIA CORPORATION.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const json_good = {
+  'json_good.txt':
+    ` {
+          "nodes":
+            [
+              {
+                "key": "customer data management",
+                "label": "Customer data management",
+                "tag": "Field",
+                "URL": "https://en.wikipedia.org/wiki/Customer%20data%20management",
+                "cluster": "7",
+                "x": -278.2200012207031,
+                "y": 436.0100402832031,
+                "score": 0
+              },
+              {
+                "key": "educational data mining",
+                "label": "Educational data mining",
+                "tag": "Field",
+                "URL": "https://en.wikipedia.org/wiki/Educational%20data%20mining",
+                "cluster": "7",
+                "x": -1.9823756217956543,
+                "y": 250.4990692138672,
+                "score": 0
+              }
+            ],
+            "edges":
+              [
+                ["office suite", "human interactome"],
+                ["educational data mining", "human interactome"],
+              ],
+            "clusters":
+              [
+                {"key": "0", "color": "#6c3e81", "clusterLabel": "human interactome"},
+                {"key": "1", "color": "#666666", "clusterLabel": "Spreadsheets"},
+              ],
+            "tags": [
+              {"key": "Chart type", "image": "charttype.svg"},
+              {"key": "Company", "image": "company.svg"},
+            ]
+        } `
+};
+
+const json_large = {
+  'json_large.txt':
+    ` {
+      "attributes": {},
+      "nodes": [
+        {
+          "key": "0",
+          "attributes": {
+            "cluster": 0,
+            "x": -13.364310772761677,
+            "y": 4.134339113107921,
+            "size": 0,
+            "label": "Node n°291, in cluster n°0",
+            "color": "#e24b04"
+          }
+        },
+        {
+          "key": "1",
+          "attributes": {
+            "cluster": 1,
+            "x": 1.3836898237261988,
+            "y": -11.536596764896206,
+            "size": 1,
+            "label": "Node n°292, in cluster n°1",
+            "color": "#323455"
+          }
+        }
+      ],
+      "edges": [
+        {"key": "geid_115_98", "source": "291", "target": "290"},
+        {"key": "geid_115_99", "source": "290", "target": "291"}
+      ],
+      "options": {"type": "mixed", "multi": false, "allowSelfLoops": true}
+    }`
+};
+
+const json_out_of_order = {
+  'json_out_of_order.txt':
+    ` {
+      "attributes": {},
+      "nodes": [
+        {
+          "key": "290",
+          "attributes": {
+            "cluster": 0,
+            "x": -13.364310772761677,
+            "y": 4.134339113107921,
+            "size": 0,
+            "label": "Node n°291, in cluster n°0",
+            "color": "#e24b04"
+          }
+        },
+        {
+          "key": "291",
+          "attributes": {
+            "cluster": 1,
+            "x": 1.3836898237261988,
+            "y": -11.536596764896206,
+            "size": 1,
+            "label": "Node n°292, in cluster n°1",
+            "color": "#323455"
+          }
+        }
+      ],
+      "edges": [
+        {"key": "geid_115_98", "source": "290", "target": "291"},
+        {"key": "geid_115_99", "source": "291", "target": "290"}
+      ],
+      "options": {"type": "mixed", "multi": false, "allowSelfLoops": true}
+    }`
+};
+
+module.exports = {
+  json_good: json_good,
+  json_large: json_large,
+  json_out_of_order: json_out_of_order
+};
diff --git a/modules/demo/api-server/test/routes/graphology.test.js b/modules/demo/api-server/test/routes/graphology.test.js
@@ -14,11 +14,11 @@
 
 'use strict'
 
-const {dir}                                   = require('console');
-const {test}                                  = require('tap');
-const {build}                                 = require('../helper');
-const {tableFromIPC, RecordBatchStreamWriter} = require('apache-arrow');
-const {json_large, json_good}                 = require('../fixtures.js');
+const {dir}                                      = require('console');
+const {test}                                     = require('tap');
+const {build}                                    = require('../helper');
+const {tableFromIPC, RecordBatchStreamWriter}    = require('apache-arrow');
+const {json_large, json_good, json_out_of_order} = require('../fixtures.js');
 
 test('graphology root returns api description', async t => {
   const app = await build(t);
@@ -30,9 +30,9 @@ test('graphology root returns api description', async t => {
         read_json: {
           filename: 'A URI to a graphology json dataset file.',
           result: `Causes the node-rapids backend to attempt to load the json object specified
-                     by :filename. The GPU with attempt to parse the json file asynchronously and will
+                     by :filename. The GPU will attempt to parse the json file asynchronously and will
                      return OK/ Not Found/ or Fail based on the file status.
-                     If the load is successful, three tables will be created in the node-rapids backend:
+                     If the load is successful, four tables will be created in the node-rapids backend:
                      nodes, edges, clusters, and tags. The root objects in the json target must match
                      these names and order.`,
           returns: 'Result OK/Not Found/Fail'
@@ -187,8 +187,7 @@ test('get_column', async (t) => {
 test('nodes', async (t) => {
   const dir   = t.testdir(json_large);
   const rpath = '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_large.txt';
-  console.log(rpath);
-  const app = await build(t);
+  const app   = await build(t);
   const load =
     await app.inject({method: 'POST', url: '/graphology/read_large_demo?filename=' + rpath});
   const res = await app.inject(
@@ -235,18 +234,48 @@ test('edges', async (t) => {
     await app.inject({method: 'POST', url: '/graphology/read_large_demo?filename=' + rpath});
   const res = await app.inject(
     {method: 'GET', url: '/graphology/edges', header: {'accepts': 'application/octet-stream'}});
+  t.equal(res.statusCode, 200);
   const table   = tableFromIPC(res.rawPayload);
   const release = await app.inject({method: 'POST', url: '/graphology/release'});
   t.ok(table.getChild('edges'));
   t.same(table.getChild('edges').toArray(), new Float32Array([
            0.02944733388721943,
            1,
            -1.701910173408654e+38,
-           0.9705526828765869,
-           0,
+           0.02944733388721943,
+           1,
            -1.701910173408654e+38,
-           0.9705526828765869,
-           0,
+           0.02944733388721943,
+           1,
+           -1.701910173408654e+38,
+           0.02944733388721943,
+           1,
+           -1.701910173408654e+38
+         ]))
+});
+
+test('edges out of order', async (t) => {
+  const dir = t.testdir(json_out_of_order);
+  const rpath =
+    '../../test/routes/' + dir.substring(dir.lastIndexOf('/')) + '/json_out_of_order.txt';
+  const app = await build(t);
+  const load =
+    await app.inject({method: 'POST', url: '/graphology/read_large_demo?filename=' + rpath});
+  const res = await app.inject(
+    {method: 'GET', url: '/graphology/edges', header: {'accepts': 'application/octet-stream'}});
+  t.equal(res.statusCode, 200);
+  const table   = tableFromIPC(res.rawPayload);
+  const release = await app.inject({method: 'POST', url: '/graphology/release'});
+  t.ok(table.getChild('edges'));
+  t.same(table.getChild('edges').toArray(), new Float32Array([
+           0.02944733388721943,
+           1,
+           -1.701910173408654e+38,
+           0.02944733388721943,
+           1,
+           -1.701910173408654e+38,
+           0.02944733388721943,
+           1,
            -1.701910173408654e+38,
            0.02944733388721943,
            1,
diff --git a/modules/demo/api-server/test/routes/root.test.js b/modules/demo/api-server/test/routes/root.test.js
@@ -27,9 +27,9 @@ test('root returns API description', async (t) => {
         read_json: {
           filename: 'A URI to a graphology json dataset file.',
           result: `Causes the node-rapids backend to attempt to load the json object specified
-                     by :filename. The GPU with attempt to parse the json file asynchronously and will
+                     by :filename. The GPU will attempt to parse the json file asynchronously and will
                      return OK/ Not Found/ or Fail based on the file status.
-                     If the load is successful, three tables will be created in the node-rapids backend:
+                     If the load is successful, four tables will be created in the node-rapids backend:
                      nodes, edges, clusters, and tags. The root objects in the json target must match
                      these names and order.`,
           returns: 'Result OK/Not Found/Fail'
diff --git a/modules/demo/api-server/util/gpu_cache.js b/modules/demo/api-server/util/gpu_cache.js
@@ -28,10 +28,13 @@ function json_key_attributes_to_dataframe(str) {
   const dtypes  = [new Int32, new Float32, new Float32, new Int32, new Utf8String, new Utf8String];
   const no_open_list = str.split('[\n').gather([1], false);
   const tokenized    = no_open_list.split('},');
+  const keys         = tokenized.getJSONObject('.key');
+  keys.setNullMask(1, 0);
+  arr['key'] = keys.cast(new Int32);
   columns.forEach((col, ix) => {
     const parse_result = tokenized.getJSONObject('.attributes.' + columns[ix]);
-    const string_array = Series.new(parse_result);
-    arr[col]           = string_array.cast(dtypes[ix]);
+    parse_result.setNullMask([], 0);
+    arr[col] = parse_result.cast(dtypes[ix]);
   });
   const result = new DataFrame(arr);
   return result;
@@ -43,7 +46,8 @@ function json_aos_to_dataframe(str, columns, dtypes) {
     const no_open_list = str.split('[\n').gather([1], false);
     const tokenized    = no_open_list.split('},');
     const parse_result = tokenized.getJSONObject('.' + columns[ix]);
-    arr[col]           = Series.new(parse_result).cast(dtypes[ix]);
+    parse_result.setNullMask(1, 0);
+    arr[col] = parse_result.cast(dtypes[ix]);
   });
   const result = new DataFrame(arr);
   return result;
@@ -56,8 +60,8 @@ function json_aoa_to_dataframe(str, dtypes) {
   dtypes.forEach((_, ix) => {
     const get_ix       = `[${ix}]`;
     const parse_result = tokenized.getJSONObject(get_ix);
-    const string_array = Series.new(parse_result);
-    arr[ix]            = string_array.cast(dtypes[ix]);
+    parse_result.setNullMask([], 0);
+    arr[ix] = parse_result.cast(dtypes[ix]);
   });
   const result = new DataFrame(arr);
   return result;
@@ -99,7 +103,7 @@ module.exports = {
     const tnodes = split.gather([1], false);
     const nodes  = json_key_attributes_to_dataframe(tnodes);
     const edges  = json_aos_to_dataframe(
-      tedges, ['key', 'source', 'target'], [new Utf8String, new Int32, new Int32]);
+      tedges, ['key', 'source', 'target'], [new Utf8String, new Int64, new Int64]);
     let optionsArr               = {};
     optionsArr['type']           = Series.new(toptions.getJSONObject('.type'));
     optionsArr['multi']          = Series.new(toptions.getJSONObject('.multi'));
@@ -129,7 +133,7 @@ module.exports = {
     const tnodes = split.gather([1], false);
     const tags   = json_aos_to_dataframe(ttags, ['key', 'image'], [new Utf8String, new Utf8String]);
     const clusters = json_aos_to_dataframe(
-      tclusters, ['key', 'color', 'clusterLabel'], [new Int32, new Utf8String, new Utf8String]);
+      tclusters, ['key', 'color', 'clusterLabel'], [new Int64, new Utf8String, new Utf8String]);
     const nodes =
       json_aos_to_dataframe(tnodes, ['key', 'label', 'tag', 'URL', 'cluster', 'x', 'y', 'score'], [
         new Utf8String,
diff --git a/modules/demo/api-server/util/schema.js b/modules/demo/api-server/util/schema.js
@@ -0,0 +1,55 @@
+// Copyright (c) 2022, NVIDIA CORPORATION.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const schema = {
+  graphology: {
+    description: 'The graphology api provides GPU acceleration of graphology datasets.',
+    schema: {
+      read_json: {
+        filename: 'A URI to a graphology json dataset file.',
+        result: `Causes the node-rapids backend to attempt to load the json object specified
+                     by :filename. The GPU will attempt to parse the json file asynchronously and will
+                     return OK/ Not Found/ or Fail based on the file status.
+                     If the load is successful, four tables will be created in the node-rapids backend:
+                     nodes, edges, clusters, and tags. The root objects in the json target must match
+                     these names and order.`,
+        returns: 'Result OK/Not Found/Fail'
+      },
+      read_large_demo: {
+        filename:
+          'A URI to a graphology json dataset file matching the sigma.js/examples/large-demos spec.',
+        result: `Produces the same result as 'read_json'.
+                     If the load is successful, three tables will be created in the node-rapids backend:
+                     nodes, edges, and options.`,
+        returns: 'Result OK/Not Found/Fail'
+      },
+      list_tables: {returns: 'Tables that are available presently in GPU memory.'},
+      get_table: {
+        ':table':
+          {table: 'The name of the table that has been allocated previously into GPU memory.'}
+      },
+      get_column: {':table': {':column': {table: 'The table name', column: 'The column name'}}},
+      nodes: {
+        returns:
+          'Returns the existing nodes table after applying normalization functions for sigma.js'
+      },
+      nodes: {bounds: {returns: 'Returns the x and y bounds to be used in rendering.'}},
+      edges: {return: 'Returns the existing edges table after applying normalization for sigma.js'}
+    }
+  }
+};
+
+module.exports = schema;
