Data-connector and Database-proxy reconciliations

.env.test

@@ -1,3 +1,4 @@
 NODE_ENV=test
-MSSQL_CREDENTIALS:Server=mssql,1433;Database=master;User Id=sa;Password=Pass@word;trustServerCertificate=true;
-MSSQL_CREDENTIALS_READ_ONLY:Server=mssql,1433;Database=master;User Id=reader;Password=re@derP@ssw0rd;trustServerCertificate=true;
+MYSQL_TEST_CREDENTIALS=mysql://root@mysql:3306/mysql?sslMode=DISABLED
+MSSQL_TEST_CREDENTIALS=Server=mssql,1433;Database=master;User Id=sa;Password=Pass@word;trustServerCertificate=true;
+POSTGRES_TEST_CREDENTIALS=postgres://postgres@postgres:5432/postgres?sslmode=disable

.env.test.js

@@ -1,4 +1,7 @@
-export const MSSQL_CREDENTIALS = env("MSSQL_CREDENTIALS");
+export const MSSQL_TEST_CREDENTIALS = env("MSSQL_TEST_CREDENTIALS");
+export const MYSQL_TEST_CREDENTIALS = env("MYSQL_TEST_CREDENTIALS");
+export const POSTGRES_TEST_CREDENTIALS = env("POSTGRES_TEST_CREDENTIALS");
+export const SNOWFLAKE_TEST_CREDENTIALS = env("SNOWFLAKE_TEST_CREDENTIALS");
 export const NODE_ENV = env("NODE_ENV");
 
 function env(key, defaultValue) {

.eslintrc.json

@@ -1,7 +1,7 @@
 {
   "parserOptions": {
     "sourceType": "module",
-    "ecmaVersion": 2018
+    "ecmaVersion": 2022
   },
   "env": {
     "node": true,
@@ -18,7 +18,7 @@
     {
       "files": ["*.test.js"],
       "env": {
-        "jest": true
+        "mocha": true
       }
     }
   ],

.github/workflows/test.yml

@@ -0,0 +1,41 @@
+name: Test
+on: push
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    defaults:
+      run:
+        working-directory: .
+    env:
+      DOCKER_PACKAGE: ghcr.io/${{ github.repository }}/database-proxy_test
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Docker login
+        run: echo ${GITHUB_TOKEN} | docker login -u ${GITHUB_ACTOR} --password-stdin ghcr.io
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Republish
+        id: republish
+        continue-on-error: true
+        if: ${{ needs.Changes.outputs.connector == 'false' }}
+        run: |
+          ../.github/retry docker pull ${DOCKER_PACKAGE}:${{ github.event.before }}
+          docker tag ${DOCKER_PACKAGE}:${{ github.event.before }} ${DOCKER_PACKAGE}:${GITHUB_SHA}
+          ../.github/retry docker push ${DOCKER_PACKAGE}:${GITHUB_SHA}
+
+      - name: Build
+        if: ${{ steps.republish.outcome != 'success' }}
+        run: docker-compose build
+      - name: Lint
+        if: ${{ steps.republish.outcome != 'success' }}
+        run: docker-compose run lint
+      - name: Test
+        if: ${{ steps.republish.outcome != 'success' }}
+        run: docker-compose run test
+        env:
+          SNOWFLAKE_TEST_CREDENTIALS: ${{ secrets.SNOWFLAKE_TEST_CREDENTIALS }}
+      - name: Container logs
+        if: failure()
+        run: docker-compose logs --no-color --timestamps

.gitignore

@@ -1,2 +1,4 @@
 node_modules
 ssl/localhost.csr
+
+*.secret

Dockerfile

@@ -1,5 +1,7 @@
 FROM node:18.12.1-alpine
 
+RUN apk update && apk --no-cache add git
+
 RUN mkdir /app
 WORKDIR /app
 

data/seed.mssql.js

@@ -1,7 +1,7 @@
 import mssql from "mssql";
-import {MSSQL_CREDENTIALS} from "../.env.test.js";
+import {MSSQL_TEST_CREDENTIALS} from "../.env.test.js";
 
-const credentials = MSSQL_CREDENTIALS;
+const credentials = MSSQL_TEST_CREDENTIALS;
 
 const seed = async () => {
   await mssql.connect(credentials);

docker-compose.local.yml

@@ -1,7 +1,19 @@
 version: "3.7"
 
 services:
+  test:
+    env_file:
+      - .env.secret
+
   mssql:
     image: mcr.microsoft.com/azure-sql-edge
-    expose:
-      - "1433"
+    ports:
+      - "1433:1433"
+
+  mysql:
+    ports:
+      - "3306:3306"
+
+  postgres:
+    ports:
+      - "5432:5432"

docker-compose.yml

@@ -1,15 +1,23 @@
 version: "3.7"
 
 services:
+  lint:
+    build: .
+    command: eslint .
+
   test:
     build: .
     depends_on:
       - mssql
+      - mysql
+      - postgres
     env_file:
       - .env.test
+    environment:
+      - SNOWFLAKE_TEST_CREDENTIALS
     networks:
       - db_proxy_test
-    command: sh -c "set -o pipefail && wait-on -d 10000 -t 30000 tcp:mssql:1433 && node ./data/seed.mssql.js && TZ=UTC NODE_ENV=TEST node_modules/.bin/mocha"
+    command: sh -c "set -o pipefail && wait-on -d 15000 -t 30000 tcp:mysql:3306 tcp:mssql:1433 tcp:postgres:5432 && node ./data/seed.mssql.js && TZ=UTC NODE_ENV=TEST node_modules/.bin/mocha --exit"
 
   mssql:
     image: mcr.microsoft.com/mssql/server:2019-latest
@@ -20,8 +28,20 @@ services:
       - MSSQL_SLEEP=7
     volumes:
       - ./data/AdventureWorks2019.bak:/var/opt/mssql/backup/test.bak
-    ports:
-      - "1433:1433"
+    networks:
+      - db_proxy_test
+
+  mysql:
+    image: mariadb:10.6.4
+    environment:
+      - MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=yes
+    networks:
+      - db_proxy_test
+
+  postgres:
+    image: postgres:13.8-alpine3.16
+    environment:
+      - POSTGRES_HOST_AUTH_METHOD=trust
     networks:
       - db_proxy_test
 

lib/databricks.js

@@ -253,14 +253,14 @@ export async function queryStream(req, res, connection) {
     res.write(`${JSON.stringify(responseSchema)}`);
     res.write("\n");
 
-    await new Promise(async (resolve, reject) => {
+    await new Promise((resolve, reject) => {
       const stream = new Readable.from(rows);
 
       stream.once("data", () => {
         clearInterval(keepAlive);
       });
 
-      stream.on("close", (error) => {
+      stream.on("close", () => {
         resolve();
         stream.destroy();
       });
@@ -345,8 +345,6 @@ export async function check(req, res, connection) {
     });
 
     return {ok: true};
-  } catch (e) {
-    throw e;
   } finally {
     if (connection) {
       try {

lib/mssql.js

@@ -5,6 +5,7 @@ import {Transform} from "stream";
 
 import {failedCheck, badRequest, notImplemented} from "./errors.js";
 import {validateQueryPayload} from "./validate.js";
+import Pools from "./pools.js";
 
 const TYPES = mssql.TYPES;
 const READ_ONLY = new Set(["SELECT", "USAGE", "CONNECT"]);
@@ -124,8 +125,6 @@ export async function check(req, res, pool) {
   return {ok: true};
 }
 
-export const ConnectionPool = mssql.ConnectionPool;
-
 export default (credentials) => {
   const pool = new mssql.ConnectionPool(credentials);
 
@@ -144,6 +143,14 @@ export default (credentials) => {
   };
 };
 
+export const pools = new Pools((credentials) =>
+  Object.defineProperty(new mssql.ConnectionPool(credentials), "end", {
+    value() {
+      this.close();
+    },
+  })
+);
+
 // See https://github.com/tediousjs/node-mssql/blob/66587d97c9ce21bffba8ca360c72a540f2bc47a6/lib/datatypes.js#L6
 const boolean = ["null", "boolean"],
   integer = ["null", "integer"],

lib/mysql.js

@@ -1,11 +1,26 @@
 import JSONStream from "JSONStream";
 import {json} from "micro";
-import mysql, {createConnection} from "mysql2";
+import mysql, {createConnection, createPool} from "mysql2";
 import {failedCheck} from "./errors.js";
 import {notFound} from "./errors.js";
+import Pools from "./pools.js";
 
 const {Types, ConnectionConfig} = mysql;
 
+export const pools = new Pools(({host, port, database, user, password, ssl}) =>
+  createPool({
+    host,
+    port,
+    database,
+    user,
+    password,
+    ssl: ssl === "required" ? {} : false,
+    connectTimeout: 25e3,
+    connectionLimit: 30,
+    decimalNumbers: true,
+  })
+);
+
 export async function query(req, res, pool) {
   const {sql, params} = await json(req);
   const keepAlive = setInterval(() => res.write("\n"), 25e3);

lib/oracle.js

@@ -4,6 +4,7 @@ import {Transform} from "stream";
 
 import {badRequest, failedCheck} from "./errors.js";
 import {validateQueryPayload} from "./validate.js";
+import Pools from "./pools.js";
 
 const READ_ONLY = new Set(["SELECT", "USAGE", "CONNECT"]);
 export class OracleSingleton {
@@ -204,8 +205,6 @@ export async function check(req, res, pool) {
       );
 
     return {ok: true};
-  } catch (e) {
-    throw e;
   } finally {
     if (connection) {
       try {
@@ -217,13 +216,30 @@ export async function check(req, res, pool) {
   }
 }
 
+export const pools = new Pools(async (credentials) => {
+  const oracledb = await OracleSingleton.getInstance();
+  credentials.connectionString = decodeURI(credentials.connectionString);
+  const pool = await oracledb.createPool(credentials);
+
+  Object.defineProperty(pool, "end", {
+    value() {
+      // We must ensure there is no query still running before we close the pool.
+      if (this._connectionsOut === 0) {
+        this.close();
+      }
+    },
+  });
+
+  return pool;
+});
+
 export default async ({url, username, password}) => {
   OracleSingleton.initialize();
   // We do not want to import the oracledb library until we are sure that the user is looking to use Oracle.
   // Installing the oracledb library is a pain, so we want to avoid it if possible.
   const config = {
-    username: username,
-    password: password,
+    username,
+    password,
     connectionString: decodeURI(url),
   };
 

lib/pools.js

@@ -0,0 +1,55 @@
+import LRU from "lru-cache";
+import * as Sentry from "@sentry/node";
+
+const maxAge = 1000 * 60 * 10; // 10m
+
+export default class Pools {
+  constructor(createPool) {
+    this.createPool = createPool;
+    this.cache = new LRU({
+      max: 100,
+      maxAge,
+      updateAgeOnGet: true,
+      dispose(_key, pool) {
+        pool.end();
+      },
+    });
+
+    let loop;
+    (loop = () => {
+      this.cache.prune();
+      this.timeout = setTimeout(loop, maxAge / 2);
+    })();
+  }
+
+  async get(credentials) {
+    const key = JSON.stringify(credentials);
+    if (this.cache.has(key)) return this.cache.get(key);
+    const pool = await this.createPool(credentials);
+
+    pool.on("error", (error) => {
+      // We need to attach a handler otherwise the process could exit, but we
+      // just don't care about these errors because the client will get cleaned
+      // up already. For debugging purposes, we'll add a Sentry breadcrumb if
+      // something else errors more loudly.
+      Sentry.addBreadcrumb({
+        message: error.message,
+        category: "pool",
+        level: "error",
+        data: error,
+      });
+    });
+
+    this.cache.set(key, pool);
+    return pool;
+  }
+
+  del(credentials) {
+    this.cache.del(JSON.stringify(credentials));
+  }
+
+  end() {
+    if (this.timeout) clearTimeout(this.timeout);
+    for (const pool of this.cache.values()) pool.end();
+  }
+}