added pools and mysql pools implementation

Author: Sylvestre Gug

lib/mysql.js

@@ -1,11 +1,26 @@
 import JSONStream from "JSONStream";
 import {json} from "micro";
-import mysql, {createConnection} from "mysql2";
+import mysql, {createConnection, createPool} from "mysql2";
 import {failedCheck} from "./errors.js";
 import {notFound} from "./errors.js";
+import Pools from "./pools.js";
 
 const {Types, ConnectionConfig} = mysql;
 
+export const pools = new Pools(({host, port, database, user, password, ssl}) =>
+  createPool({
+    host,
+    port,
+    database,
+    user,
+    password,
+    ssl: ssl === "required" ? {} : false,
+    connectTimeout: 25e3,
+    connectionLimit: 30,
+    decimalNumbers: true,
+  })
+);
+
 export async function query(req, res, pool) {
   const {sql, params} = await json(req);
   const keepAlive = setInterval(() => res.write("\n"), 25e3);

lib/pools.js

@@ -0,0 +1,55 @@
+import LRU from "lru-cache";
+import * as Sentry from "@sentry/node";
+
+const ttl = 1000 * 60 * 10; // 10m
+
+export default class Pools {
+  constructor(createPool) {
+    this.createPool = createPool;
+    this.cache = new LRU({
+      max: 100,
+      ttl,
+      updateAgeOnGet: true,
+      dispose(_key, pool) {
+        pool.end();
+      },
+    });
+
+    let loop;
+    (loop = () => {
+      this.cache.purgeStale();
+      this.timeout = setTimeout(loop, ttl / 2);
+    })();
+  }
+
+  async get(credentials) {
+    const key = JSON.stringify(credentials);
+    if (this.cache.has(key)) return this.cache.get(key);
+    const pool = await this.createPool(credentials);
+
+    pool.on("error", (error) => {
+      // We need to attach a handler otherwise the process could exit, but we
+      // just don't care about these errors because the client will get cleaned
+      // up already. For debugging purposes, we'll add a Sentry breadcrumb if
+      // something else errors more loudly.
+      Sentry.addBreadcrumb({
+        message: error.message,
+        category: "pool",
+        level: "error",
+        data: error,
+      });
+    });
+
+    this.cache.set(key, pool);
+    return pool;
+  }
+
+  del(credentials) {
+    this.cache.del(JSON.stringify(credentials));
+  }
+
+  end() {
+    if (this.timeout) clearTimeout(this.timeout);
+    for (const pool of this.cache.values()) pool.end();
+  }
+}

package.json

@@ -19,8 +19,10 @@
   },
   "dependencies": {
     "@databricks/sql": "https://github.com/observablehq/databricks-sql-nodejs",
+    "@sentry/node": "^7.33.0",
     "JSONStream": "^1.3.5",
     "ajv": "^8.11.0",
+    "lru-cache": "^7.14.1",
     "micro": "^9.3.4",
     "mssql": "^9.0.1",
     "mysql2": "^3.0.1",

yarn.lock

@@ -1191,6 +1191,41 @@
   resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.2.0.tgz#89ef99401cde6208cff98760b67663726ef26686"
   integrity sha512-0nBr+VZNKm9tvNDZFstI3Pq1fCTEDK5OZTnVKNvBNAKgd0yIvmwsP4m61rEv7ZP+tOUjWJhROpxK5MsnlF911g==
 
+"@sentry/[email protected]":
+  version "7.33.0"
+  resolved "https://registry.yarnpkg.com/@sentry/core/-/core-7.33.0.tgz#7cba1670c041fae02794729b74e9fb9d1f519755"
+  integrity sha512-mrSgUnXjxHVi0cVea1lv7gC/Y66ya2a3atCHaPEij/+l+3APg5d0Ixt1zMx5YllMiZKf6wpxlZ0uwXcqdAAw+w==
+  dependencies:
+    "@sentry/types" "7.33.0"
+    "@sentry/utils" "7.33.0"
+    tslib "^1.9.3"
+
+"@sentry/node@^7.33.0":
+  version "7.33.0"
+  resolved "https://registry.yarnpkg.com/@sentry/node/-/node-7.33.0.tgz#d5c7c7094543dd9819422dfc69952ed40416bfab"
+  integrity sha512-isQVF9LLSG4EZLHiSJ3chgK6f3ZBdGxm8fX6YGm8HWz07CubJddes3yBPLPRNXrRLd7X3SK8pPcK5oc3LIKqAw==
+  dependencies:
+    "@sentry/core" "7.33.0"
+    "@sentry/types" "7.33.0"
+    "@sentry/utils" "7.33.0"
+    cookie "^0.4.1"
+    https-proxy-agent "^5.0.0"
+    lru_map "^0.3.3"
+    tslib "^1.9.3"
+
+"@sentry/[email protected]":
+  version "7.33.0"
+  resolved "https://registry.yarnpkg.com/@sentry/types/-/types-7.33.0.tgz#7d4893a783360a868382e5194b50dbf034ba23c0"
+  integrity sha512-5kkmYjtBWSbPxfYGiXdZFPS6xpFBNlXvDqeX4NpCFXz6/LiEDn6tZ61kuCSFb8MZlyqyCX5WsP3aiI2FJfpGIA==
+
+"@sentry/[email protected]":
+  version "7.33.0"
+  resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-7.33.0.tgz#e6910139328b49b9cc21186521bdb10390dfd915"
+  integrity sha512-msp02GV1gOfaN5FjKjWxI00rtbYLXEE5cTGldhs/Dt9KI63dDk1nwPDkSLhg6joqRItAq0thlBh6un717HdWbg==
+  dependencies:
+    "@sentry/types" "7.33.0"
+    tslib "^1.9.3"
+
 "@sideway/address@^4.1.3":
   version "4.1.4"
   resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.4.tgz#03dccebc6ea47fdc226f7d3d1ad512955d4783f0"
@@ -1975,6 +2010,11 @@ convert-source-map@^1.7.0:
   resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.9.0.tgz#7faae62353fb4213366d0ca98358d22e8368b05f"
   integrity sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==
 
+cookie@^0.4.1:
+  version "0.4.2"
+  resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432"
+  integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA==
+
 copy-descriptor@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d"
@@ -3584,6 +3624,11 @@ lru-cache@^7.14.1:
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.14.1.tgz#8da8d2f5f59827edb388e63e459ac23d6d408fea"
   integrity sha512-ysxwsnTKdAx96aTRdhDOCQfDgbHnt8SK0KY8SEjO0wHinhWOFTESbjVCMPbU1uGXg/ch4lifqx0wfjOawU2+WA==
 
+lru_map@^0.3.3:
+  version "0.3.3"
+  resolved "https://registry.yarnpkg.com/lru_map/-/lru_map-0.3.3.tgz#b5c8351b9464cbd750335a79650a0ec0e56118dd"
+  integrity sha512-Pn9cox5CsMYngeDbmChANltQl+5pi6XmTrraMSzhPmMBbmgcxmqWry0U3PGapCU1yB4/LqCcom7qhHZiF/jGfQ==
+
 make-dir@^1.0.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c"
@@ -5235,6 +5280,11 @@ triple-beam@^1.3.0:
   resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9"
   integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==
 
+tslib@^1.9.3:
+  version "1.14.1"
+  resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
+  integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==
+
 tslib@^2.0.1, tslib@^2.1.0, tslib@^2.2.0:
   version "2.4.0"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3"