Skip to content

Commit eade79c

Browse files
author
Sylvestre
authoredNov 3, 2022
Add MSSQL DB client (#37)
undefined
·
v4.0.0v2.1.0-mssql.0
1 parent 7563330 commit eade79c

15 files changed

+4413
-1089
lines changed
 

‎.babelrc

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"presets": ["@babel/preset-env"]
3+
}

‎.env.test

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
NODE_ENV=test
2+
MSSQL_CREDENTIALS:Server=mssql,1433;Database=master;User Id=sa;Password=Pass@word;trustServerCertificate=true;
3+
MSSQL_CREDENTIALS_READ_ONLY:Server=mssql,1433;Database=master;User Id=reader;Password=re@derP@ssw0rd;trustServerCertificate=true;

‎.env.test.js

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
2+
export const MSSQL_CREDENTIALS = env("MSSQL_CREDENTIALS");
3+
export const MSSQL_CREDENTIALS_READ_ONLY = env("MSSQL_CREDENTIALS_READ_ONLY");
4+
export const NODE_ENV = env("NODE_ENV");
5+
6+
function env(key, defaultValue) {
7+
const value = process.env[key]; // eslint-disable-line no-process-env
8+
if (value !== undefined) return value;
9+
if (defaultValue !== undefined) return defaultValue;
10+
throw new Error(`Missing environment variable: ${key}`);
11+
}

‎.mocharc.cjs

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
module.exports = {
2+
color: true,
3+
extension: ["js"],
4+
global: [],
5+
ignore: [],
6+
require: "@babel/register",
7+
spec: ["test/**/*.test.js"],
8+
};

‎Dockerfile

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
FROM node:16.17.0-alpine
2+
3+
RUN mkdir /app
4+
WORKDIR /app
5+
6+
RUN apk --no-cache add bash
7+
8+
COPY package.json yarn.lock /app/
9+
RUN \
10+
yarn --frozen-lockfile && \
11+
yarn cache clean
12+
13+
ENV PATH="/app/node_modules/.bin:${PATH}"
14+
15+
COPY . /app/
16+
17+
CMD yarn test

‎data/AdventureWorks2019.bak

8.12 MB
Binary file not shown.

‎data/seed.mssql.js

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import mssql from "mssql";
2+
import {MSSQL_CREDENTIALS} from "../.env.test.js";
3+
4+
const credentials = MSSQL_CREDENTIALS;
5+
6+
const seed = async () => {
7+
await mssql.connect(credentials);
8+
9+
await mssql.query`RESTORE DATABASE test
10+
FROM DISK = '/var/opt/mssql/backup/test.bak'
11+
WITH REPLACE, RECOVERY,
12+
MOVE 'AdventureWorksLT2012_Data' TO '/var/opt/mssql/data/aw2019.mdf',
13+
MOVE 'AdventureWorksLT2012_Log' TO '/var/opt/mssql/data/aw2019.ldf';`;
14+
15+
await mssql.query`IF NOT EXISTS(SELECT name
16+
FROM sys.syslogins
17+
WHERE name='reader')
18+
BEGIN
19+
CREATE LOGIN reader WITH PASSWORD = 're@derP@ssw0rd'
20+
CREATE USER reader FOR LOGIN reader
21+
END`;
22+
};
23+
24+
seed()
25+
.then(() => {
26+
console.log(`MS_SQL DB seeded.`);
27+
process.exit(0);
28+
})
29+
.catch((err) => {
30+
console.error(err.message, err);
31+
process.exit(1);
32+
});

‎docker-compose.local.yml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
version: "3.7"
2+
3+
services:
4+
mssql:
5+
image: mcr.microsoft.com/azure-sql-edge
6+
expose:
7+
- "1433"

‎docker-compose.yml

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
version: "3.7"
2+
3+
services:
4+
test:
5+
build: .
6+
depends_on:
7+
- mssql
8+
env_file:
9+
- .env.test
10+
networks:
11+
- db_proxy_test
12+
command: sh -c "set -o pipefail && wait-on -d 10000 -t 30000 tcp:mssql:1433 && node ./data/seed.mssql.js && TZ=UTC NODE_ENV=TEST node_modules/.bin/mocha"
13+
14+
mssql:
15+
image: mcr.microsoft.com/mssql/server:2019-latest
16+
environment:
17+
- MSSQL_SA_PASSWORD=Pass@word
18+
- ACCEPT_EULA=Y
19+
- MSSQL_DATABASE=test
20+
- MSSQL_SLEEP=7
21+
volumes:
22+
- ./data/AdventureWorks2019.bak:/var/opt/mssql/backup/test.bak
23+
ports:
24+
- "1433:1433"
25+
networks:
26+
- db_proxy_test
27+
28+
networks:
29+
db_proxy_test:
30+
name: db_proxy_test

‎lib/errors.js

+9-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,13 @@
11
import {createError} from "micro";
2-
export const unauthorized = error => createError(401, "Unauthorized", error);
3-
export const notFound = error => createError(404, "Not Found", error);
4-
export const exit = message => {
2+
export const unauthorized = (error) => createError(401, "Unauthorized", error);
3+
export const notFound = (error) => createError(404, "Not Found", error);
4+
export const notImplemented = (error) =>
5+
createError(501, "Not Implemented", error);
6+
export const badRequest = (error) =>
7+
createError(400, typeof error === "string" ? error : "Bad request", error);
8+
export const failedCheck = (error) =>
9+
createError(200, typeof error === "string" ? error : "Failed check", error);
10+
export const exit = (message) => {
511
console.error(message); // eslint-disable-line no-console
612
process.exit(1);
713
};

‎lib/mssql.js

+205
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
import Ajv from "ajv";
2+
import JSONStream from "JSONStream";
3+
import {json} from "micro";
4+
import mssql from "mssql";
5+
import {failedCheck, badRequest, notImplemented} from "./errors.js";
6+
7+
const TYPES = mssql.TYPES;
8+
9+
const pools = new Map();
10+
11+
const READ_ONLY = new Set(["SELECT", "USAGE"]);
12+
13+
const ajv = new Ajv();
14+
const validate = ajv.compile({
15+
type: "object",
16+
additionalProperties: false,
17+
required: ["sql"],
18+
properties: {
19+
sql: {type: "string", minLength: 1},
20+
params: {type: "array"},
21+
},
22+
});
23+
24+
// See: https://tediousjs.github.io/node-mssql/#connection-pools
25+
export const mssqlPool = {
26+
get: (name, config) => {
27+
if (!pools.has(name)) {
28+
if (!config) {
29+
throw new Error("Database configuration required");
30+
}
31+
32+
const pool = new mssql.ConnectionPool(config);
33+
const close = pool.close.bind(pool);
34+
pool.close = (...args) => {
35+
pools.delete(name);
36+
return close(...args);
37+
};
38+
39+
pools.set(name, pool.connect());
40+
}
41+
42+
return pools.get(name);
43+
},
44+
45+
closeAll: () =>
46+
Promise.all(
47+
Array.from(pools.values()).map((connect) => {
48+
return connect.then((pool) => pool.close());
49+
})
50+
),
51+
};
52+
53+
export async function queryStream(req, res, pool) {
54+
const db = await pool;
55+
const body = await json(req);
56+
57+
if (!validate(body)) throw badRequest();
58+
59+
res.setHeader("Content-Type", "text/plain");
60+
const keepAlive = setInterval(() => res.write("\n"), 25e3);
61+
62+
let {sql, params = []} = body;
63+
64+
try {
65+
await new Promise((resolve, reject) => {
66+
const request = new mssql.Request(db);
67+
const stream = request.toReadableStream();
68+
69+
params.forEach((param, idx) => {
70+
request.input(`${idx + 1}`, param);
71+
});
72+
73+
request.query(sql);
74+
request.once("recordset", () => clearInterval(keepAlive));
75+
request.on("recordset", (columns) => {
76+
const schema = {
77+
type: "array",
78+
items: {
79+
type: "object",
80+
properties: Object.entries(columns).reduce(
81+
(schema, [name, props]) => {
82+
return {
83+
...schema,
84+
...{[name]: dataTypeSchema({type: props.type.name})},
85+
};
86+
},
87+
{}
88+
),
89+
},
90+
};
91+
92+
res.write(`${JSON.stringify(schema)}`);
93+
res.write("\n");
94+
});
95+
96+
stream.pipe(JSONStream.stringify("", "\n", "\n")).pipe(res);
97+
stream.on("done", () => {
98+
resolve();
99+
});
100+
stream.on("error", (error) => {
101+
if (!request.canceled) {
102+
request.cancel();
103+
}
104+
reject(error);
105+
});
106+
});
107+
} catch (error) {
108+
if (!error.statusCode) error.statusCode = 400;
109+
throw error;
110+
} finally {
111+
clearInterval(keepAlive);
112+
}
113+
114+
res.end();
115+
}
116+
117+
/*
118+
* This function is checking for the permission of the given credentials. It alerts the user setting
119+
* them up that these may be too permissive.
120+
* */
121+
export async function check(req, res, pool) {
122+
const db = await pool;
123+
124+
// See: https://learn.microsoft.com/en-us/sql/relational-databases/system-functions/sys-fn-my-permissions-transact-sql
125+
const rows = await db.request().query(
126+
`USE ${db.config.database};
127+
SELECT * FROM fn_my_permissions (NULL, 'DATABASE');`
128+
);
129+
130+
const grants = rows.recordset.map((rs) => rs.permission_name);
131+
const permissive = grants.filter((g) => !READ_ONLY.has(g));
132+
133+
if (permissive.length)
134+
throw failedCheck(
135+
`User has too permissive grants: ${permissive.join(", ")}`
136+
);
137+
138+
return {ok: true};
139+
}
140+
141+
export default (credentials) => async (req, res) => {
142+
const pool = mssqlPool.get(JSON.stringify(credentials), credentials);
143+
144+
if (req.method === "POST") {
145+
if (req.url === "/check") {
146+
return check(req, res, pool);
147+
}
148+
149+
if (["/query-stream"].includes(req.url)) {
150+
return queryStream(req, res, pool);
151+
}
152+
153+
throw notImplemented();
154+
}
155+
};
156+
157+
// See https://github.com/tediousjs/node-mssql/blob/66587d97c9ce21bffba8ca360c72a540f2bc47a6/lib/datatypes.js#L6
158+
const boolean = ["null", "boolean"],
159+
integer = ["null", "integer"],
160+
number = ["null", "number"],
161+
object = ["null", "object"],
162+
string = ["null", "string"];
163+
export function dataTypeSchema({type}) {
164+
switch (type) {
165+
case TYPES.Bit.name:
166+
return {type: boolean};
167+
case TYPES.TinyInt.name:
168+
return {type: integer, tiny: true};
169+
case TYPES.SmallInt.name:
170+
return {type: integer, short: true};
171+
case TYPES.BigInt.name:
172+
return {type: integer, long: true};
173+
case TYPES.Int.name:
174+
return {type: integer};
175+
case TYPES.Float.name:
176+
return {type: number, float: true};
177+
case TYPES.Numeric.name:
178+
return {type: number};
179+
case TYPES.Decimal.name:
180+
return {type: number, decimal: true};
181+
case TYPES.Real.name:
182+
return {type: number};
183+
case TYPES.Date.name:
184+
case TYPES.DateTime.name:
185+
case TYPES.DateTime2.name:
186+
case TYPES.DateTimeOffset.name:
187+
case TYPES.SmallDateTime.name:
188+
case TYPES.Time.name:
189+
return {type: string, date: true};
190+
case TYPES.Binary.name:
191+
case TYPES.VarBinary.name:
192+
case TYPES.Image.name:
193+
return {type: object, buffer: true};
194+
case TYPES.SmallMoney.name: // TODO
195+
case TYPES.Money.name: //TODO
196+
case TYPES.Xml.name: //TODO
197+
case TYPES.TVP.name: //TODO
198+
case TYPES.UDT.name: //TODO
199+
case TYPES.Geography.name: //TODO
200+
case TYPES.Geometry.name: //TODO
201+
case TYPES.Variant.name: //TODO
202+
default:
203+
return {type: string};
204+
}
205+
}

‎lib/server.js

+5-2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import {notFound, unauthorized, exit} from "./errors.js";
1111
import mysql from "./mysql.js";
1212
import postgres from "./postgres.js";
1313
import snowflake from "./snowflake.js";
14+
import mssql from "./mssql.js";
1415

1516
export function server(config, argv) {
1617
const development = process.env.NODE_ENV === "development";
@@ -22,7 +23,7 @@ export function server(config, argv) {
2223
url,
2324
ssl = "disabled",
2425
host = "127.0.0.1",
25-
port = 2899
26+
port = 2899,
2627
} = config;
2728

2829
const handler =
@@ -32,6 +33,8 @@ export function server(config, argv) {
3233
? postgres(url)
3334
: type === "snowflake"
3435
? snowflake(url)
36+
: type === "mssql"
37+
? mssql(url)
3538
: null;
3639
if (!handler) {
3740
return exit(`Unknown database type: ${type}`);
@@ -85,7 +88,7 @@ export function server(config, argv) {
8588

8689
const [payload, hmac] = authorization
8790
.split(".")
88-
.map(encoded => Buffer.from(encoded, "base64"));
91+
.map((encoded) => Buffer.from(encoded, "base64"));
8992
const {name} = JSON.parse(payload);
9093
if (config.name !== name) throw notFound();
9194
const {origin, secret} = config;

‎package.json

+22-8
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,23 @@
33
"description": "A local proxy to connect private Observable notebooks to private databases",
44
"version": "2.0.0",
55
"type": "module",
6-
"engines" : {
7-
"node" : ">=14.19.0"
6+
"engines": {
7+
"node": ">=14.19.0"
88
},
9-
"exports":{
10-
"./postgres.js": "./lib/postgres.js",
11-
"./mysql.js": "./lib/mysql.js",
12-
"./snowflake.js": "./lib/snowflake.js"
9+
"exports": {
10+
"./postgres.js": "./lib/postgres.js",
11+
"./mysql.js": "./lib/mysql.js",
12+
"./snowflake.js": "./lib/snowflake.js",
13+
"./mssql.js": "./lib/mssql.js"
1314
},
1415
"bin": {
1516
"observable-database-proxy": "./bin/observable-database-proxy.js"
1617
},
1718
"dependencies": {
1819
"JSONStream": "^1.3.5",
20+
"ajv": "^8.11.0",
1921
"micro": "^9.3.4",
22+
"mssql": "^9.0.1",
2023
"mysql": "^2.17.1",
2124
"open": "^6.3.0",
2225
"pg": "^8.7.1",
@@ -26,11 +29,22 @@
2629
"yargs": "^13.2.4"
2730
},
2831
"devDependencies": {
29-
"nodemon": "^1.19.1"
32+
"@babel/core": "^7.19.6",
33+
"@babel/preset-env": "^7.19.4",
34+
"@babel/register": "^7.18.9",
35+
"chai": "^4.3.6",
36+
"mocha": "^10.1.0",
37+
"mock-req": "^0.2.0",
38+
"mock-res": "^0.6.0",
39+
"nodemon": "^1.19.1",
40+
"wait-on": "^6.0.1"
3041
},
3142
"scripts": {
3243
"dev": "NODE_ENV=development nodemon bin/observable-database-proxy.js",
33-
"test": "echo \"Error: no test specified\" && exit 1"
44+
"test": "mocha",
45+
"test:local": "docker-compose -f docker-compose.yml -f docker-compose.local.yml up --build",
46+
"test:ci": "docker-compose -f docker-compose.yml up --build --exit-code-from test",
47+
"test:db": "docker-compose -f docker-compose.yml -f docker-compose.local.yml up mssql"
3448
},
3549
"author": "Observable",
3650
"license": "ISC",

‎test/mssql.test.js

+135
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import {expect} from "chai";
2+
import MockReq from "mock-req";
3+
import MockRes from "mock-res";
4+
5+
import {MSSQL_CREDENTIALS} from "../.env.test.js";
6+
import mssql, {dataTypeSchema} from "../lib/mssql.js";
7+
8+
const credentials = MSSQL_CREDENTIALS;
9+
describe("mssql", () => {
10+
describe("when checking", () => {
11+
describe("with system admin user", () => {
12+
it("should throw a too permissive error", () => {
13+
const req = new MockReq({
14+
method: "POST",
15+
url: "/check",
16+
});
17+
const res = new MockRes();
18+
const index = mssql(credentials);
19+
20+
return index(req, res).then(
21+
() => Promise.reject("Expect call to throw!"),
22+
(err) => {
23+
expect(err.statusCode).to.equal(200);
24+
expect(
25+
err.message.includes("User has too permissive grants")
26+
).to.equal(true);
27+
}
28+
);
29+
});
30+
});
31+
});
32+
33+
describe("when querying", () => {
34+
it("should stream the results of simple query", () => {
35+
return new Promise(async (resolve, reject) => {
36+
const req = new MockReq({method: "POST", url: "/query-stream"}).end({
37+
sql: "SELECT TOP 2 CustomerID FROM test.SalesLT.Customer",
38+
params: [],
39+
});
40+
41+
const res = new MockRes(onEnd);
42+
43+
const index = mssql(credentials);
44+
await index(req, res);
45+
46+
function onEnd() {
47+
const [schema, row] = this._getString().split("\n");
48+
49+
expect(schema).to.equal(
50+
JSON.stringify({
51+
type: "array",
52+
items: {
53+
type: "object",
54+
properties: {CustomerID: {type: ["null", "integer"]}},
55+
},
56+
})
57+
);
58+
expect(row).to.equal(JSON.stringify({CustomerID: 12}));
59+
60+
resolve();
61+
}
62+
});
63+
});
64+
it("should handle parameter graciously", () => {
65+
return new Promise(async (resolve, reject) => {
66+
const testCustomerId = 3;
67+
const req = new MockReq({method: "POST", url: "/query-stream"}).end({
68+
sql: "SELECT TOP 2 CustomerID FROM test.SalesLT.Customer WHERE CustomerID=@1",
69+
params: [testCustomerId],
70+
});
71+
72+
const res = new MockRes(onEnd);
73+
74+
const index = mssql(credentials);
75+
await index(req, res);
76+
77+
function onEnd() {
78+
const [schema, row] = this._getString().split("\n");
79+
80+
expect(schema).to.equal(
81+
JSON.stringify({
82+
type: "array",
83+
items: {
84+
type: "object",
85+
properties: {CustomerID: {type: ["null", "integer"]}},
86+
},
87+
})
88+
);
89+
expect(row).to.equal(JSON.stringify({CustomerID: testCustomerId}));
90+
91+
resolve();
92+
}
93+
});
94+
});
95+
it("should replace cell reference in the SQL query", () => {
96+
return new Promise(async (resolve, reject) => {
97+
const testCustomerId = 5;
98+
const req = new MockReq({method: "POST", url: "/query-stream"}).end({
99+
sql: "SELECT TOP 2 CustomerID FROM test.SalesLT.Customer WHERE CustomerID=@1",
100+
params: [testCustomerId],
101+
});
102+
103+
const res = new MockRes(onEnd);
104+
105+
const index = mssql(credentials);
106+
await index(req, res);
107+
108+
function onEnd() {
109+
const [schema, row] = this._getString().split("\n");
110+
111+
expect(schema).to.equal(
112+
JSON.stringify({
113+
type: "array",
114+
items: {
115+
type: "object",
116+
properties: {CustomerID: {type: ["null", "integer"]}},
117+
},
118+
})
119+
);
120+
expect(row).to.equal(JSON.stringify({CustomerID: testCustomerId}));
121+
122+
resolve();
123+
}
124+
});
125+
});
126+
});
127+
128+
describe("when check the dataTypeSchema", () => {
129+
it("should TYPES.Image.name to object", () => {
130+
const {type} = dataTypeSchema({type: "Image"});
131+
expect(type[0]).to.equal("null");
132+
expect(type[1]).to.equal("object");
133+
});
134+
});
135+
});

‎yarn.lock

+3,926-1,076
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)
Please sign in to comment.