Skip to content

Commit 1613450

Browse files
Sylvestremootari
andauthored
Oracle DB client (#46)
* centralized pool of pools * exports pools * make it driver agnostic, use callback to setup pool into pools * use controller to abort all * Apply suggestions from code review Co-authored-by: Fabian Iwand <[email protected]> * no need to monkey patch close * keyOf + key in closure * mssql signal to handle closing and deleting * Update lib/mssql.js Co-authored-by: Fabian Iwand <[email protected]> * install oracledb * basic oracle routing * WIP * WIP * oracle check method * add queryStream sql query * basic streaming test * sql query stream pipe * tweaked integer mapping * cleaned up unused var * singleton initialized + bind params syntax friendly * adjustments to pool * oracle db dep * oracledb to peer dep * update yarn lock * oracle pools setup * clean up logs * remove pool of pools for mssql * imports * await for the connection * remove pools from exports * bump node version for Dockerfile * no pool to test * no pools of pool * adjust oracle to only import if used * add a start command * cleanup + comment * comments * fixed oraclesingletong invokation * typo * fix BLOB schema mapping * use same payload validation * refacotred the OracleSingleton initialize step to not be async Co-authored-by: Fabian Iwand <[email protected]>
1 parent 1a54c62 commit 1613450

File tree

6 files changed

+256
-16
lines changed

6 files changed

+256
-16
lines changed

lib/config.js

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,18 @@ export function readDecodedConfig(name) {
2121
if (name) {
2222
const raw = config && config[name];
2323
if (!raw) exit(`No configuration found for "${name}"`);
24-
return {...decodeSecret(raw.secret), url: raw.url};
24+
return {
25+
...decodeSecret(raw.secret),
26+
url: raw.url,
27+
username: raw.username || "", // oracle client requires username
28+
password: raw.password || "", // oracle client requires password
29+
};
2530
} else {
2631
return Object.values(config).map((c) => ({
2732
...decodeSecret(c.secret),
2833
url: c.url,
34+
username: c.username || "", // oracle client requires username
35+
password: c.password || "", // oracle client requires password
2936
}));
3037
}
3138
}

lib/mssql.js

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,21 @@
1-
import Ajv from "ajv";
21
import JSONStream from "JSONStream";
32
import {json} from "micro";
43
import mssql from "mssql";
54
import {Transform} from "stream";
65

76
import {failedCheck, badRequest, notImplemented} from "./errors.js";
7+
import {validateQueryPayload} from "./validate.js";
88

99
const TYPES = mssql.TYPES;
1010
const READ_ONLY = new Set(["SELECT", "USAGE", "CONNECT"]);
1111

12-
const ajv = new Ajv();
13-
const validate = ajv.compile({
14-
type: "object",
15-
additionalProperties: false,
16-
required: ["sql"],
17-
properties: {
18-
sql: {type: "string", minLength: 1},
19-
params: {type: "array"},
20-
},
21-
});
22-
2312
export async function queryStream(req, res, pool) {
2413
const connection = await pool;
2514
const db = await connection.connect();
2615

2716
const body = await json(req);
2817

29-
if (!validate(body)) throw badRequest();
18+
if (!validateQueryPayload(body)) throw badRequest();
3019

3120
res.setHeader("Content-Type", "text/plain");
3221
const keepAlive = setInterval(() => res.write("\n"), 25e3);

lib/oracle.js

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
import {json} from "micro";
2+
import JSONStream from "JSONStream";
3+
import {Transform} from "stream";
4+
5+
import {badRequest, failedCheck} from "./errors.js";
6+
import {validateQueryPayload} from "./validate.js";
7+
8+
const READ_ONLY = new Set(["SELECT", "USAGE", "CONNECT"]);
9+
10+
export class OracleSingleton {
11+
static instance = null;
12+
constructor() {
13+
throw new Error(
14+
"Do not use new OracleSingleton(). Call OracleSingleton.initialize() instead."
15+
);
16+
}
17+
static initialize() {
18+
if (!OracleSingleton.instance) {
19+
try {
20+
OracleSingleton.instance = import("oracledb").then((module) => {
21+
const oracledb = module.default;
22+
oracledb.initOracleClient({
23+
libDir: process.env.LIB_DIR_PATH,
24+
});
25+
return oracledb;
26+
});
27+
} catch (err) {
28+
console.error(err);
29+
}
30+
}
31+
}
32+
static getInstance() {
33+
if (!OracleSingleton.instance)
34+
throw new Error(
35+
"OracleSingleton not initialized. Call OracleSingleton.initialize() first."
36+
);
37+
return OracleSingleton.instance;
38+
}
39+
}
40+
41+
export async function queryStream(req, res, pool) {
42+
const db = await pool;
43+
const connection = await db.getConnection();
44+
const body = await json(req);
45+
46+
if (!validateQueryPayload(body)) throw badRequest();
47+
48+
res.setHeader("Content-Type", "text/plain");
49+
const keepAlive = setInterval(() => res.write("\n"), 25e3);
50+
51+
let {sql, params = []} = body;
52+
53+
try {
54+
await new Promise((resolve, reject) => {
55+
const columnNameMap = new Map();
56+
const stream = connection.queryStream(sql, params, {
57+
extendedMetaData: true,
58+
});
59+
60+
stream
61+
.on("error", function (e) {
62+
this.destroy();
63+
reject(e);
64+
})
65+
.on("metadata", (columns) => {
66+
clearInterval(keepAlive);
67+
68+
const schema = {
69+
type: "array",
70+
items: {
71+
type: "object",
72+
properties: columns.reduce((schema, {dbType, name}, idx) => {
73+
columnNameMap.set(idx, name);
74+
return {
75+
...schema,
76+
...{[name]: dataTypeSchema({type: dbType})},
77+
};
78+
}, {}),
79+
},
80+
};
81+
res.write(`${JSON.stringify(schema)}`);
82+
res.write("\n");
83+
})
84+
.on("end", function () {
85+
this.destroy();
86+
})
87+
.on("close", resolve)
88+
.pipe(
89+
new Transform({
90+
objectMode: true,
91+
transform(chunk, encoding, cb) {
92+
let row = null;
93+
try {
94+
row = chunk.reduce((acc, r, idx) => {
95+
const key = columnNameMap.get(idx);
96+
return {...acc, [key]: r};
97+
}, {});
98+
} catch (e) {
99+
console.error("row has unexpected format");
100+
// TODO: Add error handling once server supports handling error for in flight streamed response
101+
// cb(new Error(e));
102+
}
103+
cb(null, row);
104+
},
105+
})
106+
)
107+
.pipe(JSONStream.stringify("", "\n", "\n"))
108+
.pipe(res);
109+
});
110+
} catch (error) {
111+
if (!error.statusCode) error.statusCode = 400;
112+
throw error;
113+
} finally {
114+
clearInterval(keepAlive);
115+
if (connection) {
116+
try {
117+
await connection.close();
118+
} catch (err) {
119+
console.error(err);
120+
}
121+
}
122+
}
123+
124+
res.end();
125+
}
126+
127+
/*
128+
* This function is checking for the permission of the given credentials. It alerts the user setting
129+
* them up that these may be too permissive.
130+
* */
131+
export async function check(req, res, pool) {
132+
let connection;
133+
try {
134+
const db = await pool;
135+
connection = await db.getConnection();
136+
137+
// see: https://docs.oracle.com/en/database/oracle/oracle-database/12.2/refrn/SESSION_PRIVS.html
138+
const {rows} = await connection.execute(`SELECT * FROM session_privs`);
139+
140+
const permissive = rows
141+
.map(([permission]) => permission)
142+
.filter((g) => !READ_ONLY.has(g));
143+
144+
if (permissive.length)
145+
throw failedCheck(
146+
`User has too permissive grants: ${permissive.join(", ")}`
147+
);
148+
149+
return {ok: true};
150+
} catch (e) {
151+
throw e;
152+
} finally {
153+
if (connection) {
154+
try {
155+
await connection.close();
156+
} catch (err) {
157+
console.error(err.message);
158+
}
159+
}
160+
}
161+
}
162+
163+
export default async ({url, username, password}) => {
164+
OracleSingleton.initialize();
165+
// We do not want to import the oracledb library until we are sure that the user is looking to use Oracle.
166+
// Installing the oracledb library is a pain, so we want to avoid it if possible.
167+
const config = {
168+
username: username,
169+
password: password,
170+
connectionString: decodeURI(url),
171+
};
172+
173+
const oracledb = await OracleSingleton.getInstance();
174+
const pool = oracledb.createPool(config);
175+
176+
return async (req, res) => {
177+
return queryStream(req, res, pool);
178+
};
179+
};
180+
181+
// See https://oracle.github.io/node-oracledb/doc/api.html#-312-oracle-database-type-constants
182+
const boolean = ["null", "boolean"],
183+
number = ["null", "number"],
184+
object = ["null", "object"],
185+
string = ["null", "string"];
186+
export function dataTypeSchema({type}) {
187+
const oracledb = OracleSingleton.getInstance();
188+
189+
switch (type) {
190+
case oracledb.DB_TYPE_BOOLEAN:
191+
return {type: boolean};
192+
case oracledb.DB_TYPE_NUMBER:
193+
case oracledb.DB_TYPE_BINARY_DOUBLE:
194+
case oracledb.DB_TYPE_BINARY_FLOAT:
195+
case oracledb.DB_TYPE_BINARY_INTEGER:
196+
return {type: number};
197+
case oracledb.DB_TYPE_TIMESTAMP:
198+
case oracledb.DB_TYPE_DATE:
199+
case oracledb.DB_TYPE_TIMESTAMP_TZ:
200+
case oracledb.DB_TYPE_TIMESTAMP_LTZ:
201+
case oracledb.DB_TYPE_INTERVAL_DS:
202+
case oracledb.DB_TYPE_INTERVAL_YM:
203+
return {type: string, date: true};
204+
case oracledb.DB_TYPE_BLOB:
205+
return {type: object, buffer: true};
206+
case oracledb.DB_TYPE_CHAR:
207+
case oracledb.DB_TYPE_BFILE:
208+
case oracledb.DB_TYPE_CLOB:
209+
case oracledb.DB_TYPE_NCLOB:
210+
case oracledb.DB_TYPE_CURSOR:
211+
case oracledb.DB_TYPE_LONG_RAW:
212+
case oracledb.DB_TYPE_NCHAR:
213+
case oracledb.DB_TYPE_NVARCHAR:
214+
case oracledb.DB_TYPE_OBJECT:
215+
case oracledb.DB_TYPE_RAW:
216+
case oracledb.DB_TYPE_ROWID:
217+
case oracledb.DB_TYPE_VARCHAR:
218+
default:
219+
return {type: string};
220+
}
221+
}

lib/server.js

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,9 @@ import mysql from "./mysql.js";
1212
import postgres from "./postgres.js";
1313
import snowflake from "./snowflake.js";
1414
import mssql from "./mssql.js";
15+
import oracle from "./oracle.js";
1516

16-
export function server(config, argv) {
17+
export async function server(config, argv) {
1718
const development = process.env.NODE_ENV === "development";
1819
const developmentOrigin = "https://worker.test:5000";
1920

@@ -24,6 +25,8 @@ export function server(config, argv) {
2425
ssl = "disabled",
2526
host = "127.0.0.1",
2627
port = 2899,
28+
username,
29+
password,
2730
} = config;
2831

2932
const handler =
@@ -35,6 +38,8 @@ export function server(config, argv) {
3538
? snowflake(url)
3639
: type === "mssql"
3740
? mssql(url)
41+
: type === "oracle"
42+
? await oracle({url, username, password})
3843
: null;
3944
if (!handler) {
4045
return exit(`Unknown database type: ${type}`);

lib/validate.js

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import Ajv from "ajv";
2+
3+
const ajv = new Ajv();
4+
5+
export const validateQueryPayload = ajv.compile({
6+
type: "object",
7+
additionalProperties: false,
8+
required: ["sql"],
9+
properties: {
10+
sql: {type: "string", minLength: 1, maxLength: 32 * 1000},
11+
params: {type: ["object", "array"]},
12+
},
13+
});

package.json

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@
1010
"./postgres.js": "./lib/postgres.js",
1111
"./mysql.js": "./lib/mysql.js",
1212
"./snowflake.js": "./lib/snowflake.js",
13-
"./mssql.js": "./lib/mssql.js"
13+
"./mssql.js": "./lib/mssql.js",
14+
"./oracle.js": "./lib/oracle.js"
1415
},
1516
"bin": {
1617
"observable-database-proxy": "./bin/observable-database-proxy.js"
@@ -39,7 +40,11 @@
3940
"nodemon": "^1.19.1",
4041
"wait-on": "^6.0.1"
4142
},
43+
"peerDependencies": {
44+
"oracledb": "https://github.com/oracle/node-oracledb/releases/download/v5.5.0/oracledb-src-5.5.0.tgz"
45+
},
4246
"scripts": {
47+
"start": "nodemon bin/observable-database-proxy.js",
4348
"dev": "NODE_ENV=development nodemon bin/observable-database-proxy.js",
4449
"test": "mocha",
4550
"test:local": "docker-compose -f docker-compose.yml -f docker-compose.local.yml up --build",

0 commit comments

Comments
 (0)