Skip to content

Commit

Permalink
Add async support for jq (#20)
Browse files Browse the repository at this point in the history
- Convert nan to napi
- Add async support
- Add ref counter to support multiple threads accessing the same cache value.
- Pass jv result as string in work to avoid sync with main thread.
- Parse jv in main thread to convert to napi.
- Set debug as compiled flag.
- Add cache resize from js.


---------

Co-authored-by: Ivan Kalinovski <[email protected]>
Co-authored-by: talsabagport <[email protected]>
  • Loading branch information
3 people authored Jan 28, 2025
1 parent c01bba1 commit 7eab7fe
Show file tree
Hide file tree
Showing 16 changed files with 1,231 additions and 186 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ jobs:
os: [ubuntu-22.04]
include:
- node: 16
os: macos-12
os: macos-14
- node: 20
os: macos-12
os: macos-14
runs-on: ${{ matrix.os }}
steps:
- name: Install node-gyp deps
Expand Down
62 changes: 42 additions & 20 deletions binding.gyp
Original file line number Diff line number Diff line change
Expand Up @@ -3,47 +3,69 @@
{
"target_name": "jq-node-bindings",
"sources": [
"src/binding.cc",
"src/binding.cc"
],
"include_dirs": [
"<!(node -e \"require('nan')\")",
"<!(node -p \"require('node-addon-api').include_dir\")",
"<(module_root_dir)/",
"deps/jq/src"
"deps/jq/src",
"../node-addon-api/"
],
'conditions': [
"defines": [
"NAPI_VERSION=8",
'NAPI_DISABLE_CPP_EXCEPTIONS'
],
"conditions": [
[
'OS=="linux"',
"OS=='linux'",
{
"libraries": [
"-Wl,-rpath='$$ORIGIN/../deps'",
"../build/deps/libjq.so.1",
"../build/deps/libjq.so.1"
],
'cflags_cc': [
'-std=c++17'
"cflags_cc": [
"-std=c++17"
],
'cflags_cc!': [
'-fno-rtti -fno-exceptions'
"cflags_cc!": [
"-fno-rtti -fno-exceptions"
]
}
],
[
'OS=="mac"',
"OS=='mac'",
{
"libraries": [
"../build/deps/libjq.dylib",
# "../build/deps/libonig.4.dylib",
# "../build/deps/libonig.dylib",
"../build/deps/libjq.dylib"
],
'xcode_settings': {
'MACOSX_DEPLOYMENT_TARGET': '12.0.1',
'GCC_ENABLE_CPP_RTTI': 'YES',
'GCC_ENABLE_CPP_EXCEPTIONS': 'YES'
"xcode_settings": {
"MACOSX_DEPLOYMENT_TARGET": "12.0.1",
"GCC_ENABLE_CPP_RTTI": "YES",
"GCC_ENABLE_CPP_EXCEPTIONS": "YES"
},
'OTHER_CPLUSPLUSFLAGS': [
'-std=c++17'
"OTHER_CPLUSPLUSFLAGS": [
"-std=c++17"
],
"include_dirs": [
"deps/jq/src"
], "cflags": ["-fsanitize=address", "-fno-omit-frame-pointer"],
"ldflags": ["-fsanitize=address"],
}
],
[
"OS=='win'",
{
"msvs_settings": {
"VCCLCompilerTool": {
"AdditionalOptions": ["/std:c++17"],
"ExceptionHandling": 1,
"RuntimeTypeInfo": "true"
}
},
"libraries": [
"deps\\jq\\build\\Release\\libjq.lib"
],
"include_dirs": [
"deps\\jq\\src"
]
}
]
Expand Down
2 changes: 1 addition & 1 deletion configure
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ make -j8
cp modules/oniguruma/src/.libs/libonig.a ${scriptdir}/build/deps/libonig.a
cp modules/oniguruma/src/.libs/libonig.la ${scriptdir}/build/deps/libonig.la
cp modules/oniguruma/src/.libs/libonig.lai ${scriptdir}/build/deps/libonig.lai
cp modules/oniguruma/src/.libs/libonig.4.dylib ${scriptdir}/build/deps/libonig.4.dylib
cp modules/oniguruma/src/.libs/libonig.5.dylib ${scriptdir}/build/deps/libonig.5.dylib
cp modules/oniguruma/src/.libs/libonig.dylib ${scriptdir}/build/deps/libonig.dylib

make install-libLTLIBRARIES install-includeHEADERS
Expand Down
2 changes: 1 addition & 1 deletion deps/jq
Submodule jq updated from 2e01ff to 588ff1
2 changes: 2 additions & 0 deletions index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ declare module '@port-labs/jq-node-bindings' {
}

export function exec(json: object, input: string, options?: ExecOptions): object | Array<any> | string | number | boolean | null;
export function execAsync(json: object, input: string, options?: ExecOptions): Promise<object | Array<any> | string | number | boolean | null>;

export function renderRecursively(json: object, input: object | Array<any> | string | number | boolean | null, execOptions?: ExecOptions): object | Array<any> | string | number | boolean | null;
export function renderRecursivelyAsync(json: object, input: object | Array<any> | string | number | boolean | null, execOptions?: ExecOptions): Promise<object | Array<any> | string | number | boolean | null>;
}
3 changes: 3 additions & 0 deletions lib/index.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
const jq = require('./jq');
const template = require('./template');
const templateAsync = require('./templateAsync');


module.exports = {
exec: jq.exec,
execAsync: jq.execAsync,
renderRecursively: template.renderRecursively,
renderRecursivelyAsync: templateAsync.renderRecursivelyAsync,
JqExecError: jq.JqExecError,
JqExecCompileError: jq.JqExecCompileError,
};
14 changes: 13 additions & 1 deletion lib/jq.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ class JqExecCompileError extends JqExecError {

const exec = (object, filter, {enableEnv = false, throwOnError = false} = {}) => {
try {
const data = nativeJq.exec(JSON.stringify(object), formatFilter(filter, {enableEnv}))
const data = nativeJq.execSync(JSON.stringify(object), formatFilter(filter, {enableEnv}))

return data?.value;
} catch (err) {
Expand All @@ -27,8 +27,20 @@ const exec = (object, filter, {enableEnv = false, throwOnError = false} = {}) =>
}
}

const execAsync = async (object, filter, {enableEnv = false, throwOnError = false} = {}) => {
try {
const data = await nativeJq.execAsync(JSON.stringify(object), formatFilter(filter, {enableEnv}))
return data?.value;
} catch (err) {
if (throwOnError) {
throw new (err?.message?.startsWith('jq: compile error') ? JqExecCompileError : JqExecError)(err.message);
}
return null
}
}
module.exports = {
exec,
execAsync,
JqExecError,
JqExecCompileError
};
136 changes: 136 additions & 0 deletions lib/templateAsync.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
const jq = require('./jq');

const findInsideDoubleBracesIndices = (input) => {
let wrappingQuote = null;
let insideDoubleBracesStart = null;
const indices = [];

for (let i = 0; i < input.length; i += 1) {
const char = input[i];

if (insideDoubleBracesStart && char === '\\') {
// If next character is escaped, skip it
i += 1;
}
if (insideDoubleBracesStart && (char === '"' || char === "'")) {
// If inside double braces and inside quotes, ignore braces
if (!wrappingQuote) {
wrappingQuote = char;
} else if (wrappingQuote === char) {
wrappingQuote = null;
}
} else if (!wrappingQuote && char === '{' && i > 0 && input[i - 1] === '{') {
// if opening double braces that not wrapped with quotes
if (insideDoubleBracesStart) {
throw new Error(`Found double braces in index ${i - 1} inside other one in index ${insideDoubleBracesStart - '{{'.length}`);
}
insideDoubleBracesStart = i + 1;
if (input[i + 1] === '{') {
// To overcome three "{" in a row considered as two different opening double braces
i += 1;
}
} else if (!wrappingQuote && char === '}' && i > 0 && input[i - 1] === '}') {
// if closing double braces that not wrapped with quotes
if (insideDoubleBracesStart) {
indices.push({start: insideDoubleBracesStart, end: i - 1});
insideDoubleBracesStart = null;
if (input[i + 1] === '}') {
// To overcome three "}" in a row considered as two different closing double braces
i += 1;
}
} else {
throw new Error(`Found closing double braces in index ${i - 1} without opening double braces`);
}
}
}

if (insideDoubleBracesStart) {
throw new Error(`Found opening double braces in index ${insideDoubleBracesStart - '{{'.length} without closing double braces`);
}

return indices;
}

const renderAsync =async (inputJson, template, execOptions = {}) => {
if (typeof template !== 'string') {
return null;
}
const indices = findInsideDoubleBracesIndices(template);
if (!indices.length) {
// If no jq templates in string, return it
return template;
}

const firstIndex = indices[0];
if (indices.length === 1 && template.trim().startsWith('{{') && template.trim().endsWith('}}')) {
// If entire string is a template, evaluate and return the result with the original type
return await jq.execAsync(inputJson, template.slice(firstIndex.start, firstIndex.end), execOptions);
}

let result = template.slice(0, firstIndex.start - '{{'.length); // Initiate result with string until first template start index
for (let i = 0; i < indices.length; i++) {
const index = indices[i];

// }
// indices.forEach((index, i) => {
const jqResult = await jq.execAsync(inputJson, template.slice(index.start, index.end), execOptions);
result +=
// Add to the result the stringified evaluated jq of the current template
(typeof jqResult === 'string' ? jqResult : JSON.stringify(jqResult)) +
// Add to the result from template end index. if last template index - until the end of string, else until next start index
template.slice(
index.end + '}}'.length,
i + 1 === indices.length ? template.length : indices[i + 1].start - '{{'.length,
);
// });
}

return result;
}

const renderRecursivelyAsync = async(inputJson, template, execOptions = {}) => {
if (typeof template === 'string') {
return await renderAsync(inputJson, template, execOptions);
}
if (Array.isArray(template)) {
return await Promise.all(template.map((value) => renderRecursivelyAsync(inputJson, value, execOptions)));
}
if (typeof template === 'object' && template !== null) {



const t =Object.entries(template).map(async([key, value]) => {
const SPREAD_KEYWORD = "spreadValue";
const keywordMatcher = `^\\{\\{\\s*${SPREAD_KEYWORD}\\(\\s*\\)\\s*\\}\\}$`; // matches {{ <Keyword>() }} with white spaces where you'd expect them

if (key.trim().match(keywordMatcher)) {
const evaluatedValue = await renderRecursivelyAsync(inputJson, value, execOptions);
if (typeof evaluatedValue !== "object") {
throw new Error(
`Evaluated value should be an object if the key is ${key}. Original value: ${value}, evaluated to: ${JSON.stringify(evaluatedValue)}`
);
}
return Object.entries(evaluatedValue);
}

const evaluatedKey = await renderRecursivelyAsync(inputJson, key, execOptions);
if (!['undefined', 'string'].includes(typeof evaluatedKey) && evaluatedKey !== null) {
throw new Error(
`Evaluated object key should be undefined, null or string. Original key: ${key}, evaluated to: ${JSON.stringify(evaluatedKey)}`,
);
}
return evaluatedKey ? [[evaluatedKey, await renderRecursivelyAsync(inputJson, value, execOptions)]] : [];
});


return Object.fromEntries((await Promise.all(t)).flat());


}

return template;
}

module.exports = {
renderRecursivelyAsync
};
21 changes: 18 additions & 3 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
{
"name": "@port-labs/jq-node-bindings",
"version": "v0.0.14",
"version": "v1.0.0",
"description": "Node.js bindings for JQ",
"jq-node-bindings": "0.0.14",
"jq-node-bindings": "1.0.0",
"main": "lib/index.js",
"scripts": {
"configure": "node-gyp configure",
Expand Down Expand Up @@ -39,7 +39,7 @@
},
"dependencies": {
"bindings": "^1.3.1",
"nan": "^2.20.0"
"node-addon-api": "^8.3.0"
},
"engines": {
"node": ">=6.0.0"
Expand Down
Loading

0 comments on commit 7eab7fe

Please sign in to comment.