From 7eab7fe1e0d3563f49c44f31ed7edd912346eafb Mon Sep 17 00:00:00 2001 From: Ivan <62664893+ivankalinovski@users.noreply.github.com> Date: Tue, 28 Jan 2025 18:14:47 +0200 Subject: [PATCH] Add async support for jq (#20) - Convert nan to napi - Add async support - Add ref counter to support multiple threads accessing the same cache value. - Pass jv result as string in work to avoid sync with main thread. - Parse jv in main thread to convert to napi. - Set debug as compiled flag. - Add cache resize from js. --------- Co-authored-by: Ivan Kalinovski Co-authored-by: talsabagport --- .github/workflows/test.yml | 4 +- binding.gyp | 62 ++- configure | 2 +- deps/jq | 2 +- index.d.ts | 2 + lib/index.js | 3 + lib/jq.js | 14 +- lib/templateAsync.js | 136 ++++++ package-lock.json | 21 +- package.json | 6 +- src/binding.cc | 796 +++++++++++++++++++++++++++++------- src/binding.h | 7 +- test/santiy-async.test.js | 171 ++++++++ test/santiy.test.js | 4 +- test/template-async.test.js | 182 +++++++++ test/template.test.js | 5 +- 16 files changed, 1231 insertions(+), 186 deletions(-) create mode 100644 lib/templateAsync.js create mode 100644 test/santiy-async.test.js create mode 100644 test/template-async.test.js diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 756a198..7bba2a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,9 +14,9 @@ jobs: os: [ubuntu-22.04] include: - node: 16 - os: macos-12 + os: macos-14 - node: 20 - os: macos-12 + os: macos-14 runs-on: ${{ matrix.os }} steps: - name: Install node-gyp deps diff --git a/binding.gyp b/binding.gyp index 0aebc55..4b2d68a 100755 --- a/binding.gyp +++ b/binding.gyp @@ -3,47 +3,69 @@ { "target_name": "jq-node-bindings", "sources": [ - "src/binding.cc", + "src/binding.cc" ], "include_dirs": [ - " | string | number | boolean | null; + export function execAsync(json: object, input: string, options?: ExecOptions): Promise | string | number | boolean | null>; export function renderRecursively(json: object, input: object | Array | string | number | boolean | null, execOptions?: ExecOptions): object | Array | string | number | boolean | null; + export function renderRecursivelyAsync(json: object, input: object | Array | string | number | boolean | null, execOptions?: ExecOptions): Promise | string | number | boolean | null>; } diff --git a/lib/index.js b/lib/index.js index 41d308c..e995fd4 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,10 +1,13 @@ const jq = require('./jq'); const template = require('./template'); +const templateAsync = require('./templateAsync'); module.exports = { exec: jq.exec, + execAsync: jq.execAsync, renderRecursively: template.renderRecursively, + renderRecursivelyAsync: templateAsync.renderRecursivelyAsync, JqExecError: jq.JqExecError, JqExecCompileError: jq.JqExecCompileError, }; diff --git a/lib/jq.js b/lib/jq.js index bcfb2ae..17edcd2 100644 --- a/lib/jq.js +++ b/lib/jq.js @@ -16,7 +16,7 @@ class JqExecCompileError extends JqExecError { const exec = (object, filter, {enableEnv = false, throwOnError = false} = {}) => { try { - const data = nativeJq.exec(JSON.stringify(object), formatFilter(filter, {enableEnv})) + const data = nativeJq.execSync(JSON.stringify(object), formatFilter(filter, {enableEnv})) return data?.value; } catch (err) { @@ -27,8 +27,20 @@ const exec = (object, filter, {enableEnv = false, throwOnError = false} = {}) => } } +const execAsync = async (object, filter, {enableEnv = false, throwOnError = false} = {}) => { + try { + const data = await nativeJq.execAsync(JSON.stringify(object), formatFilter(filter, {enableEnv})) + return data?.value; + } catch (err) { + if (throwOnError) { + throw new (err?.message?.startsWith('jq: compile error') ? JqExecCompileError : JqExecError)(err.message); + } + return null + } +} module.exports = { exec, + execAsync, JqExecError, JqExecCompileError }; diff --git a/lib/templateAsync.js b/lib/templateAsync.js new file mode 100644 index 0000000..c2d5d17 --- /dev/null +++ b/lib/templateAsync.js @@ -0,0 +1,136 @@ +const jq = require('./jq'); + +const findInsideDoubleBracesIndices = (input) => { + let wrappingQuote = null; + let insideDoubleBracesStart = null; + const indices = []; + + for (let i = 0; i < input.length; i += 1) { + const char = input[i]; + + if (insideDoubleBracesStart && char === '\\') { + // If next character is escaped, skip it + i += 1; + } + if (insideDoubleBracesStart && (char === '"' || char === "'")) { + // If inside double braces and inside quotes, ignore braces + if (!wrappingQuote) { + wrappingQuote = char; + } else if (wrappingQuote === char) { + wrappingQuote = null; + } + } else if (!wrappingQuote && char === '{' && i > 0 && input[i - 1] === '{') { + // if opening double braces that not wrapped with quotes + if (insideDoubleBracesStart) { + throw new Error(`Found double braces in index ${i - 1} inside other one in index ${insideDoubleBracesStart - '{{'.length}`); + } + insideDoubleBracesStart = i + 1; + if (input[i + 1] === '{') { + // To overcome three "{" in a row considered as two different opening double braces + i += 1; + } + } else if (!wrappingQuote && char === '}' && i > 0 && input[i - 1] === '}') { + // if closing double braces that not wrapped with quotes + if (insideDoubleBracesStart) { + indices.push({start: insideDoubleBracesStart, end: i - 1}); + insideDoubleBracesStart = null; + if (input[i + 1] === '}') { + // To overcome three "}" in a row considered as two different closing double braces + i += 1; + } + } else { + throw new Error(`Found closing double braces in index ${i - 1} without opening double braces`); + } + } + } + + if (insideDoubleBracesStart) { + throw new Error(`Found opening double braces in index ${insideDoubleBracesStart - '{{'.length} without closing double braces`); + } + + return indices; +} + +const renderAsync =async (inputJson, template, execOptions = {}) => { + if (typeof template !== 'string') { + return null; + } + const indices = findInsideDoubleBracesIndices(template); + if (!indices.length) { + // If no jq templates in string, return it + return template; + } + + const firstIndex = indices[0]; + if (indices.length === 1 && template.trim().startsWith('{{') && template.trim().endsWith('}}')) { + // If entire string is a template, evaluate and return the result with the original type + return await jq.execAsync(inputJson, template.slice(firstIndex.start, firstIndex.end), execOptions); + } + + let result = template.slice(0, firstIndex.start - '{{'.length); // Initiate result with string until first template start index + for (let i = 0; i < indices.length; i++) { + const index = indices[i]; + +// } +// indices.forEach((index, i) => { + const jqResult = await jq.execAsync(inputJson, template.slice(index.start, index.end), execOptions); + result += + // Add to the result the stringified evaluated jq of the current template + (typeof jqResult === 'string' ? jqResult : JSON.stringify(jqResult)) + + // Add to the result from template end index. if last template index - until the end of string, else until next start index + template.slice( + index.end + '}}'.length, + i + 1 === indices.length ? template.length : indices[i + 1].start - '{{'.length, + ); +// }); + } + + return result; +} + +const renderRecursivelyAsync = async(inputJson, template, execOptions = {}) => { + if (typeof template === 'string') { + return await renderAsync(inputJson, template, execOptions); + } + if (Array.isArray(template)) { + return await Promise.all(template.map((value) => renderRecursivelyAsync(inputJson, value, execOptions))); + } + if (typeof template === 'object' && template !== null) { + + + + const t =Object.entries(template).map(async([key, value]) => { + const SPREAD_KEYWORD = "spreadValue"; + const keywordMatcher = `^\\{\\{\\s*${SPREAD_KEYWORD}\\(\\s*\\)\\s*\\}\\}$`; // matches {{ () }} with white spaces where you'd expect them + + if (key.trim().match(keywordMatcher)) { + const evaluatedValue = await renderRecursivelyAsync(inputJson, value, execOptions); + if (typeof evaluatedValue !== "object") { + throw new Error( + `Evaluated value should be an object if the key is ${key}. Original value: ${value}, evaluated to: ${JSON.stringify(evaluatedValue)}` + ); + } + return Object.entries(evaluatedValue); + } + + const evaluatedKey = await renderRecursivelyAsync(inputJson, key, execOptions); + if (!['undefined', 'string'].includes(typeof evaluatedKey) && evaluatedKey !== null) { + throw new Error( + `Evaluated object key should be undefined, null or string. Original key: ${key}, evaluated to: ${JSON.stringify(evaluatedKey)}`, + ); + } + return evaluatedKey ? [[evaluatedKey, await renderRecursivelyAsync(inputJson, value, execOptions)]] : []; + }); + + + return Object.fromEntries((await Promise.all(t)).flat()); + + + } + + return template; +} + +module.exports = { + renderRecursivelyAsync +}; diff --git a/package-lock.json b/package-lock.json index f2c8807..29ddb02 100755 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,18 @@ { "name": "@port-labs/jq-node-bindings", - "version": "v0.0.14", + "version": "v0.0.15-dev1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@port-labs/jq-node-bindings", - "version": "v0.0.14", + "version": "v0.0.15-dev1", "hasInstallScript": true, "license": "MIT", "dependencies": { "bindings": "^1.3.1", - "nan": "^2.20.0" + "nan": "^2.20.0", + "node-addon-api": "^8.3.0" }, "devDependencies": { "bluebird": "^3.5.3", @@ -4153,6 +4154,15 @@ "node": ">= 0.6" } }, + "node_modules/node-addon-api": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.3.0.tgz", + "integrity": "sha512-8VOpLHFrOQlAH+qA0ZzuGRlALRA6/LVh8QJldbrC4DY0hXoMP0l4Acq8TzFC018HztWiRqyCEj2aTWY2UvnJUg==", + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, "node_modules/node-gyp": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-10.2.0.tgz", @@ -8606,6 +8616,11 @@ "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", "dev": true }, + "node-addon-api": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.3.0.tgz", + "integrity": "sha512-8VOpLHFrOQlAH+qA0ZzuGRlALRA6/LVh8QJldbrC4DY0hXoMP0l4Acq8TzFC018HztWiRqyCEj2aTWY2UvnJUg==" + }, "node-gyp": { "version": "10.2.0", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-10.2.0.tgz", diff --git a/package.json b/package.json index 9e684d6..557817e 100755 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "@port-labs/jq-node-bindings", - "version": "v0.0.14", + "version": "v1.0.0", "description": "Node.js bindings for JQ", - "jq-node-bindings": "0.0.14", + "jq-node-bindings": "1.0.0", "main": "lib/index.js", "scripts": { "configure": "node-gyp configure", @@ -39,7 +39,7 @@ }, "dependencies": { "bindings": "^1.3.1", - "nan": "^2.20.0" + "node-addon-api": "^8.3.0" }, "engines": { "node": ">=6.0.0" diff --git a/src/binding.cc b/src/binding.cc index f71dd0c..bd1062d 100644 --- a/src/binding.cc +++ b/src/binding.cc @@ -1,128 +1,77 @@ -#include "src/binding.h" #include #include #include +#include +#include +#include +#include -using namespace std; +#include "src/binding.h" -// LRUCache to save filter to the compiled jq_state -template class LRUCache{ -private: - list< pair > item_list; - unordered_map item_map; - size_t cache_size; -private: - void clean(void){ - while(item_map.size()>cache_size){ - auto last_it = item_list.end(); last_it --; - item_map.erase(last_it->first); - item_list.pop_back(); - jq_teardown(&last_it->second); - } - }; -public: - LRUCache(int cache_size_):cache_size(cache_size_){ - ; - }; - - void put(const KEY_T &key, const VAL_T &val){ - auto it = item_map.find(key); - if(it != item_map.end()){ - item_list.erase(it->second); - item_map.erase(it); - } - item_list.push_front(make_pair(key,val)); - item_map.insert(make_pair(key, item_list.begin())); - clean(); - }; - bool exist(const KEY_T &key){ - return (item_map.count(key)>0); - }; - VAL_T get(const KEY_T &key){ - assert(exist(key)); - auto it = item_map.find(key); - item_list.splice(item_list.begin(), item_list, it->second); - return it->second->second; - }; -}; +// #ifdef DEBUG_MODE +// static bool debug_enabled = true; +// #else +// static bool debug_enabled = false; +// #endif -LRUCache cache(100); +// #define DEBUG_LOG(fmt, ...) \ +// do { if (debug_enabled) printf("[DEBUG] " fmt "\n", ##__VA_ARGS__); } while (0) -void jv_object_to_v8(std::string key, jv actual, v8::Local ret) { - jv_kind k = jv_get_kind(actual); +// #define ASYNC_DEBUG_LOG(work, fmt, ...) \ +// do { if (debug_enabled) printf("[DEBUG][ASYNC][%p] " fmt "\n", (jv*)work, ##__VA_ARGS__); } while (0) - v8::Local v8_key = Nan::New(key).ToLocalChecked(); - v8::Local v8_val; +// #define CACHE_DEBUG_LOG(cache, fmt, ...) \ +// do { if (debug_enabled) printf("[DEBUG][CACHE][%p] " fmt "\n", (void*)cache, ##__VA_ARGS__); } while (0) - switch (k) { - case JV_KIND_INVALID: { - jv msg = jv_invalid_get_msg(jv_copy(actual)); - char err[4096]; - if (jv_get_kind(msg) == JV_KIND_STRING) { - snprintf(err, sizeof(err), "jq: error: %s", jv_string_value(msg)); - jv_free(msg); - jv_free(actual); - Nan::ThrowError(err); - return; - } - jv_free(msg); - jv_free(actual); - break; - } - case JV_KIND_NULL: { - v8_val = Nan::Null(); - break; - } - case JV_KIND_TRUE: { - v8_val = Nan::True(); - break; - } - case JV_KIND_FALSE: { - v8_val = Nan::False(); - break; - } - case JV_KIND_NUMBER: { - v8_val = Nan::New(jv_number_value(actual)); - break; - } - case JV_KIND_STRING: { - v8_val = Nan::New(jv_string_value(actual)).ToLocalChecked(); - jv_free(actual); - break; - } - case JV_KIND_ARRAY: { - v8::Local ret_arr = Nan::New(); - for (int i = 0; i < jv_array_length(jv_copy(actual)); i++) { - jv_object_to_v8(std::to_string(i), jv_array_get(jv_copy(actual), i), ret_arr); - } - Nan::Set(ret, v8_key, ret_arr); - jv_free(actual); - break; - } - case JV_KIND_OBJECT: { - v8::Local ret_obj = Nan::New(); - jv_object_foreach(actual, itr_key, value) { - jv_object_to_v8(jv_string_value(itr_key), value, ret_obj); - } - jv_free(actual); - Nan::Set(ret, v8_key, ret_obj); - break; - } - } - - if (v8_val.IsEmpty()) { - return; +// #define WRAPPER_DEBUG_LOG(wrapper, fmt, ...) \ +// do { if (debug_enabled) printf("[DEBUG][WRAPPER:%p] " fmt "\n", (void*)wrapper, ##__VA_ARGS__); } while (0) +#ifdef ENABLE_DEBUG // We'll use ENABLE_DEBUG as our flag name +#define DEBUG_ENABLED 1 +#else +#define DEBUG_ENABLED 0 +#endif + +#ifdef ENABLE_DEBUG +#define DEBUG_LOG(fmt, ...) fprintf(stderr, "[DEBUG] " fmt "\n", ##__VA_ARGS__) +#define ASYNC_DEBUG_LOG(work, fmt, ...) fprintf(stderr, "[DEBUG][ASYNC][%p] " fmt "\n", (void*)work, ##__VA_ARGS__) +#define CACHE_DEBUG_LOG(cache, fmt, ...) fprintf(stderr, "[DEBUG][CACHE][%p] " fmt "\n", (void*)cache, ##__VA_ARGS__) +#define WRAPPER_DEBUG_LOG(wrapper, fmt, ...) fprintf(stderr, "[DEBUG][WRAPPER][%p] " fmt "\n", (void*)wrapper, ##__VA_ARGS__) +#else +#define DEBUG_LOG(fmt, ...) ((void)0) +#define ASYNC_DEBUG_LOG(work, fmt, ...) ((void)0) +#define CACHE_DEBUG_LOG(cache, fmt, ...) ((void)0) +#define WRAPPER_DEBUG_LOG(wrapper, fmt, ...) ((void)0) +#endif + +static size_t global_cache_size = 100; + +static size_t get_uv_thread_pool_size() { + const char* uv_threads = getenv("UV_THREADPOOL_SIZE"); + if (uv_threads != nullptr) { + int thread_count = atoi(uv_threads); + if (thread_count > 0) { + return static_cast(thread_count); + } } + return global_cache_size; +} - Nan::Set(ret, v8_key, v8_val); +static size_t validate_cache_size(size_t requested_size) { + size_t min_size = get_uv_thread_pool_size(); + size_t new_size = std::max(requested_size, min_size); + if(requested_size < min_size){ + DEBUG_LOG("Requested cache size %zu adjusted to minimum %zu (UV thread pool size)",requested_size,min_size); + return min_size; + } + return new_size; } +/* err_data and throw_err_cb to get jq error message*/ struct err_data { - char buf[4096]; + char buf[4096]; }; - -void throw_err_cb(void *data, jv msg) { +void throw_err_cb(void* data, jv msg) { struct err_data *err_data = (struct err_data *)data; if (jv_get_kind(msg) != JV_KIND_STRING) msg = jv_dump_string(msg, JV_PRINT_INVALID); @@ -133,72 +82,627 @@ void throw_err_cb(void *data, jv msg) { jv_free(msg); } -void jq_exec(std::string json, std::string filter,const Nan::FunctionCallbackInfo& info) { - jq_state *jq = NULL; +/* check napi status to throw error if napi_status is not ok */ +inline bool CheckNapiStatus(napi_env env, napi_status status, const char* message) { + if (status != napi_ok) { + napi_throw_error(env, nullptr, message); + return false; + } + return true; +} + +template class LRUCache; + +struct JqFilterWrapper { + friend class LRUCache; +public: + std::string filter_name; + std::list::iterator cache_pos; + /* init mutex and set filter_name */ + explicit JqFilterWrapper(jq_state* jq_, std::string filter_name_) : + filter_name(filter_name_), + jq(jq_) { + DEBUG_LOG("[WRAPPER:%p] Creating wrapper for filter: %s", (void*)this, filter_name_.c_str()); + pthread_mutex_init(&filter_mutex, nullptr); + } + + /* free jq and destroy mutex */ + ~JqFilterWrapper() { + WRAPPER_DEBUG_LOG(this, "Destroying wrapper: %s", filter_name.c_str()); + if (jq) { + WRAPPER_DEBUG_LOG(this, "Tearing down jq state"); + jq_teardown(&jq); + } + pthread_mutex_destroy(&filter_mutex); + WRAPPER_DEBUG_LOG(this, "Destroyed"); + } + jq_state* get_jq(){ + return jq; + } + void lock(){ + WRAPPER_DEBUG_LOG(this, "Attempting to lock mutex"); + pthread_mutex_lock(&filter_mutex); + WRAPPER_DEBUG_LOG(this, "Mutex locked"); + } + void unlock(){ + WRAPPER_DEBUG_LOG(this, "Unlocking mutex"); + pthread_mutex_unlock(&filter_mutex); + WRAPPER_DEBUG_LOG(this, "Mutex unlocked"); + } +private: + jq_state* jq; + pthread_mutex_t filter_mutex; + +}; + +template class LRUCache { +private: + pthread_mutex_t cache_mutex; + std::list item_list; + std::unordered_map item_map; + std::unordered_map item_refcnt; + + size_t cache_size; + + void clean() { + pthread_mutex_lock(&cache_mutex); + CACHE_DEBUG_LOG(nullptr, "Starting cleanup. Current size=%zu, target=%zu", item_map.size(), cache_size); + if(item_map.size() < cache_size){ + pthread_mutex_unlock(&cache_mutex); + return; + } + while (item_list.size() > cache_size) { + auto last_it = item_list.end(); + last_it--; + JqFilterWrapper* wrapper = *last_it; + CACHE_DEBUG_LOG((void*)wrapper, "Examining wrapper: name='%s', refcnt=%zu", wrapper->filter_name.c_str(), item_refcnt[wrapper]); + if(item_refcnt[wrapper]>0){ + CACHE_DEBUG_LOG((void*)wrapper, "Wrapper is busy, skipping"); + break; + } + if(wrapper->filter_name == ""){ + CACHE_DEBUG_LOG((void*)wrapper, "WARNING: Empty filter name found"); + } + CACHE_DEBUG_LOG((void*)wrapper, "attempting to remove wrapper from cache"); + if(item_map.find(wrapper->filter_name)->second == wrapper){ + CACHE_DEBUG_LOG((void*)wrapper, "Removing wrapper from cache"); + item_map.erase(wrapper->filter_name); + }; + item_refcnt.erase(wrapper); + item_list.pop_back(); + CACHE_DEBUG_LOG((void*)wrapper, "Deleting wrapper"); + delete wrapper; + + } + CACHE_DEBUG_LOG(this, "Cleanup complete. New size=%zu", item_map.size()); + pthread_mutex_unlock(&cache_mutex); + } + +public: + LRUCache(int cache_size_) : cache_size(global_cache_size) { + pthread_mutex_init(&cache_mutex, nullptr); + CACHE_DEBUG_LOG(this, "Created cache with size %zu", cache_size); + } + ~LRUCache() { + //clear cache + pthread_mutex_destroy(&cache_mutex); + } + void inc_refcnt(JqFilterWrapper* val){ + CACHE_DEBUG_LOG((void*)val, "Incrementing refcnt for wrapper:%p", (void*)val); + item_refcnt[val]++; + } + void dec_refcnt(JqFilterWrapper* val){ + pthread_mutex_lock(&cache_mutex); + CACHE_DEBUG_LOG((void*)val, "Decrementing refcnt for wrapper:%p", (void*)val); + item_refcnt[val]--; + pthread_mutex_unlock(&cache_mutex); + } + void put(const KEY_T &key, JqFilterWrapper* val) { + CACHE_DEBUG_LOG((void*)val, "Putting key='%s' wrapper:%p", key.c_str(), (void*)val); + pthread_mutex_lock(&cache_mutex); + inc_refcnt(val); + CACHE_DEBUG_LOG((void*)val, "Got cache lock for put operation"); + + auto it = item_map.find(key); + if (it != item_map.end()) { + CACHE_DEBUG_LOG((void*)val, "Replacing existing entry for key='%s', old_ptr=%p , new_ptr=%p", key.c_str(), (void*)it->second, (void*)val); + item_map.erase(it); + } + item_list.push_front(val); + val->cache_pos = item_list.begin(); + + item_map.insert(std::make_pair(key, val)); + CACHE_DEBUG_LOG((void*)val, "Added wrapper:%p to cache", (void*)val); + pthread_mutex_unlock(&cache_mutex); + CACHE_DEBUG_LOG((void*)val, "Released cache lock after put"); + clean(); + } + + JqFilterWrapper* get(const KEY_T &key) { + pthread_mutex_lock(&cache_mutex); + CACHE_DEBUG_LOG(nullptr, "Got cache lock for get operation, key='%s'", key.c_str()); + + if(!(item_map.count(key) > 0)){ + CACHE_DEBUG_LOG(nullptr, "Cache miss for key='%s'", key.c_str()); + pthread_mutex_unlock(&cache_mutex); + return nullptr; + } + + auto it = item_map.find(key); + JqFilterWrapper* wrapper = it->second; + item_list.erase(wrapper->cache_pos); + item_list.push_front(wrapper); + wrapper->cache_pos = item_list.begin(); + inc_refcnt(wrapper); + CACHE_DEBUG_LOG((void*)wrapper, "Cache hit for jq wrapper,pointer=%p,name=%s,refcnt=%zu", + (void*)wrapper, wrapper->filter_name.c_str(),item_refcnt[wrapper]); + pthread_mutex_unlock(&cache_mutex); + CACHE_DEBUG_LOG((void*)wrapper, "Released cache lock after get"); + return wrapper; + } + void resize(size_t new_size) { + pthread_mutex_lock(&cache_mutex); + CACHE_DEBUG_LOG(this, "Resizing cache from %zu to %zu", cache_size, new_size); + cache_size = new_size; + pthread_mutex_unlock(&cache_mutex); + clean(); // Trigger cleanup if needed + } +}; + +LRUCache cache(100); + +std::string FromNapiString(napi_env env, napi_value value) { + size_t str_size; + size_t str_size_out; + napi_status status; + status=napi_get_value_string_utf8(env, value, nullptr, 0, &str_size); + if(!CheckNapiStatus(env,status,"error loading string lenth")){ + return ""; + } + char* str = new char[str_size + 1]; + status=napi_get_value_string_utf8(env, value, str, str_size + 1, &str_size_out); + if(!CheckNapiStatus(env,status,"error loading string")){ + delete[] str; + return ""; + } + + std::string result(str); + delete[] str; + return result; +} + +bool jv_object_to_napi(std::string key, napi_env env, jv actual, napi_value ret,std::string& err_msg) { + jv_kind kind = jv_get_kind(actual); + napi_value value; + napi_status status = napi_invalid_arg; + switch (kind) { + case JV_KIND_INVALID: { + jv msg = jv_invalid_get_msg(jv_copy(actual)); + + if (jv_get_kind(msg) == JV_KIND_STRING) { + err_msg = std::string("jq: error: ") + jv_string_value(msg); + jv_free(msg); + return false; + } + napi_get_undefined(env, &ret); + return true; + } + case JV_KIND_NULL: { + status=napi_get_null(env, &value); + break; + } + case JV_KIND_TRUE: { + status=napi_get_boolean(env, true, &value); + break; + } + case JV_KIND_FALSE: { + status=napi_get_boolean(env, false, &value); + break; + } + case JV_KIND_NUMBER: { + double num = jv_number_value(actual); + status=napi_create_double(env, num, &value); + break; + } + case JV_KIND_STRING: { + status=napi_create_string_utf8(env, jv_string_value(actual), NAPI_AUTO_LENGTH, &value); + break; + } + case JV_KIND_ARRAY: { + size_t arr_len = jv_array_length(jv_copy(actual)); + status=napi_create_array_with_length(env, arr_len, &value); + + for (size_t i = 0; i < arr_len; i++) { + jv v = jv_array_get(jv_copy(actual), i); + bool success = jv_object_to_napi(std::to_string(i), env, v, value,err_msg); + if(!success){ + jv_free(v); + return false; + } + jv_free(v); + } + break; + } + case JV_KIND_OBJECT: { + status=napi_create_object(env, &value); + + int iter = jv_object_iter(actual); + while (jv_object_iter_valid(actual, iter)) { + + jv obj_key = jv_object_iter_key(actual, iter); + jv obj_value = jv_object_iter_value(actual, iter); + + bool success = jv_object_to_napi(jv_string_value(obj_key), env, obj_value, value,err_msg); + if(!success){ + jv_free(obj_key); + jv_free(obj_value); + return false; + } + + jv_free(obj_key); + jv_free(obj_value); + + iter = jv_object_iter_next(actual, iter); + } + break; + } + // default: + // napi_throw_error(env, nullptr, "Unsupported jv type"); + // break; + } + if(status != napi_ok){ + err_msg = "error creating napi object"; + return false; + } + napi_set_named_property(env, ret, key.c_str(), value); + return true; +} + + + + +napi_value ExecSync(napi_env env, napi_callback_info info) { + size_t argc = 2; + napi_value args[2]; + napi_status status; + status=napi_get_cb_info(env, info, &argc, args, nullptr, nullptr); + CheckNapiStatus(env,status,"Error loading info"); + + if (argc < 2) { + napi_throw_type_error(env, nullptr, "Wrong number of arguments. Expected 2."); + return nullptr; + } + + std::string json = FromNapiString(env, args[0]); + if(json == ""){ + napi_throw_error(env, nullptr, "Invalid JSON input"); + return nullptr; + } + std::string filter = FromNapiString(env, args[1]); + if(filter == ""){ + napi_throw_error(env, nullptr, "Invalid filter input"); + return nullptr; + } + struct err_data err_msg; + JqFilterWrapper* wrapper; - if (cache.exist(filter)) { - jq = cache.get(filter); - } else { + DEBUG_LOG("[SYNC] ExecSync called with filter='%s'", filter.c_str()); + + wrapper = cache.get(filter); + if (wrapper == nullptr) { + DEBUG_LOG("[SYNC] Creating new wrapper for filter='%s'", filter.c_str()); + jq_state* jq; jq = jq_init(); jq_set_error_cb(jq, throw_err_cb, &err_msg); if (!jq_compile(jq, filter.c_str())) { - Nan::ThrowError(err_msg.buf); - return; + napi_throw_error(env, nullptr, err_msg.buf); + return nullptr; + } + if (jq == nullptr) { + napi_throw_error(env, nullptr, "Failed to initialize jq"); + return nullptr; } - cache.put(filter, jq); + wrapper = new JqFilterWrapper(jq, filter); + cache.put(filter, wrapper ); } - if (jq == NULL) { - info.GetReturnValue().Set(Nan::Null()); - return; + jv input = jv_parse(json.c_str()); + if (!jv_is_valid(input)) { + jv_free(input); + napi_throw_error(env, nullptr, "Invalid JSON input"); + wrapper->unlock(); + return nullptr; } - jv input = jv_parse(json.c_str()); + wrapper->lock(); + + jq_start(wrapper->get_jq(), input, 0); + jv result = jq_next(wrapper->get_jq()); + + napi_value ret; + napi_create_object(env, &ret); + std::string err_msg_conversion; + bool success = jv_object_to_napi("value",env,result,ret,err_msg_conversion); + if(!success){ + napi_throw_error(env, nullptr, err_msg_conversion.c_str()); + jv_free(result); + wrapper->unlock(); + cache.dec_refcnt(wrapper); + return nullptr; + } + + jv_free(result); + wrapper->unlock(); + cache.dec_refcnt(wrapper); + return ret; +} + +struct AsyncWork { + /* input */ + std::string json; + std::string filter; + /* promise */ + napi_deferred deferred; + napi_async_work async_work; + /* output */ + bool is_undefined; + std::string result; + std::string error; + bool success; +}; + +void ExecuteAsync(napi_env env, void* data) { + AsyncWork* work = static_cast(data); + ASYNC_DEBUG_LOG(work, "ExecuteAsync started for filter='%s'", work->filter.c_str()); + + struct err_data err_msg; + JqFilterWrapper* wrapper; + + wrapper = cache.get(work->filter); + if (wrapper == nullptr) { + ASYNC_DEBUG_LOG(work, "Creating new jq wrapper for filter='%s'", work->filter.c_str()); + jq_state* jq; + jq = jq_init(); + jq_set_error_cb(jq, throw_err_cb, &err_msg); + if (!jq_compile(jq, work->filter.c_str())) { + ASYNC_DEBUG_LOG(work, "jq compilation failed"); + work->error = err_msg.buf; + work->success = false; + return; + } + wrapper=new JqFilterWrapper(jq, work->filter); + cache.put(work->filter, wrapper ); + } + + jv input = jv_parse_sized(work->json.c_str(), work->json.size()); + ASYNC_DEBUG_LOG(work, "JSON input parsed"); if (!jv_is_valid(input)) { - info.GetReturnValue().Set(Nan::Null()); + ASYNC_DEBUG_LOG(work, "Invalid JSON input"); + work->error = "Invalid JSON input"; + work->success = false; jv_free(input); + cache.dec_refcnt(wrapper); + wrapper->unlock(); + return; } + wrapper->lock(); + jq_start(wrapper->get_jq(), input, 0); + ASYNC_DEBUG_LOG(work, "jq execution started"); - jq_start(jq, input, 0); - jv result = jq_next(jq); + jv result=jq_next(wrapper->get_jq()); + if(jv_get_kind(result) == JV_KIND_INVALID){ + jv msg = jv_invalid_get_msg(jv_copy(result)); - v8::Local ret = Nan::New(); - jv_object_to_v8("value", result, ret); + if (jv_get_kind(msg) == JV_KIND_STRING) { + work->error = std::string("jq: error: ") + jv_string_value(msg); + jv_free(msg); + work->success=false; + }else{ + work->is_undefined = true; + work->success=true; + } + }else{ + jv dump = jv_dump_string(result, JV_PRINT_INVALID); + if(jv_is_valid(dump)){ + work->result = jv_string_value(dump); + work->success = true; + }else{ + ASYNC_DEBUG_LOG(work, "failed to get result"); + work->error = "failed to get result"; + work->success = false; + } + jv_free(dump); + } + wrapper->unlock(); + cache.dec_refcnt(wrapper); - info.GetReturnValue().Set(ret); + ASYNC_DEBUG_LOG(work, "jq execution finished - got result, %s", work->result.c_str()); } - -std::string FromV8String(v8::Local val) { - Nan::Utf8String keyUTF8(val); - return std::string(*keyUTF8); +void reject_with_error_message(napi_env env, napi_deferred deferred, std::string error_message){ + napi_value error; + napi_create_string_utf8(env, error_message.c_str(), NAPI_AUTO_LENGTH, &error); + napi_value error_obj; + napi_create_object(env, &error_obj); + napi_set_named_property(env, error_obj, "message", error); + napi_reject_deferred(env, deferred, error_obj); } -void Exec(const Nan::FunctionCallbackInfo& info) { - if (info.Length() < 2) { - Nan::ThrowTypeError("Wrong number of arguments"); - return; +void CompleteAsync(napi_env env, napi_status status, void* data) { + AsyncWork* work = static_cast(data); + bool cleanup_done = false; + + auto cleanup = [&]() { + if (!cleanup_done) { + napi_delete_async_work(env, work->async_work); + ASYNC_DEBUG_LOG(work, "Deleting AsyncWork"); + delete work; + cleanup_done = true; + } + }; + + if(status != napi_ok || !work->success){ + std::string error_message = work->error; + if(error_message == ""){ + error_message = "Got error from async work"; + } + reject_with_error_message(env, work->deferred, error_message); + cleanup(); + return; + } + napi_handle_scope scope; + status = napi_open_handle_scope(env, &scope); + if (status != napi_ok) { + reject_with_error_message(env, work->deferred, "Failed to create handle scope"); + cleanup(); + return; + } + + napi_value ret; + + status=napi_create_object(env, &ret); + + jv result_jv; + if(work->is_undefined){ + result_jv = jv_invalid(); + }else{ + result_jv= jv_parse(work->result.c_str()); } + std::string err_msg_conversion; + bool success = jv_object_to_napi("value", env, result_jv, ret,err_msg_conversion); + jv_free(result_jv); - if (!info[0]->IsString() || !info[1]->IsString()) { - Nan::ThrowTypeError("Wrong arguments"); + if(!success){ + reject_with_error_message(env, work->deferred, err_msg_conversion); + napi_close_handle_scope(env, scope); return; } + napi_resolve_deferred(env, work->deferred, ret); + cleanup(); + napi_close_handle_scope(env, scope); +} + + - std::string json = FromV8String(Nan::To(info[0]).ToLocalChecked()); - std::string filter = FromV8String(Nan::To(info[1]).ToLocalChecked()); +napi_value ExecAsync(napi_env env, napi_callback_info info) { + napi_handle_scope scope; - jq_exec(json, filter, info); + size_t argc = 2; + napi_value args[2]; + napi_value promise; + + napi_get_cb_info(env, info, &argc, args, nullptr, nullptr); + + if (argc < 2) { + napi_throw_type_error(env, nullptr, "Wrong number of arguments. Expected 2."); + return nullptr; + } + + AsyncWork* work = new AsyncWork(); + work->json = FromNapiString(env, args[0]); + if(work->json == ""){ + napi_throw_error(env, nullptr, "Invalid JSON input"); + return nullptr; + } + work->filter = FromNapiString(env, args[1]); + if(work->filter == ""){ + napi_throw_error(env, nullptr, "Invalid filter input"); + return nullptr; + } + work->success = false; + + napi_create_promise(env, &work->deferred, &promise); + + napi_value resource_name; + napi_create_string_utf8(env, "ExecAsync", NAPI_AUTO_LENGTH, &resource_name); + + napi_create_async_work(env, nullptr, resource_name, ExecuteAsync, CompleteAsync, work, &work->async_work); + napi_queue_async_work(env, work->async_work); + + return promise; } -void Init(v8::Local exports) { - v8::Local context = exports->GetCreationContext().ToLocalChecked(); - (void)exports->Set(context, - Nan::New("exec").ToLocalChecked(), - Nan::New(Exec)->GetFunction(context).ToLocalChecked()); +// napi_value SetDebugMode(napi_env env, napi_callback_info info) { +// size_t argc = 1; +// napi_value args[1]; +// bool enable; + +// napi_get_cb_info(env, info, &argc, args, nullptr, nullptr); + +// if (argc < 1) { +// napi_throw_type_error(env, nullptr, "Wrong number of arguments"); +// return nullptr; +// } + +// napi_get_value_bool(env, args[0], &enable); +// debug_enabled = enable; +// DEBUG_LOG("Debug mode %s", enable ? "enabled" : "disabled"); + +// napi_value result; +// napi_get_boolean(env, debug_enabled, &result); +// return result; +// } + +// napi_value GetCacheStats(napi_env env, napi_callback_info info) { +// napi_value result; +// napi_create_object(env, &result); +// struct rusage usage; +// getrusage(RUSAGE_SELF, &usage); +// napi_value maxrss; +// napi_create_int64(env, usage.ru_maxrss, &maxrss); +// napi_set_named_property(env, result, "max_rss", maxrss); +// return result; +// } + +napi_value SetCacheSize(napi_env env, napi_callback_info info) { + size_t argc = 1; + napi_value args[1]; + int64_t new_size; + + napi_get_cb_info(env, info, &argc, args, nullptr, nullptr); + napi_status status; + if (argc < 1) { + napi_throw_type_error(env, nullptr, "Wrong number of arguments"); + return nullptr; + } + + status=napi_get_value_int64(env, args[0], &new_size); + if(!CheckNapiStatus(env,status,"error loading int64")){ + return nullptr; + } + if (new_size <= 0) { + napi_throw_error(env, nullptr, "Cache size must be positive"); + return nullptr; + } + + DEBUG_LOG("Changing cache size from %zu to %lld", global_cache_size, new_size); + size_t old_size = global_cache_size; + + global_cache_size = validate_cache_size(static_cast(new_size)); + cache.resize(global_cache_size); // Update cache size + + napi_value result; + napi_create_int64(env, global_cache_size, &result); + return result; +} + +napi_value Init(napi_env env, napi_value exports) { + napi_value exec_sync, exec_async, cache_size_fn,cache_stats_fn; + + napi_create_function(env, "execSync", NAPI_AUTO_LENGTH, ExecSync, nullptr, &exec_sync); + napi_create_function(env, "execAsync", NAPI_AUTO_LENGTH, ExecAsync, nullptr, &exec_async); + // napi_create_function(env, "setDebugMode", NAPI_AUTO_LENGTH, SetDebugMode, nullptr, &debug_fn); + napi_create_function(env, "setCacheSize", NAPI_AUTO_LENGTH, SetCacheSize, nullptr, &cache_size_fn); + // napi_create_function(env, "getCacheStats", NAPI_AUTO_LENGTH, GetCacheStats, nullptr, &cache_stats_fn); + napi_set_named_property(env, exports, "execSync", exec_sync); + napi_set_named_property(env, exports, "execAsync", exec_async); + // napi_set_named_property(env, exports, "setDebugMode", debug_fn); + napi_set_named_property(env, exports, "setCacheSize", cache_size_fn); + // napi_set_named_property(env, exports, "getCacheStats", cache_stats_fn); + return exports; } -NODE_MODULE(exec, Init) +NAPI_MODULE(NODE_GYP_MODULE_NAME, Init) diff --git a/src/binding.h b/src/binding.h index 9b4cae8..5cb1dd1 100644 --- a/src/binding.h +++ b/src/binding.h @@ -1,13 +1,12 @@ #ifndef SRC_BINDING_H_ #define SRC_BINDING_H_ -#include +#include -extern "C" -{ +extern "C" { #include "jq.h" } #include -#endif // SRC_BINDING_H_ +#endif diff --git a/test/santiy-async.test.js b/test/santiy-async.test.js new file mode 100644 index 0000000..0feaf6c --- /dev/null +++ b/test/santiy-async.test.js @@ -0,0 +1,171 @@ +const jq = require('../lib'); + +describe('jq - async', () => { + it('should break', async () => { + const json = { foo2: 'bar' }; + const input = 'foo'; + const result = await jq.execAsync(json, input); + + expect(result).toBe(null); + }), + it('should break for invalid input', async () => { + const json = { foo2: 'bar' }; + const input = 123; + const result = await jq.execAsync(json, input); + + expect(result).toBe(null); + }), + it('should break for invalid input', async () => { + const json = { foo2: 'bar' }; + const input = undefined; + const result = await jq.execAsync(json, input); + + expect(result).toBe(null); + }), + it('should break for invalid input', async () => { + const json = { foo2: 'bar' }; + const input = null; + const result = await jq.execAsync(json, input); + + expect(result).toBe(null); + }), + it('string should work', async () => { + const json = { foo: 'bar' }; + const input = '.foo'; + const result = await jq.execAsync(json, input); + + expect(result).toBe('bar'); + }); + + it('number should work', async ()=> { + const json = { foo: 1 }; + const input = '.foo'; + const result = await jq.execAsync(json, input); + + expect(result).toBe(1); + }); + + it ('should return null', async () => { + const json = { foo: 'bar' }; + const input = '.bar'; + const result = await jq.execAsync(json, input); + + expect(result).toBe(null); + }); + + it ('should return array with object', async () => { + const json = { foo: ['bar'] }; + const input = '.foo'; + const result = await jq.execAsync(json, input); + + expect(result[0]).toBe('bar'); + }); + + it ('should return an item of an array', async () => { + const json = { foo: ['bar'] }; + const input = '.foo[0]'; + const result = await jq.execAsync(json, input); + + expect(result).toBe('bar'); + }) + + it ('should return array with objects', async () => { + const json = { foo: [{ bar: 'bar' }] }; + const input = '.foo'; + const result = await jq.execAsync(json, input); + + expect(result[0].bar).toBe('bar'); + }); + + it ('should return boolean', async () => { + const json = { foo: true }; + const input = '.foo'; + const result = await jq.execAsync(json, input); + + expect(result).toBe(true); + }); + it ('should return object', async () => { + const json = { foo: {prop1: "1"} }; + const input = '.foo'; + const result = await jq.execAsync(json, input); + + expect(result.prop1).toBe("1"); + }) + + it ('should return recursed obj', async () => { + const json = { foo: {obj: { obj2: { num: 1, string: "str"} }} }; + const input = '.foo'; + const result = await jq.execAsync(json, input); + + expect(result.obj.obj2.num).toBe(1); + expect(result.obj.obj2.string).toBe("str"); + }), + + it ('should return recursed obj', async () => { + const json = { foo: { obj: { obj2: { num: 1, string: "str", bool: true} }} }; + const input = '.foo'; + const result = await jq.execAsync(json, input); + + expect(result.obj.obj2.num).toBe(1); + expect(result.obj.obj2.string).toBe("str"); + expect(result.obj.obj2.bool).toBe(true); + }) + + it ('should return null on invalid json', async () => { + const json = "foo"; + const input = '.foo'; + const result = await jq.execAsync(json, input); + + expect(result).toBe(null); + }) + + it('should excape \'\' to ""', async () => { + const json = { foo: 'com' }; + const input = "'https://url.' + .foo"; + const result = await jq.execAsync(json, input); + + expect(result).toBe('https://url.com'); + }) + + it('should not escape \' in the middle of the string', async () => { + const json = { foo: 'com' }; + const input = "\"https://'url.\" + 'test.' + .foo"; + const result = await jq.execAsync(json, input); + + expect(result).toBe("https://'url.test.com"); + }); + + it ('should run a jq function succesfully', async () => { + const json = { foo: 'bar' }; + const input = '.foo | gsub("bar";"foo")'; + const result = await jq.execAsync(json, input); + + expect(result).toBe('foo'); + }) + + it ('Testing multiple the \'\' in the same expression', async () => { + const json = { foo: 'bar' }; + const input = "'https://some.random.url' + .foo + '-1' + '.' + .foo + '.' + 'longgggg' + .foo + ')test(' + .foo + 'testadsftets'"; + const result = await jq.execAsync(json, input); + + expect(result).toBe('https://some.random.urlbar-1.bar.longggggbar)test(bartestadsftets'); + }) + + it('test disable env', async () => { + expect(await jq.execAsync({}, 'env', {enableEnv: false})).toEqual({}); + expect(await jq.execAsync({}, 'env', {enableEnv: true})).not.toEqual({}); + expect(await jq.execAsync({}, 'env', {})).toEqual({}); + expect(await jq.execAsync({}, 'env')).toEqual({}); + }) + + it('test throw on error', async () => { + await expect(jq.execAsync({}, 'foo', {throwOnError: true})).rejects.toThrow("jq: compile error: foo/0 is not defined at , line 1:"); + await expect(jq.execAsync({}, '1/0', {throwOnError: true})).rejects.toThrow("number (1) and number (0) cannot be divided because the divisor is zero"); + await expect(jq.execAsync({}, '{', {throwOnError: true})).rejects.toThrow("jq: compile error: syntax error, unexpected end of file (Unix shell quoting issues?) at , line 1:"); + await expect(jq.execAsync({}, '{(0):1}', {throwOnError: true})).rejects.toThrow("jq: compile error: Cannot use number (0) as object key at , line 1:"); + await expect(jq.execAsync({}, 'if true then 1 else 0', {throwOnError: true})).rejects.toThrow("jq: compile error: Possibly unterminated 'if' statement at , line 1:"); + await expect(jq.execAsync({}, 'null | map(.+1)', {throwOnError: true})).rejects.toThrow("jq: error: Cannot iterate over null (null)"); + await expect(jq.execAsync({foo: "bar"}, '.foo + 1', {throwOnError: true})).rejects.toThrow("jq: error: string (\"bar\") and number (1) cannot be added"); + }) +}) + diff --git a/test/santiy.test.js b/test/santiy.test.js index 8ecbb04..655053d 100644 --- a/test/santiy.test.js +++ b/test/santiy.test.js @@ -160,8 +160,8 @@ describe('jq', () => { it('test throw on error', () => { expect(() => { jq.exec({}, 'foo', {throwOnError: true}) }).toThrow("jq: compile error: foo/0 is not defined at , line 1:"); - expect(() => { jq.exec({}, '1/0', {throwOnError: true}) }).toThrow("jq: compile error: Division by zero? at , line 1:"); - expect(() => { jq.exec({}, '{', {throwOnError: true}) }).toThrow("jq: compile error: syntax error, unexpected $end (Unix shell quoting issues?) at , line 1:"); + expect(() => { jq.exec({}, '1/0', {throwOnError: true}) }).toThrow("number (1) and number (0) cannot be divided because the divisor is zero"); + expect(() => { jq.exec({}, '{', {throwOnError: true}) }).toThrow("jq: compile error: syntax error, unexpected end of file (Unix shell quoting issues?) at , line 1:"); expect(() => { jq.exec({}, '{(0):1}', {throwOnError: true}) }).toThrow("jq: compile error: Cannot use number (0) as object key at , line 1:"); expect(() => { jq.exec({}, 'if true then 1 else 0', {throwOnError: true}) }).toThrow("jq: compile error: Possibly unterminated 'if' statement at , line 1:"); expect(() => { jq.exec({}, 'null | map(.+1)', {throwOnError: true}) }).toThrow("jq: error: Cannot iterate over null (null)"); diff --git a/test/template-async.test.js b/test/template-async.test.js new file mode 100644 index 0000000..96a0fc1 --- /dev/null +++ b/test/template-async.test.js @@ -0,0 +1,182 @@ +const jq = require('../lib'); + +describe('template', () => { + it('should break', async () => { + const json = { foo2: 'bar' }; + const input = '{{.foo}}'; + const result = await jq.renderRecursivelyAsync(json, input); + + expect(result).toBe(null); + }); + it('non template should work', async () => { + const json = { foo2: 'bar' }; + const render = async (input) => await jq.renderRecursivelyAsync(json, input); + + expect(await render(123)).toBe(123); + expect(await render(undefined)).toBe(undefined); + expect(await render(null)).toBe(null); + expect(await render(true)).toBe(true); + expect(await render(false)).toBe(false); + }); + it('different types should work', async () => { + const input = '{{.foo}}'; + const render = async (json) => await jq.renderRecursivelyAsync(json, input); + + expect(await render({ foo: 'bar' })).toBe('bar'); + expect(await render({ foo: 1 })).toBe(1); + expect(await render({ foo: true })).toBe(true); + expect(await render({ foo: null })).toBe(null); + expect(await render({ foo: undefined })).toBe(null); + expect(await render({ foo: ['bar'] })).toEqual(['bar']); + expect(await render({ foo: [{ bar: 'bar' }] })).toEqual([{ bar: 'bar' }]); + expect(await render({ foo: {prop1: "1"} })).toEqual({prop1: "1"}); + expect(await render({ foo: {obj: { obj2: { num: 1, string: "str"} }} })).toEqual({obj: { obj2: { num: 1, string: "str"} }}); + expect(await render({ foo: { obj: { obj2: { num: 1, string: "str", bool: true} }} })).toEqual({ obj: { obj2: { num: 1, string: "str", bool: true} }}); + }); + it ('should return undefined', async () => { + const json = { foo: 'bar' }; + const input = '{{empty}}'; + const result = await jq.renderRecursivelyAsync(json, input); + + expect(result).toBe(undefined); + }); + it ('should return null on invalid json', async () => { + const json = "foo"; + const input = '{{.foo}}'; + const result = await jq.renderRecursivelyAsync(json, input); + console.log('!!!!!',result); + expect(result).toBe(null); + }); + it('should excape \'\' to ""', async () => { + const json = { foo: 'com' }; + const input = "{{'https://url.' + .foo}}"; + const result = await jq.renderRecursivelyAsync(json, input); + + expect(result).toBe('https://url.com'); + }); + it('should not escape \' in the middle of the string', async () => { + const json = { foo: 'com' }; + const input = "{{\"https://'url.\" + 'test.' + .foo}}"; + const result = await jq.renderRecursivelyAsync(json, input); + + expect(result).toBe("https://'url.test.com"); + }); + it ('should run a jq function succesfully', async () => { + const json = { foo: 'bar' }; + const input = '{{.foo | gsub("bar";"foo")}}'; + const result = await jq.renderRecursivelyAsync(json, input); + + expect(result).toBe('foo'); + }); + it ('Testing multiple the \'\' in the same expression', async () => { + const json = { foo: 'bar' }; + const input = "{{'https://some.random.url' + .foo + '-1' + '.' + .foo + '.' + 'longgggg' + .foo + ')test(' + .foo + 'testadsftets'}}"; + const result = await jq.renderRecursivelyAsync(json, input); + + expect(result).toBe('https://some.random.urlbar-1.bar.longggggbar)test(bartestadsftets'); + }); + it ('Testing multiple the \'\' in the same expression', async () => { + const json = { foo: 'bar' }; + const input = "{{'https://some.random.url' + .foo + '-1' + '.' + .foo + '.' + 'longgggg' + .foo + ')test(' + .foo + 'testadsftets'}}"; + const result = await jq.renderRecursivelyAsync(json, input); + + expect(result).toBe('https://some.random.urlbar-1.bar.longggggbar)test(bartestadsftets'); + }); + it('should break for invalid template', async () => { + const json = { foo: 'bar' }; + const render = async (input) => async () => await jq.renderRecursivelyAsync(json, input); + + expect(await render('prefix{{.foo}postfix')).rejects.toThrow('Found opening double braces in index 6 without closing double braces'); + expect(await render('prefix{.foo}}postfix')).rejects.toThrow('Found closing double braces in index 11 without opening double braces'); + expect(await render('prefix{{ .foo {{ }}postfix')).rejects.toThrow('Found double braces in index 14 inside other one in index 6'); + expect(await render('prefix{{ .foo }} }}postfix')).rejects.toThrow('Found closing double braces in index 17 without opening double braces'); + expect(await render('prefix{{ .foo }} }}postfix')).rejects.toThrow('Found closing double braces in index 17 without opening double braces'); + expect(await render('prefix{{ "{{" + .foo }} }}postfix')).rejects.toThrow('Found closing double braces in index 24 without opening double braces'); + expect(await render('prefix{{ \'{{\' + .foo }} }}postfix')).rejects.toThrow('Found closing double braces in index 24 without opening double braces'); + expect(await render({'{{1}}': 'bar'})).rejects.toThrow('Evaluated object key should be undefined, null or string. Original key: {{1}}, evaluated to: 1'); + expect(await render({'{{true}}': 'bar'})).rejects.toThrow('Evaluated object key should be undefined, null or string. Original key: {{true}}, evaluated to: true'); + expect(await render({'{{ {} }}': 'bar'})).rejects.toThrow('Evaluated object key should be undefined, null or string. Original key: {{ {} }}, evaluated to: {}'); + }); + it('should concat string and other types', async () => { + const input = 'https://some.random.url?q={{.foo}}'; + const render = async (json) => await jq.renderRecursivelyAsync(json, input); + + expect(await render({ foo: 'bar' })).toBe('https://some.random.url?q=bar'); + expect(await render({ foo: 1 })).toBe('https://some.random.url?q=1'); + expect(await render({ foo: false })).toBe('https://some.random.url?q=false'); + expect(await render({ foo: null })).toBe('https://some.random.url?q=null'); + expect(await render({ foo: undefined })).toBe('https://some.random.url?q=null'); + expect(await render({ foo: [1] })).toBe('https://some.random.url?q=[1]'); + expect(await render({ foo: {bar: 'bar'} })).toBe('https://some.random.url?q={\"bar\":\"bar\"}'); + }); + it('testing multiple template blocks', async () => { + const json = {str: 'bar', num: 1, bool: true, 'null': null, arr: ['foo'], obj: {bar: 'bar'}}; + const input = 'https://some.random.url?str={{.str}}&num={{.num}}&bool={{.bool}}&null={{.null}}&arr={{.arr}}&obj={{.obj}}'; + const result = await jq.renderRecursivelyAsync(json, input); + + expect(result).toBe("https://some.random.url?str=bar&num=1&bool=true&null=null&arr=[\"foo\"]&obj={\"bar\":\"bar\"}"); + }); + it('testing conditional key', async () => { + const json = {}; + const render = async (input) => await jq.renderRecursivelyAsync(json, input); + + expect(await render({'{{empty}}': 'bar'})).toEqual({}); + expect(await render({'{{null}}': 'bar'})).toEqual({}); + expect(await render({'{{""}}': 'bar'})).toEqual({}); + expect(await render({'{{\'\'}}': 'bar'})).toEqual({}); + }); + it('testing spread key', async () => { + const json = { foo: "bar" }; + const render = async (input) => await jq.renderRecursivelyAsync(json, input); + + expect(await render({ "{{spreadValue()}}": { foo: "bar" } })).toEqual({foo: "bar"}); + expect(await render({ " {{ spreadValue( ) }} ": { foo: "bar" } })).toEqual({foo: "bar"}); + expect(await render({ "{{spreadValue()}}": "{{ . }}" })).toEqual({ foo: "bar" }); + }); + it('recursive templates should work', async () => { + const json = { foo: 'bar', bar: 'foo' }; + const render = async (input) => await jq.renderRecursivelyAsync(json, input); + + expect(await render({'{{.foo}}': '{{.bar}}{{.foo}}'})).toEqual({bar: 'foobar'}); + expect(await render({'{{.foo}}': {foo: '{{.foo}}'}})).toEqual({bar: {foo: 'bar'}}); + expect(await render([1, true, null, undefined, '{{.foo}}', 'https://{{.bar}}.com'])).toEqual([1, true, null, undefined, 'bar', 'https://foo.com']); + expect(await render([['{{.bar}}{{.foo}}'], 1, '{{.bar | ascii_upcase}}'])).toEqual([['foobar'], 1, 'FOO']); + expect(await render([{'{{.bar}}': [false, '/foo/{{.foo + .bar}}']}])).toEqual([{foo: [false, '/foo/barfoo']}]); + expect(await render({foo: [{bar: '{{1}}'}, '{{empty}}']})).toEqual({foo: [{bar: 1}, undefined]}); + }); + it('should accept quotes outside of template', async () => { + const json = { foo: 'bar', bar: 'foo' }; + const render = async (input) => await jq.renderRecursivelyAsync(json, input); + + expect(await render('"{{.foo}}"')).toEqual('"bar"'); + expect(await render('\'{{.foo}}\'')).toEqual('\'bar\''); + }); + it('should accept escaped quotes inside jq template', async () => { + const json = { foo: 'bar', bar: 'foo' }; + const render = async (input) => await jq.renderRecursivelyAsync(json, input); + + expect(await render('{{"\\"foo\\""}}')).toEqual('"foo"'); + }); + it('test disable env', async () => { + expect(await jq.renderRecursivelyAsync({}, '{{env}}', {enableEnv: false})).toEqual({}); + expect(await jq.renderRecursivelyAsync({}, '{{env}}', {enableEnv: true})).not.toEqual({}); + expect(await jq.renderRecursivelyAsync({}, '{{env}}', {})).toEqual({}); + expect(await jq.renderRecursivelyAsync({}, '{{env}}')).toEqual({}); + }) + it('test throw on error', async () => { + expect(async () => { await jq.renderRecursivelyAsync({}, '{{foo}}', {throwOnError: true}) }).rejects.toThrow("jq: compile error: foo/0 is not defined at , line 1:"); + expect(async () => { await jq.renderRecursivelyAsync({}, '{{1/0}}', {throwOnError: true}) }).rejects.toThrow("number (1) and number (0) cannot be divided because the divisor is zero"); + expect(async () => { await jq.renderRecursivelyAsync({}, '{{{}}', {throwOnError: true}) }).rejects.toThrow("jq: compile error: syntax error, unexpected end of file (Unix shell quoting issues?) at , line 1:"); + expect(async () => { await jq.renderRecursivelyAsync({}, '{{ {(0):1} }}', {throwOnError: true}) }).rejects.toThrow("jq: compile error: Cannot use number (0) as object key at , line 1:"); + expect(async () => { await jq.renderRecursivelyAsync({}, '{{if true then 1 else 0}}', {throwOnError: true}) }).rejects.toThrow("jq: compile error: Possibly unterminated 'if' statement at , line 1:"); + expect(async () => { await jq.renderRecursivelyAsync({}, '{{null | map(.+1)}}', {throwOnError: true}) }).rejects.toThrow("jq: error: Cannot iterate over null (null)"); + expect(async () => { await jq.renderRecursivelyAsync({foo: "bar"}, '{{.foo + 1}}', {throwOnError: true}) }).rejects.toThrow("jq: error: string (\"bar\") and number (1) cannot be added"); + expect(async () => { await jq.renderRecursivelyAsync({}, '{{foo}}/{{bar}}', {throwOnError: true}) }).rejects.toThrow("jq: compile error: foo/0 is not defined at , line 1:"); + expect(async () => { await jq.renderRecursivelyAsync({}, '/{{foo}}/', {throwOnError: true}) }).rejects.toThrow("jq: compile error: foo/0 is not defined at , line 1:"); + expect(async () => { await jq.renderRecursivelyAsync({}, { "{{ spreadValue() }}": "str" }, { throwOnError: true }) }) + .rejects.toThrow('Evaluated value should be an object if the key is {{ spreadValue() }}. Original value: str, evaluated to: "str"'); + expect(async () => { await jq.renderRecursivelyAsync({}, { "{{ spreadValue() }}": "{{ \"str\" }}" }, { throwOnError: true }) }) + .rejects.toThrow('Evaluated value should be an object if the key is {{ spreadValue() }}. Original value: {{ \"str\" }}, evaluated to: "str"'); + }) +}) + diff --git a/test/template.test.js b/test/template.test.js index 6478a0d..c74a256 100644 --- a/test/template.test.js +++ b/test/template.test.js @@ -44,7 +44,6 @@ describe('template', () => { const json = "foo"; const input = '{{.foo}}'; const result = jq.renderRecursively(json, input); - expect(result).toBe(null); }); it('should excape \'\' to ""', () => { @@ -165,8 +164,8 @@ describe('template', () => { }) it('test throw on error', () => { expect(() => { jq.renderRecursively({}, '{{foo}}', {throwOnError: true}) }).toThrow("jq: compile error: foo/0 is not defined at , line 1:"); - expect(() => { jq.renderRecursively({}, '{{1/0}}', {throwOnError: true}) }).toThrow("jq: compile error: Division by zero? at , line 1:"); - expect(() => { jq.renderRecursively({}, '{{{}}', {throwOnError: true}) }).toThrow("jq: compile error: syntax error, unexpected $end (Unix shell quoting issues?) at , line 1:"); + expect(() => { jq.renderRecursively({}, '{{1/0}}', {throwOnError: true}) }).toThrow("number (1) and number (0) cannot be divided because the divisor is zero"); + expect(() => { jq.renderRecursively({}, '{{{}}', {throwOnError: true}) }).toThrow("jq: compile error: syntax error, unexpected end of file (Unix shell quoting issues?) at , line 1:"); expect(() => { jq.renderRecursively({}, '{{ {(0):1} }}', {throwOnError: true}) }).toThrow("jq: compile error: Cannot use number (0) as object key at , line 1:"); expect(() => { jq.renderRecursively({}, '{{if true then 1 else 0}}', {throwOnError: true}) }).toThrow("jq: compile error: Possibly unterminated 'if' statement at , line 1:"); expect(() => { jq.renderRecursively({}, '{{null | map(.+1)}}', {throwOnError: true}) }).toThrow("jq: error: Cannot iterate over null (null)");