Skip to content

Commit 959d320

Browse files
kravets-levkokravets-levko
authored
Fix NULL handling in Arrow result handler (#195)
Signed-off-by: Levko Kravets <[email protected]>
1 parent fd36b2e commit 959d320

File tree

5 files changed

+371
-1
lines changed

5 files changed

+371
-1
lines changed

‎lib/result/ArrowResult.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,8 @@ export default class ArrowResult implements IOperationResult {
141141
this.schema.forEach((column) => {
142142
const typeDescriptor = column.typeDesc.types[0]?.primitiveEntry;
143143
const field = column.columnName;
144-
result[field] = convertThriftValue(typeDescriptor, record[field]);
144+
const value = record[field];
145+
result[field] = value === null ? null : convertThriftValue(typeDescriptor, value);
145146
});
146147

147148
return result;

‎tests/unit/result/ArrowResult.test.js

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
const { expect } = require('chai');
2+
const fs = require('fs');
3+
const path = require('path');
24
const ArrowResult = require('../../../dist/result/ArrowResult').default;
35

46
const sampleThriftSchema = {
@@ -66,6 +68,22 @@ const sampleRowSet4 = {
6668
arrowBatches: [sampleArrowBatch],
6769
};
6870

71+
const thriftSchemaAllNulls = JSON.parse(
72+
fs.readFileSync(path.join(__dirname, 'fixtures/thriftSchemaAllNulls.json')).toString('utf-8'),
73+
);
74+
75+
const arrowSchemaAllNulls = fs.readFileSync(path.join(__dirname, 'fixtures/arrowSchemaAllNulls.arrow'));
76+
77+
const rowSetAllNulls = {
78+
startRowOffset: 0,
79+
arrowBatches: [
80+
{
81+
batch: fs.readFileSync(path.join(__dirname, 'fixtures/dataAllNulls.arrow')),
82+
rowCount: 1,
83+
},
84+
],
85+
};
86+
6987
describe('ArrowResult', () => {
7088
it('should not buffer any data', async () => {
7189
const context = {};
@@ -95,4 +113,37 @@ describe('ArrowResult', () => {
95113
const result = new ArrowResult(context);
96114
expect(await result.getValue([sampleRowSet4])).to.be.deep.eq([]);
97115
});
116+
117+
it('should detect nulls', async () => {
118+
const context = {};
119+
const result = new ArrowResult(context, thriftSchemaAllNulls, arrowSchemaAllNulls);
120+
expect(await result.getValue([rowSetAllNulls])).to.be.deep.eq([
121+
{
122+
boolean_field: null,
123+
124+
tinyint_field: null,
125+
smallint_field: null,
126+
int_field: null,
127+
bigint_field: null,
128+
129+
float_field: null,
130+
double_field: null,
131+
decimal_field: null,
132+
133+
string_field: null,
134+
char_field: null,
135+
varchar_field: null,
136+
137+
timestamp_field: null,
138+
date_field: null,
139+
day_interval_field: null,
140+
month_interval_field: null,
141+
142+
binary_field: null,
143+
144+
struct_field: null,
145+
array_field: null,
146+
},
147+
]);
148+
});
98149
});

‎tests/unit/result/fixtures/arrowSchemaAllNulls.arrow

3.55 KB
Binary file not shown.

‎tests/unit/result/fixtures/dataAllNulls.arrow

1.41 KB
Binary file not shown.

‎tests/unit/result/fixtures/thriftSchemaAllNulls.json

Lines changed: 318 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,318 @@
1+
{
2+
"columns": [
3+
{
4+
"columnName": "boolean_field",
5+
"typeDesc": {
6+
"types": [
7+
{
8+
"primitiveEntry": { "type": 0, "typeQualifiers": null },
9+
"arrayEntry": null,
10+
"mapEntry": null,
11+
"structEntry": null,
12+
"unionEntry": null,
13+
"userDefinedTypeEntry": null
14+
}
15+
]
16+
},
17+
"position": 1,
18+
"comment": ""
19+
},
20+
{
21+
"columnName": "tinyint_field",
22+
"typeDesc": {
23+
"types": [
24+
{
25+
"primitiveEntry": { "type": 1, "typeQualifiers": null },
26+
"arrayEntry": null,
27+
"mapEntry": null,
28+
"structEntry": null,
29+
"unionEntry": null,
30+
"userDefinedTypeEntry": null
31+
}
32+
]
33+
},
34+
"position": 2,
35+
"comment": ""
36+
},
37+
{
38+
"columnName": "smallint_field",
39+
"typeDesc": {
40+
"types": [
41+
{
42+
"primitiveEntry": { "type": 2, "typeQualifiers": null },
43+
"arrayEntry": null,
44+
"mapEntry": null,
45+
"structEntry": null,
46+
"unionEntry": null,
47+
"userDefinedTypeEntry": null
48+
}
49+
]
50+
},
51+
"position": 3,
52+
"comment": ""
53+
},
54+
{
55+
"columnName": "int_field",
56+
"typeDesc": {
57+
"types": [
58+
{
59+
"primitiveEntry": { "type": 3, "typeQualifiers": null },
60+
"arrayEntry": null,
61+
"mapEntry": null,
62+
"structEntry": null,
63+
"unionEntry": null,
64+
"userDefinedTypeEntry": null
65+
}
66+
]
67+
},
68+
"position": 4,
69+
"comment": ""
70+
},
71+
{
72+
"columnName": "bigint_field",
73+
"typeDesc": {
74+
"types": [
75+
{
76+
"primitiveEntry": { "type": 4, "typeQualifiers": null },
77+
"arrayEntry": null,
78+
"mapEntry": null,
79+
"structEntry": null,
80+
"unionEntry": null,
81+
"userDefinedTypeEntry": null
82+
}
83+
]
84+
},
85+
"position": 5,
86+
"comment": ""
87+
},
88+
{
89+
"columnName": "float_field",
90+
"typeDesc": {
91+
"types": [
92+
{
93+
"primitiveEntry": { "type": 5, "typeQualifiers": null },
94+
"arrayEntry": null,
95+
"mapEntry": null,
96+
"structEntry": null,
97+
"unionEntry": null,
98+
"userDefinedTypeEntry": null
99+
}
100+
]
101+
},
102+
"position": 6,
103+
"comment": ""
104+
},
105+
{
106+
"columnName": "double_field",
107+
"typeDesc": {
108+
"types": [
109+
{
110+
"primitiveEntry": { "type": 6, "typeQualifiers": null },
111+
"arrayEntry": null,
112+
"mapEntry": null,
113+
"structEntry": null,
114+
"unionEntry": null,
115+
"userDefinedTypeEntry": null
116+
}
117+
]
118+
},
119+
"position": 7,
120+
"comment": ""
121+
},
122+
{
123+
"columnName": "decimal_field",
124+
"typeDesc": {
125+
"types": [
126+
{
127+
"primitiveEntry": {
128+
"type": 15,
129+
"typeQualifiers": {
130+
"qualifiers": {
131+
"scale": { "i32Value": 2, "stringValue": null },
132+
"precision": { "i32Value": 6, "stringValue": null }
133+
}
134+
}
135+
},
136+
"arrayEntry": null,
137+
"mapEntry": null,
138+
"structEntry": null,
139+
"unionEntry": null,
140+
"userDefinedTypeEntry": null
141+
}
142+
]
143+
},
144+
"position": 8,
145+
"comment": ""
146+
},
147+
{
148+
"columnName": "string_field",
149+
"typeDesc": {
150+
"types": [
151+
{
152+
"primitiveEntry": { "type": 7, "typeQualifiers": null },
153+
"arrayEntry": null,
154+
"mapEntry": null,
155+
"structEntry": null,
156+
"unionEntry": null,
157+
"userDefinedTypeEntry": null
158+
}
159+
]
160+
},
161+
"position": 9,
162+
"comment": ""
163+
},
164+
{
165+
"columnName": "char_field",
166+
"typeDesc": {
167+
"types": [
168+
{
169+
"primitiveEntry": { "type": 7, "typeQualifiers": null },
170+
"arrayEntry": null,
171+
"mapEntry": null,
172+
"structEntry": null,
173+
"unionEntry": null,
174+
"userDefinedTypeEntry": null
175+
}
176+
]
177+
},
178+
"position": 10,
179+
"comment": ""
180+
},
181+
{
182+
"columnName": "varchar_field",
183+
"typeDesc": {
184+
"types": [
185+
{
186+
"primitiveEntry": { "type": 7, "typeQualifiers": null },
187+
"arrayEntry": null,
188+
"mapEntry": null,
189+
"structEntry": null,
190+
"unionEntry": null,
191+
"userDefinedTypeEntry": null
192+
}
193+
]
194+
},
195+
"position": 11,
196+
"comment": ""
197+
},
198+
{
199+
"columnName": "timestamp_field",
200+
"typeDesc": {
201+
"types": [
202+
{
203+
"primitiveEntry": { "type": 8, "typeQualifiers": null },
204+
"arrayEntry": null,
205+
"mapEntry": null,
206+
"structEntry": null,
207+
"unionEntry": null,
208+
"userDefinedTypeEntry": null
209+
}
210+
]
211+
},
212+
"position": 12,
213+
"comment": ""
214+
},
215+
{
216+
"columnName": "date_field",
217+
"typeDesc": {
218+
"types": [
219+
{
220+
"primitiveEntry": { "type": 17, "typeQualifiers": null },
221+
"arrayEntry": null,
222+
"mapEntry": null,
223+
"structEntry": null,
224+
"unionEntry": null,
225+
"userDefinedTypeEntry": null
226+
}
227+
]
228+
},
229+
"position": 13,
230+
"comment": ""
231+
},
232+
{
233+
"columnName": "day_interval_field",
234+
"typeDesc": {
235+
"types": [
236+
{
237+
"primitiveEntry": { "type": 7, "typeQualifiers": null },
238+
"arrayEntry": null,
239+
"mapEntry": null,
240+
"structEntry": null,
241+
"unionEntry": null,
242+
"userDefinedTypeEntry": null
243+
}
244+
]
245+
},
246+
"position": 14,
247+
"comment": ""
248+
},
249+
{
250+
"columnName": "month_interval_field",
251+
"typeDesc": {
252+
"types": [
253+
{
254+
"primitiveEntry": { "type": 7, "typeQualifiers": null },
255+
"arrayEntry": null,
256+
"mapEntry": null,
257+
"structEntry": null,
258+
"unionEntry": null,
259+
"userDefinedTypeEntry": null
260+
}
261+
]
262+
},
263+
"position": 15,
264+
"comment": ""
265+
},
266+
{
267+
"columnName": "binary_field",
268+
"typeDesc": {
269+
"types": [
270+
{
271+
"primitiveEntry": { "type": 9, "typeQualifiers": null },
272+
"arrayEntry": null,
273+
"mapEntry": null,
274+
"structEntry": null,
275+
"unionEntry": null,
276+
"userDefinedTypeEntry": null
277+
}
278+
]
279+
},
280+
"position": 16,
281+
"comment": ""
282+
},
283+
{
284+
"columnName": "struct_field",
285+
"typeDesc": {
286+
"types": [
287+
{
288+
"primitiveEntry": { "type": 12, "typeQualifiers": null },
289+
"arrayEntry": null,
290+
"mapEntry": null,
291+
"structEntry": null,
292+
"unionEntry": null,
293+
"userDefinedTypeEntry": null
294+
}
295+
]
296+
},
297+
"position": 17,
298+
"comment": ""
299+
},
300+
{
301+
"columnName": "array_field",
302+
"typeDesc": {
303+
"types": [
304+
{
305+
"primitiveEntry": { "type": 10, "typeQualifiers": null },
306+
"arrayEntry": null,
307+
"mapEntry": null,
308+
"structEntry": null,
309+
"unionEntry": null,
310+
"userDefinedTypeEntry": null
311+
}
312+
]
313+
},
314+
"position": 18,
315+
"comment": ""
316+
}
317+
]
318+
}

0 commit comments

Comments
 (0)