Skip to content

Commit d44aa98

Browse files
cgivrecgivre
authored
DRILL-8501: Json Conversion UDF Not Respecting System JSON Options (#2921)
1 parent af62ace commit d44aa98

File tree

4 files changed

+516
-101
lines changed

4 files changed

+516
-101
lines changed

‎exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertFrom.java‎

Lines changed: 271 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -19,41 +19,99 @@
1919

2020

2121
import io.netty.buffer.DrillBuf;
22-
23-
import javax.inject.Inject;
24-
2522
import org.apache.drill.exec.expr.DrillSimpleFunc;
2623
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
2724
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
2825
import org.apache.drill.exec.expr.annotations.Output;
2926
import org.apache.drill.exec.expr.annotations.Param;
3027
import org.apache.drill.exec.expr.annotations.Workspace;
28+
import org.apache.drill.exec.expr.holders.BitHolder;
3129
import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
3230
import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
3331
import org.apache.drill.exec.expr.holders.VarBinaryHolder;
3432
import org.apache.drill.exec.expr.holders.VarCharHolder;
33+
import org.apache.drill.exec.server.options.OptionManager;
3534
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
3635

37-
public class JsonConvertFrom {
36+
import javax.inject.Inject;
3837

39-
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonConvertFrom.class);
38+
public class JsonConvertFrom {
4039

4140
private JsonConvertFrom() {
4241
}
4342

4443
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true)
4544
public static class ConvertFromJson implements DrillSimpleFunc {
4645

47-
@Param VarBinaryHolder in;
48-
@Inject DrillBuf buffer;
49-
@Workspace org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader;
46+
@Param
47+
VarBinaryHolder in;
5048

51-
@Output ComplexWriter writer;
49+
@Inject
50+
DrillBuf buffer;
51+
52+
@Inject
53+
OptionManager options;
54+
55+
@Workspace
56+
org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader;
57+
58+
@Output
59+
ComplexWriter writer;
5260

5361
@Override
5462
public void setup() {
63+
boolean allTextMode = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_ALL_TEXT_MODE);
64+
boolean readNumbersAsDouble = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE);
65+
66+
jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer)
67+
.defaultSchemaPathColumns()
68+
.allTextMode(allTextMode)
69+
.readNumbersAsDouble(readNumbersAsDouble)
70+
.build();
71+
}
72+
73+
@Override
74+
public void eval() {
75+
try {
76+
jsonReader.setSource(in.start, in.end, in.buffer);
77+
jsonReader.write(writer);
78+
buffer = jsonReader.getWorkBuf();
79+
} catch (Exception e) {
80+
throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e);
81+
}
82+
}
83+
}
84+
85+
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true)
86+
public static class ConvertFromJsonWithArgs implements DrillSimpleFunc {
87+
88+
@Param
89+
VarBinaryHolder in;
90+
91+
@Param
92+
BitHolder allTextModeHolder;
93+
94+
@Param
95+
BitHolder readNumbersAsDoubleHolder;
96+
97+
@Inject
98+
DrillBuf buffer;
99+
100+
@Workspace
101+
org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader;
102+
103+
@Output
104+
ComplexWriter writer;
105+
106+
@Override
107+
public void setup() {
108+
boolean allTextMode = allTextModeHolder.value == 1;
109+
boolean readNumbersAsDouble = readNumbersAsDoubleHolder.value == 1;
110+
55111
jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer)
56112
.defaultSchemaPathColumns()
113+
.allTextMode(allTextMode)
114+
.readNumbersAsDouble(readNumbersAsDouble)
57115
.build();
58116
}
59117

@@ -69,19 +127,79 @@ public void eval() {
69127
}
70128
}
71129

130+
72131
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true)
73132
public static class ConvertFromJsonVarchar implements DrillSimpleFunc {
74133

75-
@Param VarCharHolder in;
76-
@Inject DrillBuf buffer;
77-
@Workspace org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader;
134+
@Param
135+
VarCharHolder in;
78136

79-
@Output ComplexWriter writer;
137+
@Inject
138+
DrillBuf buffer;
139+
140+
@Inject
141+
OptionManager options;
142+
143+
@Workspace
144+
org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader;
145+
146+
@Output
147+
ComplexWriter writer;
80148

81149
@Override
82150
public void setup() {
151+
boolean allTextMode = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_ALL_TEXT_MODE);
152+
boolean readNumbersAsDouble = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE);
153+
83154
jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer)
84155
.defaultSchemaPathColumns()
156+
.allTextMode(allTextMode)
157+
.readNumbersAsDouble(readNumbersAsDouble)
158+
.build();
159+
}
160+
161+
@Override
162+
public void eval() {
163+
try {
164+
jsonReader.setSource(in.start, in.end, in.buffer);
165+
jsonReader.write(writer);
166+
buffer = jsonReader.getWorkBuf();
167+
} catch (Exception e) {
168+
throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e);
169+
}
170+
}
171+
}
172+
173+
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true)
174+
public static class ConvertFromJsonVarcharWithConfig implements DrillSimpleFunc {
175+
176+
@Param
177+
VarCharHolder in;
178+
179+
@Param
180+
BitHolder allTextModeHolder;
181+
182+
@Param
183+
BitHolder readNumbersAsDoubleHolder;
184+
185+
@Inject
186+
DrillBuf buffer;
187+
188+
@Workspace
189+
org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader;
190+
191+
@Output
192+
ComplexWriter writer;
193+
194+
@Override
195+
public void setup() {
196+
boolean allTextMode = allTextModeHolder.value == 1;
197+
boolean readNumbersAsDouble = readNumbersAsDoubleHolder.value == 1;
198+
199+
jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer)
200+
.defaultSchemaPathColumns()
201+
.allTextMode(allTextMode)
202+
.readNumbersAsDouble(readNumbersAsDouble)
85203
.build();
86204
}
87205

@@ -100,16 +218,83 @@ public void eval() {
100218
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true)
101219
public static class ConvertFromJsonNullableInput implements DrillSimpleFunc {
102220

103-
@Param NullableVarBinaryHolder in;
104-
@Inject DrillBuf buffer;
105-
@Workspace org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader;
221+
@Param
222+
NullableVarBinaryHolder in;
106223

107-
@Output ComplexWriter writer;
224+
@Inject
225+
DrillBuf buffer;
226+
227+
@Inject
228+
OptionManager options;
229+
230+
@Workspace
231+
org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader;
232+
233+
@Output
234+
ComplexWriter writer;
235+
236+
@Override
237+
public void setup() {
238+
boolean allTextMode = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_ALL_TEXT_MODE);
239+
boolean readNumbersAsDouble = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE);
240+
241+
jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer)
242+
.defaultSchemaPathColumns()
243+
.allTextMode(allTextMode)
244+
.readNumbersAsDouble(readNumbersAsDouble)
245+
.build();
246+
}
247+
248+
@Override
249+
public void eval() {
250+
if (in.isSet == 0) {
251+
// Return empty map
252+
org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter mapWriter = writer.rootAsMap();
253+
mapWriter.start();
254+
mapWriter.end();
255+
return;
256+
}
257+
258+
try {
259+
jsonReader.setSource(in.start, in.end, in.buffer);
260+
jsonReader.write(writer);
261+
buffer = jsonReader.getWorkBuf();
262+
} catch (Exception e) {
263+
throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e);
264+
}
265+
}
266+
}
267+
268+
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true)
269+
public static class ConvertFromJsonNullableInputWithArgs implements DrillSimpleFunc {
270+
271+
@Param
272+
NullableVarBinaryHolder in;
273+
274+
@Param
275+
BitHolder allTextModeHolder;
276+
277+
@Param
278+
BitHolder readNumbersAsDoubleHolder;
279+
280+
@Inject
281+
DrillBuf buffer;
282+
283+
@Workspace
284+
org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader;
285+
286+
@Output
287+
ComplexWriter writer;
108288

109289
@Override
110290
public void setup() {
291+
boolean allTextMode = allTextModeHolder.value == 1;
292+
boolean readNumbersAsDouble = readNumbersAsDoubleHolder.value == 1;
293+
111294
jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer)
112295
.defaultSchemaPathColumns()
296+
.allTextMode(allTextMode)
297+
.readNumbersAsDouble(readNumbersAsDouble)
113298
.build();
114299
}
115300

@@ -136,16 +321,29 @@ public void eval() {
136321
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true)
137322
public static class ConvertFromJsonVarcharNullableInput implements DrillSimpleFunc {
138323

139-
@Param NullableVarCharHolder in;
140-
@Inject DrillBuf buffer;
141-
@Workspace org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader;
324+
@Param
325+
NullableVarCharHolder in;
326+
327+
@Inject
328+
DrillBuf buffer;
329+
330+
@Inject
331+
OptionManager options;
332+
333+
@Workspace
334+
org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader;
142335

143336
@Output ComplexWriter writer;
144337

145338
@Override
146339
public void setup() {
340+
boolean allTextMode = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_ALL_TEXT_MODE);
341+
boolean readNumbersAsDouble = options.getBoolean(org.apache.drill.exec.ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE);
342+
147343
jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer)
148344
.defaultSchemaPathColumns()
345+
.allTextMode(allTextMode)
346+
.readNumbersAsDouble(readNumbersAsDouble)
149347
.build();
150348
}
151349

@@ -168,4 +366,57 @@ public void eval() {
168366
}
169367
}
170368
}
369+
370+
@FunctionTemplate(name = "convert_fromJSON", scope = FunctionScope.SIMPLE, isRandom = true)
371+
public static class ConvertFromJsonVarcharNullableInputWithConfigs implements DrillSimpleFunc {
372+
373+
@Param
374+
NullableVarCharHolder in;
375+
376+
@Param
377+
BitHolder allTextModeHolder;
378+
379+
@Param
380+
BitHolder readNumbersAsDoubleHolder;
381+
382+
@Inject
383+
DrillBuf buffer;
384+
385+
@Workspace
386+
org.apache.drill.exec.vector.complex.fn.JsonReader jsonReader;
387+
388+
@Output ComplexWriter writer;
389+
390+
@Override
391+
public void setup() {
392+
boolean allTextMode = allTextModeHolder.value == 1;
393+
boolean readNumbersAsDouble = readNumbersAsDoubleHolder.value == 1;
394+
395+
jsonReader = new org.apache.drill.exec.vector.complex.fn.JsonReader.Builder(buffer)
396+
.defaultSchemaPathColumns()
397+
.allTextMode(allTextMode)
398+
.readNumbersAsDouble(readNumbersAsDouble)
399+
.build();
400+
}
401+
402+
@Override
403+
public void eval() {
404+
if (in.isSet == 0) {
405+
// Return empty map
406+
org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter mapWriter = writer.rootAsMap();
407+
mapWriter.start();
408+
mapWriter.end();
409+
return;
410+
}
411+
412+
try {
413+
jsonReader.setSource(in.start, in.end, in.buffer);
414+
jsonReader.write(writer);
415+
buffer = jsonReader.getWorkBuf();
416+
} catch (Exception e) {
417+
throw new org.apache.drill.common.exceptions.DrillRuntimeException("Error while converting from JSON. ", e);
418+
}
419+
}
420+
}
421+
171422
}

0 commit comments

Comments
 (0)