serramatutu commented on code in PR #833:
URL: https://github.com/apache/arrow-go/pull/833#discussion_r3912996468
##########
arrow/array/builder.go:
##########
@@ -388,6 +390,170 @@ func (b *builder) UnsafeAppendBoolToBitmap(isValid bool) {
b.length++
}
+var jsonNull = []byte("null")
+
+func unmarshalChild(dec *json.Decoder, child Builder, field arrow.Field) error
{
+ if field.Nullable {
+ return child.UnmarshalOne(dec)
+ }
+
+ nulls := child.NullN()
+ if nulls == UnknownNullCount {
+ var val json.RawMessage
+ if err := dec.Decode(&val); err != nil {
+ return err
+ }
+ return unmarshalBufferedChild(val, child, field)
+ }
+
+ // Every builder appends a null for a JSON null and for nothing else, so
+ // the child's null count going up by one over the call means the input
+ // had a null.
+ length := child.Len()
+ if err := child.UnmarshalOne(dec); err != nil {
+ return err
+ }
+ if child.Len() == length+1 && child.NullN() == nulls+1 {
+ return fmt.Errorf("field '%s' is non-nullable but got null",
field.Name)
+ }
+ return nil
+}
+
+func unmarshalBufferedChild(val json.RawMessage, child Builder, field
arrow.Field) error {
+ if !field.Nullable && bytes.Equal(val, jsonNull) {
+ return fmt.Errorf("field '%s' is non-nullable but got null",
field.Name)
+ }
+
+ valDec := json.NewDecoder(bytes.NewReader(val))
+ valDec.UseNumber()
+ return child.UnmarshalOne(valDec)
+}
+
+// rowDecoder decodes a value out of a reused copy of that value, so that
+// unescaping a string costs O(value) instead of O(remaining document).
+//
+// NOTE: goccy/go-json unescapes in place and shifts every byte after the
escape,
+// which is a big performance cost for large JSON documents:
+//
https://github.com/goccy/go-json/blob/v0.10.6/internal/decoder/string.go#L190
+type rowDecoder struct {
+ buf json.RawMessage
+ reader bytes.Reader
+ scratch json.RawMessage
+}
+
+func (r *rowDecoder) next(dec *json.Decoder) (*json.Decoder, error) {
+ if err := dec.Decode(&r.buf); err != nil {
+ return nil, err
+ }
+
+ r.reader.Reset(r.buf)
+ rowDec := json.NewDecoder(&r.reader)
+ rowDec.UseNumber()
+ return rowDec, nil
+}
+
+func (r *rowDecoder) skip(dec *json.Decoder) error {
+ return dec.Decode(&r.scratch)
+}
+
+// fieldContainer is what nestedJSONDecoder needs out of *arrow.Schema and
+// *arrow.StructType.
+type fieldContainer interface {
+ NumFields() int
+ Field(i int) arrow.Field
+}
+
+// nestedJSONDecoder decodes JSON objects into one builder per field.
+type nestedJSONDecoder struct {
Review Comment:
I added this as a way to reuse code between `struct.go` and `builder.go`
since they were more or less doing the same logic when unmarshaling JSON
objects into a list of field builders.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
