serramatutu commented on code in PR #833:
URL: https://github.com/apache/arrow-go/pull/833#discussion_r3912992301
##########
arrow/array/builder.go:
##########
@@ -388,6 +390,170 @@ func (b *builder) UnsafeAppendBoolToBitmap(isValid bool) {
b.length++
}
+var jsonNull = []byte("null")
+
+func unmarshalChild(dec *json.Decoder, child Builder, field arrow.Field) error
{
+ if field.Nullable {
+ return child.UnmarshalOne(dec)
+ }
+
+ nulls := child.NullN()
+ if nulls == UnknownNullCount {
+ var val json.RawMessage
+ if err := dec.Decode(&val); err != nil {
+ return err
+ }
+ return unmarshalBufferedChild(val, child, field)
+ }
+
+ // Every builder appends a null for a JSON null and for nothing else, so
+ // the child's null count going up by one over the call means the input
+ // had a null.
+ length := child.Len()
+ if err := child.UnmarshalOne(dec); err != nil {
+ return err
+ }
+ if child.Len() == length+1 && child.NullN() == nulls+1 {
+ return fmt.Errorf("field '%s' is non-nullable but got null",
field.Name)
+ }
+ return nil
+}
+
+func unmarshalBufferedChild(val json.RawMessage, child Builder, field
arrow.Field) error {
+ if !field.Nullable && bytes.Equal(val, jsonNull) {
+ return fmt.Errorf("field '%s' is non-nullable but got null",
field.Name)
+ }
+
+ valDec := json.NewDecoder(bytes.NewReader(val))
+ valDec.UseNumber()
+ return child.UnmarshalOne(valDec)
+}
+
+// rowDecoder decodes a value out of a reused copy of that value, so that
+// unescaping a string costs O(value) instead of O(remaining document).
+//
+// NOTE: goccy/go-json unescapes in place and shifts every byte after the
escape,
+// which is a big performance cost for large JSON documents:
+//
https://github.com/goccy/go-json/blob/v0.10.6/internal/decoder/string.go#L190
+type rowDecoder struct {
Review Comment:
I had to add this due to bad performance from goccy when escaping newlines...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
