hcrosse commented on code in PR #1596:
URL: https://github.com/apache/arrow-go/pull/1596#discussion_r4604941972


##########
arrow/array/dictionary_test.go:
##########
@@ -2200,3 +2205,299 @@ func BenchmarkBinaryDictionaryBuilder(b *testing.B) {
                assert.NoError(b, builder.AppendString(randString()))
        }
 }
+
+func TestDictionaryBuilderBinaryLikeValueTypes(t *testing.T) {
+       const long = "a value longer than twelve bytes"
+       cases := []struct {
+               name   string
+               vt     arrow.DataType
+               binary bool
+       }{
+               {"string_view", arrow.BinaryTypes.StringView, false},
+               {"binary_view", arrow.BinaryTypes.BinaryView, true},
+               {"large_utf8", arrow.BinaryTypes.LargeString, false},
+               {"large_binary", arrow.BinaryTypes.LargeBinary, true},
+       }
+
+       for _, tc := range cases {
+               t.Run(tc.name, func(t *testing.T) {
+                       dt := &arrow.DictionaryType{IndexType: 
arrow.PrimitiveTypes.Int32, ValueType: tc.vt}
+                       isView := tc.vt.ID() == arrow.STRING_VIEW || tc.vt.ID() 
== arrow.BINARY_VIEW
+
+                       jsonValues := func(vals ...any) string {
+                               out := make([]any, len(vals))
+                               for i, v := range vals {
+                                       if s, ok := v.(string); ok && tc.binary 
{
+                                               v = 
base64.StdEncoding.EncodeToString([]byte(s))
+                                       }
+                                       out[i] = v
+                               }
+                               b, err := json.Marshal(out)
+                               require.NoError(t, err)
+                               return string(b)
+                       }
+                       fromJSON := func(mem memory.Allocator, typ 
arrow.DataType, js string) arrow.Array {
+                               arr, _, err := array.FromJSON(mem, typ, 
strings.NewReader(js))
+                               require.NoError(t, err)
+                               return arr
+                       }
+                       requireDict := func(t *testing.T, mem memory.Allocator, 
got arrow.Array, indices, dict string) {
+                               t.Helper()
+                               require.NoError(t, array.ValidateFull(got))
+                               want, err := array.DictArrayFromJSON(mem, dt, 
indices, dict)
+                               require.NoError(t, err)
+                               defer want.Release()
+                               assert.Truef(t, array.Equal(want, got), "got 
%v, want %v", got, want)
+                       }
+                       newBuilder := func(t *testing.T, mem memory.Allocator) 
*array.BinaryDictionaryBuilder {
+                               t.Helper()
+                               bldr := array.NewBuilder(mem, dt)
+                               b, ok := bldr.(*array.BinaryDictionaryBuilder)
+                               require.Truef(t, ok, "got builder %T", bldr)
+                               return b
+                       }
+
+                       t.Run("append", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               b := newBuilder(t, mem)
+                               defer b.Release()
+                               require.NoError(t, b.AppendString("alpha"))
+                               b.AppendNull()
+                               require.NoError(t, b.AppendString(long))
+                               require.NoError(t, b.Append([]byte("alpha")))
+                               b.AppendEmptyValue()
+                               require.NoError(t, b.AppendString(""))
+
+                               arr := b.NewArray()
+                               defer arr.Release()
+                               requireDict(t, mem, arr, `[0, null, 1, 0, 2, 
2]`, jsonValues("alpha", long, ""))
+                       })
+
+                       t.Run("delta", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               b := newBuilder(t, mem)
+                               defer b.Release()
+                               require.NoError(t, b.AppendString("alpha"))
+                               require.NoError(t, b.AppendString("beta"))
+                               first := b.NewArray()
+                               defer first.Release()
+                               requireDict(t, mem, first, `[0, 1]`, 
jsonValues("alpha", "beta"))
+
+                               require.NoError(t, b.AppendString("beta"))
+                               require.NoError(t, b.AppendString(long))
+                               require.NoError(t, b.AppendString("gamma"))
+                               indices, delta, err := b.NewDelta()
+                               require.NoError(t, err)
+                               defer indices.Release()
+                               defer delta.Release()
+
+                               require.NoError(t, array.ValidateFull(delta))
+                               wantIdx := fromJSON(mem, 
arrow.PrimitiveTypes.Int32, `[1, 2, 3]`)
+                               defer wantIdx.Release()
+                               assert.Truef(t, array.Equal(wantIdx, indices), 
"indices: got %v", indices)
+                               wantDelta := fromJSON(mem, tc.vt, 
jsonValues(long, "gamma"))
+                               defer wantDelta.Release()
+                               assert.Truef(t, array.Equal(wantDelta, delta), 
"delta: got %v", delta)
+                       })
+
+                       t.Run("initial dictionary", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               init := fromJSON(mem, tc.vt, 
jsonValues("alpha", long))
+                               defer init.Release()
+                               bldr := array.NewDictionaryBuilderWithDict(mem, 
dt, init)
+                               defer bldr.Release()
+                               assert.Equal(t, 2, bldr.DictionarySize())
+
+                               b := bldr.(*array.BinaryDictionaryBuilder)
+                               require.NoError(t, b.AppendString(long))
+                               require.NoError(t, b.AppendString("beta"))
+                               require.NoError(t, b.AppendString("alpha"))
+
+                               arr := b.NewArray()
+                               defer arr.Release()
+                               requireDict(t, mem, arr, `[1, 2, 0]`, 
jsonValues("alpha", long, "beta"))
+                       })
+
+                       t.Run("FromJSON", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               arr := fromJSON(mem, dt, jsonValues("alpha", 
nil, long, "alpha"))
+                               defer arr.Release()
+                               requireDict(t, mem, arr, `[0, null, 1, 0]`, 
jsonValues("alpha", long))
+                       })
+
+                       t.Run("record builder", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               schema := arrow.NewSchema([]arrow.Field{{Name: 
"f", Type: dt, Nullable: true}}, nil)
+                               rb := array.NewRecordBuilder(mem, schema)
+                               defer rb.Release()
+                               fb, ok := 
rb.Field(0).(*array.BinaryDictionaryBuilder)
+                               require.Truef(t, ok, "got builder %T", 
rb.Field(0))
+                               require.NoError(t, fb.AppendString("alpha"))
+                               fb.AppendNull()
+                               require.NoError(t, fb.AppendString(long))
+
+                               rec := rb.NewRecordBatch()
+                               defer rec.Release()
+                               require.NoError(t, 
array.ValidateRecordFull(rec))
+                               requireDict(t, mem, rec.Column(0), `[0, null, 
1]`, jsonValues("alpha", long))
+                       })
+
+                       t.Run("empty delta", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               b := newBuilder(t, mem)
+                               defer b.Release()
+                               require.NoError(t, b.AppendString(long))
+                               first := b.NewArray()
+                               defer first.Release()
+
+                               indices, delta, err := b.NewDelta()
+                               require.NoError(t, err)
+                               defer indices.Release()
+                               defer delta.Release()
+                               assert.Zero(t, indices.Len())
+                               assert.Zero(t, delta.Len())
+                               require.NoError(t, array.ValidateFull(delta))
+                       })
+
+                       if isView {
+                               t.Run("single data buffer", func(t *testing.T) {
+                                       mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                                       defer mem.AssertSize(t, 0)
+
+                                       b := newBuilder(t, mem)
+                                       defer b.Release()
+                                       // 600 values of 100 bytes exceed the 
32 KiB default block size.
+                                       for i := 0; i < 600; i++ {
+                                               require.NoError(t, 
b.AppendString(fmt.Sprintf("%0100d", i)))
+                                       }
+                                       arr := b.NewArray()
+                                       defer arr.Release()
+                                       require.NoError(t, 
array.ValidateFull(arr))
+
+                                       bufs := 
arr.(*array.Dictionary).Dictionary().Data().Buffers()
+                                       require.Len(t, bufs, 3, "validity, 
views, one data buffer")
+                                       assert.Equal(t, 600*100, bufs[2].Len())
+                               })
+
+                               t.Run("inline values have no data buffer", 
func(t *testing.T) {
+                                       mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                                       defer mem.AssertSize(t, 0)
+
+                                       b := newBuilder(t, mem)
+                                       defer b.Release()
+                                       require.NoError(t, b.AppendString("a"))
+                                       require.NoError(t, 
b.AppendString("twelve bytes"))
+                                       arr := b.NewArray()
+                                       defer arr.Release()
+                                       require.NoError(t, 
array.ValidateFull(arr))
+
+                                       assert.Len(t, 
arr.(*array.Dictionary).Dictionary().Data().Buffers(), 2, "validity and views 
only")
+                               })
+
+                               t.Run("initial dictionary null slot", func(t 
*testing.T) {
+                                       mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                                       defer mem.AssertSize(t, 0)
+
+                                       init := fromJSON(mem, tc.vt, 
jsonValues("alpha", nil, long))
+                                       defer init.Release()
+                                       views := 
init.Data().Buffers()[1].Bytes()
+                                       hdrSize := 
arrow.ViewHeaderTraits.BytesRequired(1)
+                                       for i := hdrSize; i < 2*hdrSize; i++ {
+                                               views[i] = 0xFF
+                                       }
+                                       require.True(t, init.IsNull(1))
+                                       require.NoError(t, 
array.ValidateFull(init))
+
+                                       bldr := 
array.NewDictionaryBuilderWithDict(mem, dt, init)
+                                       defer bldr.Release()
+                                       assert.Equal(t, 3, 
bldr.DictionarySize(), "the null slot is inserted as the empty value")
+
+                                       b := 
bldr.(*array.BinaryDictionaryBuilder)
+                                       require.NoError(t, b.AppendString(""))
+                                       require.NoError(t, b.AppendString(long))
+                                       arr := b.NewArray()
+                                       defer arr.Release()
+                                       requireDict(t, mem, arr, `[1, 2]`, 
jsonValues("alpha", "", long))
+                               })
+                       }
+
+                       t.Run("unifier", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               u, err := array.NewDictionaryUnifier(mem, tc.vt)
+                               require.NoError(t, err)
+                               defer u.Release()
+
+                               d1 := fromJSON(mem, tc.vt, jsonValues("alpha", 
nil, long))
+                               defer d1.Release()
+                               d2 := fromJSON(mem, tc.vt, jsonValues(long, 
"beta", "alpha"))
+                               defer d2.Release()
+
+                               require.NoError(t, u.Unify(d1))
+                               transposed, err := u.UnifyAndTranspose(d2)
+                               require.NoError(t, err)
+                               defer transposed.Release()
+                               assert.Equal(t, []int32{2, 3, 0}, 
arrow.Int32Traits.CastFromBytes(transposed.Bytes())[:3])
+
+                               outType, outDict, err := u.GetResult()
+                               require.NoError(t, err)
+                               defer outDict.Release()
+                               assert.True(t, arrow.TypeEqual(tc.vt, 
outType.(*arrow.DictionaryType).ValueType))
+                               require.NoError(t, array.ValidateFull(outDict))
+                               want := fromJSON(mem, tc.vt, 
jsonValues("alpha", nil, long, "beta"))
+                               defer want.Release()
+                               assert.Truef(t, array.Equal(want, outDict), 
"unified: got %v", outDict)
+                       })
+               })
+       }
+}
+
+func TestDictionaryBuilderLargeValuesOver2GiB(t *testing.T) {

Review Comment:
   Will do!



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to