zeroshade commented on code in PR #1596:
URL: https://github.com/apache/arrow-go/pull/1596#discussion_r4604003655


##########
arrow/array/dictionary_test.go:
##########
@@ -2200,3 +2205,299 @@ func BenchmarkBinaryDictionaryBuilder(b *testing.B) {
                assert.NoError(b, builder.AppendString(randString()))
        }
 }
+
+func TestDictionaryBuilderBinaryLikeValueTypes(t *testing.T) {
+       const long = "a value longer than twelve bytes"
+       cases := []struct {
+               name   string
+               vt     arrow.DataType
+               binary bool
+       }{
+               {"string_view", arrow.BinaryTypes.StringView, false},
+               {"binary_view", arrow.BinaryTypes.BinaryView, true},
+               {"large_utf8", arrow.BinaryTypes.LargeString, false},
+               {"large_binary", arrow.BinaryTypes.LargeBinary, true},
+       }
+
+       for _, tc := range cases {
+               t.Run(tc.name, func(t *testing.T) {
+                       dt := &arrow.DictionaryType{IndexType: 
arrow.PrimitiveTypes.Int32, ValueType: tc.vt}
+                       isView := tc.vt.ID() == arrow.STRING_VIEW || tc.vt.ID() 
== arrow.BINARY_VIEW
+
+                       jsonValues := func(vals ...any) string {
+                               out := make([]any, len(vals))
+                               for i, v := range vals {
+                                       if s, ok := v.(string); ok && tc.binary 
{
+                                               v = 
base64.StdEncoding.EncodeToString([]byte(s))
+                                       }
+                                       out[i] = v
+                               }
+                               b, err := json.Marshal(out)
+                               require.NoError(t, err)
+                               return string(b)
+                       }
+                       fromJSON := func(mem memory.Allocator, typ 
arrow.DataType, js string) arrow.Array {
+                               arr, _, err := array.FromJSON(mem, typ, 
strings.NewReader(js))
+                               require.NoError(t, err)
+                               return arr
+                       }
+                       requireDict := func(t *testing.T, mem memory.Allocator, 
got arrow.Array, indices, dict string) {
+                               t.Helper()
+                               require.NoError(t, array.ValidateFull(got))
+                               want, err := array.DictArrayFromJSON(mem, dt, 
indices, dict)
+                               require.NoError(t, err)
+                               defer want.Release()
+                               assert.Truef(t, array.Equal(want, got), "got 
%v, want %v", got, want)
+                       }
+                       newBuilder := func(t *testing.T, mem memory.Allocator) 
*array.BinaryDictionaryBuilder {
+                               t.Helper()
+                               bldr := array.NewBuilder(mem, dt)
+                               b, ok := bldr.(*array.BinaryDictionaryBuilder)
+                               require.Truef(t, ok, "got builder %T", bldr)
+                               return b
+                       }
+
+                       t.Run("append", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               b := newBuilder(t, mem)
+                               defer b.Release()
+                               require.NoError(t, b.AppendString("alpha"))
+                               b.AppendNull()
+                               require.NoError(t, b.AppendString(long))
+                               require.NoError(t, b.Append([]byte("alpha")))
+                               b.AppendEmptyValue()
+                               require.NoError(t, b.AppendString(""))
+
+                               arr := b.NewArray()
+                               defer arr.Release()
+                               requireDict(t, mem, arr, `[0, null, 1, 0, 2, 
2]`, jsonValues("alpha", long, ""))
+                       })
+
+                       t.Run("delta", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               b := newBuilder(t, mem)
+                               defer b.Release()
+                               require.NoError(t, b.AppendString("alpha"))
+                               require.NoError(t, b.AppendString("beta"))
+                               first := b.NewArray()
+                               defer first.Release()
+                               requireDict(t, mem, first, `[0, 1]`, 
jsonValues("alpha", "beta"))
+
+                               require.NoError(t, b.AppendString("beta"))
+                               require.NoError(t, b.AppendString(long))
+                               require.NoError(t, b.AppendString("gamma"))
+                               indices, delta, err := b.NewDelta()
+                               require.NoError(t, err)
+                               defer indices.Release()
+                               defer delta.Release()
+
+                               require.NoError(t, array.ValidateFull(delta))
+                               wantIdx := fromJSON(mem, 
arrow.PrimitiveTypes.Int32, `[1, 2, 3]`)
+                               defer wantIdx.Release()
+                               assert.Truef(t, array.Equal(wantIdx, indices), 
"indices: got %v", indices)
+                               wantDelta := fromJSON(mem, tc.vt, 
jsonValues(long, "gamma"))
+                               defer wantDelta.Release()
+                               assert.Truef(t, array.Equal(wantDelta, delta), 
"delta: got %v", delta)
+                       })
+
+                       t.Run("initial dictionary", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               init := fromJSON(mem, tc.vt, 
jsonValues("alpha", long))
+                               defer init.Release()
+                               bldr := array.NewDictionaryBuilderWithDict(mem, 
dt, init)
+                               defer bldr.Release()
+                               assert.Equal(t, 2, bldr.DictionarySize())
+
+                               b := bldr.(*array.BinaryDictionaryBuilder)
+                               require.NoError(t, b.AppendString(long))
+                               require.NoError(t, b.AppendString("beta"))
+                               require.NoError(t, b.AppendString("alpha"))
+
+                               arr := b.NewArray()
+                               defer arr.Release()
+                               requireDict(t, mem, arr, `[1, 2, 0]`, 
jsonValues("alpha", long, "beta"))
+                       })
+
+                       t.Run("FromJSON", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               arr := fromJSON(mem, dt, jsonValues("alpha", 
nil, long, "alpha"))
+                               defer arr.Release()
+                               requireDict(t, mem, arr, `[0, null, 1, 0]`, 
jsonValues("alpha", long))
+                       })
+
+                       t.Run("record builder", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               schema := arrow.NewSchema([]arrow.Field{{Name: 
"f", Type: dt, Nullable: true}}, nil)
+                               rb := array.NewRecordBuilder(mem, schema)
+                               defer rb.Release()
+                               fb, ok := 
rb.Field(0).(*array.BinaryDictionaryBuilder)
+                               require.Truef(t, ok, "got builder %T", 
rb.Field(0))
+                               require.NoError(t, fb.AppendString("alpha"))
+                               fb.AppendNull()
+                               require.NoError(t, fb.AppendString(long))
+
+                               rec := rb.NewRecordBatch()
+                               defer rec.Release()
+                               require.NoError(t, 
array.ValidateRecordFull(rec))
+                               requireDict(t, mem, rec.Column(0), `[0, null, 
1]`, jsonValues("alpha", long))
+                       })
+
+                       t.Run("empty delta", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               b := newBuilder(t, mem)
+                               defer b.Release()
+                               require.NoError(t, b.AppendString(long))
+                               first := b.NewArray()
+                               defer first.Release()
+
+                               indices, delta, err := b.NewDelta()
+                               require.NoError(t, err)
+                               defer indices.Release()
+                               defer delta.Release()
+                               assert.Zero(t, indices.Len())
+                               assert.Zero(t, delta.Len())
+                               require.NoError(t, array.ValidateFull(delta))
+                       })
+
+                       if isView {
+                               t.Run("single data buffer", func(t *testing.T) {
+                                       mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                                       defer mem.AssertSize(t, 0)
+
+                                       b := newBuilder(t, mem)
+                                       defer b.Release()
+                                       // 600 values of 100 bytes exceed the 
32 KiB default block size.
+                                       for i := 0; i < 600; i++ {
+                                               require.NoError(t, 
b.AppendString(fmt.Sprintf("%0100d", i)))
+                                       }
+                                       arr := b.NewArray()
+                                       defer arr.Release()
+                                       require.NoError(t, 
array.ValidateFull(arr))
+
+                                       bufs := 
arr.(*array.Dictionary).Dictionary().Data().Buffers()
+                                       require.Len(t, bufs, 3, "validity, 
views, one data buffer")
+                                       assert.Equal(t, 600*100, bufs[2].Len())
+                               })
+
+                               t.Run("inline values have no data buffer", 
func(t *testing.T) {
+                                       mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                                       defer mem.AssertSize(t, 0)
+
+                                       b := newBuilder(t, mem)
+                                       defer b.Release()
+                                       require.NoError(t, b.AppendString("a"))
+                                       require.NoError(t, 
b.AppendString("twelve bytes"))
+                                       arr := b.NewArray()
+                                       defer arr.Release()
+                                       require.NoError(t, 
array.ValidateFull(arr))
+
+                                       assert.Len(t, 
arr.(*array.Dictionary).Dictionary().Data().Buffers(), 2, "validity and views 
only")
+                               })
+
+                               t.Run("initial dictionary null slot", func(t 
*testing.T) {
+                                       mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                                       defer mem.AssertSize(t, 0)
+
+                                       init := fromJSON(mem, tc.vt, 
jsonValues("alpha", nil, long))
+                                       defer init.Release()
+                                       views := 
init.Data().Buffers()[1].Bytes()
+                                       hdrSize := 
arrow.ViewHeaderTraits.BytesRequired(1)
+                                       for i := hdrSize; i < 2*hdrSize; i++ {
+                                               views[i] = 0xFF
+                                       }
+                                       require.True(t, init.IsNull(1))
+                                       require.NoError(t, 
array.ValidateFull(init))
+
+                                       bldr := 
array.NewDictionaryBuilderWithDict(mem, dt, init)
+                                       defer bldr.Release()
+                                       assert.Equal(t, 3, 
bldr.DictionarySize(), "the null slot is inserted as the empty value")
+
+                                       b := 
bldr.(*array.BinaryDictionaryBuilder)
+                                       require.NoError(t, b.AppendString(""))
+                                       require.NoError(t, b.AppendString(long))
+                                       arr := b.NewArray()
+                                       defer arr.Release()
+                                       requireDict(t, mem, arr, `[1, 2]`, 
jsonValues("alpha", "", long))
+                               })
+                       }
+
+                       t.Run("unifier", func(t *testing.T) {
+                               mem := 
memory.NewCheckedAllocator(memory.DefaultAllocator)
+                               defer mem.AssertSize(t, 0)
+
+                               u, err := array.NewDictionaryUnifier(mem, tc.vt)
+                               require.NoError(t, err)
+                               defer u.Release()
+
+                               d1 := fromJSON(mem, tc.vt, jsonValues("alpha", 
nil, long))
+                               defer d1.Release()
+                               d2 := fromJSON(mem, tc.vt, jsonValues(long, 
"beta", "alpha"))
+                               defer d2.Release()
+
+                               require.NoError(t, u.Unify(d1))
+                               transposed, err := u.UnifyAndTranspose(d2)
+                               require.NoError(t, err)
+                               defer transposed.Release()
+                               assert.Equal(t, []int32{2, 3, 0}, 
arrow.Int32Traits.CastFromBytes(transposed.Bytes())[:3])
+
+                               outType, outDict, err := u.GetResult()
+                               require.NoError(t, err)
+                               defer outDict.Release()
+                               assert.True(t, arrow.TypeEqual(tc.vt, 
outType.(*arrow.DictionaryType).ValueType))
+                               require.NoError(t, array.ValidateFull(outDict))
+                               want := fromJSON(mem, tc.vt, 
jsonValues("alpha", nil, long, "beta"))
+                               defer want.Release()
+                               assert.Truef(t, array.Equal(want, outDict), 
"unified: got %v", outDict)
+                       })
+               })
+       }
+}
+
+func TestDictionaryBuilderLargeValuesOver2GiB(t *testing.T) {

Review Comment:
   Please drop this test. Nothing in CI sets `ARROW_TEST_LARGE_MEMORY`, so it 
will never run there. It would also add a new environment-variable convention 
to the repo, and it needs about 18 GiB of memory. The change it covers is just 
which builder the large types use for the memo table, and the 
`large_utf8`/`large_binary` subtests of 
`TestDictionaryBuilderBinaryLikeValueTypes` already exercise the large-offset 
output path.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to