zeroshade commented on code in PR #1585:
URL: https://github.com/apache/arrow-go/pull/1585#discussion_r4591514928


##########
parquet/file/column_reader_test.go:
##########
@@ -507,6 +508,64 @@ func TestSkipEmptyRepeatedRows(t *testing.T) {
        assert.Equal(t, []int16{0}, reps)
 }
 
+func TestSkipStopsAtDecodeError(t *testing.T) {
+       mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+       defer mem.AssertSize(t, 0)
+
+       node, err := schema.NewPrimitiveNode("value", 
parquet.Repetitions.Required, parquet.Types.ByteArray, -1, -1)
+       require.NoError(t, err)
+       descr := schema.NewColumn(node, 0, 0)
+
+       // the length prefix runs past the data, so only decoding fails
+       body := memory.NewBufferBytes([]byte{3, 0, 0, 0, 'a'})
+       page := file.NewDataPageV1(body, 20, parquet.Encodings.Plain, 
parquet.Encodings.RLE, parquet.Encodings.RLE, int32(body.Len()))
+       defer page.Release()
+
+       pages := new(testutils.MockPageReader)
+       pages.Test(t)
+       pages.TestData().Set("pages", []file.Page{page})
+       pages.On("Err").Return((error)(nil))
+       r := file.NewColumnReader(descr, pages, mem, 
nil).(*file.ByteArrayColumnChunkReader)
+       defer r.Close()
+
+       n, err := r.Skip(10)
+       require.Error(t, err)
+       assert.Less(t, n, int64(10))

Review Comment:
   The first value is the corrupt one, so the count here should be exactly 0. 
`Less` would still pass if a later change returned something like 9.
   
   ```suggestion
        assert.EqualValues(t, 0, n)
   ```



##########
parquet/file/column_reader_types.gen.go.tmpl:
##########
@@ -30,9 +30,13 @@ type {{.Name}}ColumnChunkReader struct {
 
 // Skip skips the next nvalues so that the next call to ReadBatch
 // will start reading *after* the skipped values.
+//
+// It returns the number of rows skipped before it stopped, which is less than
+// nvalues when the column ends or an error occurs first. After an error on a
+// repeated column, the count can include a row that continues onto the page
+// that failed.

Review Comment:
   Could this describe the error case in general terms instead of only the 
repeated-column case? Something like:
   
   ```
   // nvalues when the column ends or an error occurs first. On error, the count
   // covers only the rows skipped before the step that failed, and the reader
   // stops at that error.
   ```
   
   The generated file needs regenerating to match.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to