This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new d8a1e7df fix(arrow/csv): preserve first row during headerless
inference (#1145)
d8a1e7df is described below
commit d8a1e7df31312fac2108199e445ae82f2c4978ff
Author: Minh Vu <[email protected]>
AuthorDate: Tue Aug 11 22:36:03 2026 +0200
fix(arrow/csv): preserve first row during headerless inference (#1145)
## What
When schema inference runs with WithHeader(false), the first CSV record
is consumed while generating f0, f1, and so on, but it is not sent
through the builders. This patch keeps that record pending and replays
it through the normal reader path.
## Test
- go test ./arrow/csv -run
TestInferringSchemaWithoutHeaderPreservesFirstRow -count=1
---
arrow/csv/reader.go | 25 ++++++++++++++++++++++---
arrow/csv/reader_test.go | 25 +++++++++++++++++++++++++
2 files changed, 47 insertions(+), 3 deletions(-)
diff --git a/arrow/csv/reader.go b/arrow/csv/reader.go
index 0d4ee0f3..9387f23d 100644
--- a/arrow/csv/reader.go
+++ b/arrow/csv/reader.go
@@ -61,6 +61,7 @@ type Reader struct {
columnFilter []string
columnTypes map[string]arrow.DataType
conversions []conversionColumn
+ pendingRecord []string
stringsCanBeNull bool
nulls []string
@@ -214,6 +215,9 @@ func (r *Reader) readHeader() error {
r.columnFilter = nil
}
r.columnTypes = nil
+ if !r.header {
+ r.pendingRecord = append([]string(nil), records...)
+ }
return nil
}
@@ -272,7 +276,7 @@ func (r *Reader) Next() bool {
// from that row.
func (r *Reader) next1() bool {
var recs []string
- recs, r.err = r.r.Read()
+ recs, r.err = r.readRecord()
if r.err != nil {
r.done = true
if errors.Is(r.err, io.EOF) {
@@ -296,11 +300,17 @@ func (r *Reader) nextall() bool {
}()
var recs [][]string
+ if r.pendingRecord != nil {
+ recs = append(recs, r.pendingRecord)
+ r.pendingRecord = nil
+ }
- recs, r.err = r.r.ReadAll()
+ var remaining [][]string
+ remaining, r.err = r.r.ReadAll()
if r.err != nil {
return false
}
+ recs = append(recs, remaining...)
for _, rec := range recs {
r.validate(rec)
@@ -321,7 +331,7 @@ func (r *Reader) nextn() bool {
)
for i := 0; i < r.chunk && !r.done; i++ {
- recs, err = r.r.Read()
+ recs, err = r.readRecord()
if err != nil {
if !errors.Is(err, io.EOF) {
r.err = err
@@ -343,6 +353,15 @@ func (r *Reader) nextn() bool {
return n > 0
}
+func (r *Reader) readRecord() ([]string, error) {
+ if r.pendingRecord != nil {
+ record := r.pendingRecord
+ r.pendingRecord = nil
+ return record, nil
+ }
+ return r.r.Read()
+}
+
func (r *Reader) validate(recs []string) {
if r.err != nil {
return
diff --git a/arrow/csv/reader_test.go b/arrow/csv/reader_test.go
index 80c78482..c7cd61f4 100644
--- a/arrow/csv/reader_test.go
+++ b/arrow/csv/reader_test.go
@@ -957,6 +957,31 @@ func TestInferringSchema(t *testing.T) {
assert.False(t, r.Next())
}
+func TestInferringSchemaWithoutHeaderPreservesFirstRow(t *testing.T) {
+ mem := memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer mem.AssertSize(t, 0)
+
+ r := csv.NewInferringReader(strings.NewReader("1,a\n2,b\n"),
+ csv.WithAllocator(mem), csv.WithHeader(false))
+ defer r.Release()
+
+ var (
+ values []int64
+ strings []string
+ )
+ for r.Next() {
+ rec := r.RecordBatch()
+ values = append(values, rec.Column(0).(*array.Int64).Value(0))
+ strings = append(strings,
rec.Column(1).(*array.String).Value(0))
+ }
+
+ require.NoError(t, r.Err())
+ require.Equal(t, []int64{1, 2}, values)
+ require.Equal(t, []string{"a", "b"}, strings)
+ require.Equal(t, "f0", r.Schema().Field(0).Name)
+ require.Equal(t, "f1", r.Schema().Field(1).Name)
+}
+
func TestInferCSVOptions(t *testing.T) {
mem := memory.NewCheckedAllocator(memory.DefaultAllocator)
defer mem.AssertSize(t, 0)
