emkornfield commented on a change in pull request #11146: URL: https://github.com/apache/arrow/pull/11146#discussion_r734235367
########## File path: go/parquet/file/page_reader.go ########## @@ -0,0 +1,616 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package file + +import ( + "bytes" + "io" + "sync" + + "github.com/JohnCGriffin/overflow" + "github.com/apache/arrow/go/arrow/ipc" + "github.com/apache/arrow/go/arrow/memory" + "github.com/apache/arrow/go/parquet" + "github.com/apache/arrow/go/parquet/compress" + "github.com/apache/arrow/go/parquet/internal/encryption" + format "github.com/apache/arrow/go/parquet/internal/gen-go/parquet" + "github.com/apache/arrow/go/parquet/internal/thrift" + "github.com/apache/arrow/go/parquet/metadata" + "golang.org/x/xerrors" +) + +// PageReader is the interface used by the columnreader in order to read +// and handle DataPages and loop through them. +type PageReader interface { + // Set the maximum Page header size allowed to be read + SetMaxPageHeaderSize(int) + // Return the current page, or nil if there are no more + Page() Page + // Fetch the next page, returns false if there are no more pages + Next() bool + // if Next returns false, Err will return the error encountered or + // nil if there was no error and you just hit the end of the page + Err() error + // Reset allows reusing a page reader + Reset(r parquet.ReaderAtSeeker, nrows int64, compressType compress.Compression, ctx *CryptoContext) +} + +// Page is an interface for handling DataPages or Dictionary Pages +type Page interface { + // Returns which kind of page this is + Type() format.PageType + // Get the raw bytes of this page + Data() []byte + // return the encoding used for this page, Plain/RLE, etc. + Encoding() format.Encoding + // get the number of values in this page + NumValues() int32 + // release this page object back into the page pool for re-use + Release() +} + +type page struct { + buf *memory.Buffer + typ format.PageType + + nvals int32 + encoding format.Encoding +} + +func (p *page) Type() format.PageType { return p.typ } +func (p *page) Data() []byte { return p.buf.Bytes() } +func (p *page) NumValues() int32 { return p.nvals } +func (p *page) Encoding() format.Encoding { return p.encoding } + +// DataPage is the base interface for both DataPageV1 and DataPageV2 of the +// parquet spec. +type DataPage interface { + Page + UncompressedSize() int64 + Statistics() metadata.EncodedStatistics +} + +// Create some pools to use for reusing the data page objects themselves so that +// we can avoid tight loops that are creating and destroying tons of individual +// objects. This combined with a Release function on the pages themselves +// which will put them back into the pool creates significant memory usage Review comment: ```suggestion // which will put them back into the pool yields significant memory reduction ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
