This is an automated email from the ASF dual-hosted git repository. hanahmily pushed a commit to branch sidx/query in repository https://gitbox.apache.org/repos/asf/skywalking-banyandb.git
commit 38d88711f089a05d4acc9d9a0d82f6ef03baa47c Author: Gao Hongtao <[email protected]> AuthorDate: Wed Aug 27 08:59:55 2025 +0800 Refactor user keys handling: Update the processing of user-provided int64 keys to write encoded data directly without compression. Adjust related comments and remove decompression steps in the readAll and loadBlockData methods for improved efficiency and clarity. --- banyand/internal/sidx/block.go | 7 +++---- banyand/internal/sidx/part.go | 16 +++------------- banyand/internal/sidx/query_result.go | 12 ++++-------- 3 files changed, 10 insertions(+), 25 deletions(-) diff --git a/banyand/internal/sidx/block.go b/banyand/internal/sidx/block.go index cc638542..24be89d9 100644 --- a/banyand/internal/sidx/block.go +++ b/banyand/internal/sidx/block.go @@ -335,11 +335,10 @@ func mustWriteKeysTo(kb *dataBlock, userKeys []int64, keysWriter *writer) (encod var firstValue int64 bb.Buf, encodeType, firstValue = encoding.Int64ListToBytes(bb.Buf[:0], userKeys) - // Compress and write - compressedData := zstd.Compress(nil, bb.Buf, 1) + // Write encoded data directly without compression kb.offset = keysWriter.bytesWritten - kb.size = uint64(len(compressedData)) - keysWriter.MustWrite(compressedData) + kb.size = uint64(len(bb.Buf)) + keysWriter.MustWrite(bb.Buf) return encodeType, firstValue } diff --git a/banyand/internal/sidx/part.go b/banyand/internal/sidx/part.go index 7aa6805d..5753a300 100644 --- a/banyand/internal/sidx/part.go +++ b/banyand/internal/sidx/part.go @@ -52,7 +52,7 @@ const ( // Each part contains multiple files organized by type: // - primary.bin: Block metadata and structure information // - data.bin: User payload data (compressed) -// - keys.bin: User-provided int64 keys (compressed) +// - keys.bin: User-provided int64 keys (encoded but not compressed) // - meta.bin: Part metadata // - <name>.td: Tag data files (one per tag) // - <name>.tm: Tag metadata files (one per tag) @@ -272,7 +272,6 @@ func (p *part) readAll() ([]*elements, error) { compressedDataBuf := make([]byte, 0, 1024) dataBuf := make([]byte, 0, 1024) compressedKeysBuf := make([]byte, 0, 1024) - keysBuf := make([]byte, 0, 1024) for _, pbm := range p.primaryBlockMetadata { // Read and decompress primary block metadata @@ -310,17 +309,8 @@ func (p *part) readAll() ([]*elements, error) { compressedKeysBuf = bytes.ResizeOver(compressedKeysBuf, int(bm.keysBlock.size)) fs.MustReadData(p.keys, int64(bm.keysBlock.offset), compressedKeysBuf) - keysBuf, err = zstd.Decompress(keysBuf[:0], compressedKeysBuf) - if err != nil { - releaseElements(elems) - for _, e := range result { - releaseElements(e) - } - return nil, fmt.Errorf("cannot decompress keys block: %w", err) - } - - // Decode user keys using the stored encoding information - elems.userKeys, err = encoding.BytesToInt64List(elems.userKeys[:0], keysBuf, bm.keysEncodeType, bm.minKey, int(bm.count)) + // Decode user keys directly using the stored encoding information + elems.userKeys, err = encoding.BytesToInt64List(elems.userKeys[:0], compressedKeysBuf, bm.keysEncodeType, bm.minKey, int(bm.count)) if err != nil { releaseElements(elems) for _, e := range result { diff --git a/banyand/internal/sidx/query_result.go b/banyand/internal/sidx/query_result.go index d7789428..dfe07d80 100644 --- a/banyand/internal/sidx/query_result.go +++ b/banyand/internal/sidx/query_result.go @@ -178,7 +178,7 @@ func (qr *queryResult) loadBlockData(tmpBlock *block, p *part, bm *blockMetadata return false } - // Read and decompress user keys (always needed) + // Read user keys (always needed) bb := bigValuePool.Get() if bb == nil { bb = &bytes.Buffer{} @@ -191,13 +191,9 @@ func (qr *queryResult) loadBlockData(tmpBlock *block, p *part, bm *blockMetadata bb.Buf = bytes.ResizeOver(bb.Buf[:0], int(bm.keysBlock.size)) fs.MustReadData(p.keys, int64(bm.keysBlock.offset), bb.Buf) - keysBuf, err := zstd.Decompress(nil, bb.Buf) - if err != nil { - return false - } - - // Decode user keys - tmpBlock.userKeys, err = encoding.BytesToInt64List(tmpBlock.userKeys[:0], keysBuf, bm.keysEncodeType, bm.minKey, int(bm.count)) + // Decode user keys directly + var err error + tmpBlock.userKeys, err = encoding.BytesToInt64List(tmpBlock.userKeys[:0], bb.Buf, bm.keysEncodeType, bm.minKey, int(bm.count)) if err != nil { return false }
