ZTE-EBASE commented on code in PR #1887:
URL: https://github.com/apache/cloudberry/pull/1887#discussion_r3793297264
##########
src/backend/utils/misc/fstream/gfile.c:
##########
@@ -253,7 +253,628 @@ static int bz_file_open(gfile_t *fd)
fd->u.bz->s.next_in = fd->u.bz->in;
fd->read = bz_file_read;
fd->close = bz_file_close;
-
+
+ return 0;
+}
+#endif
+
+#ifdef USE_LZO
+/*
+ * LZO-compressed file support (standard lzop container format).
+ *
+ * Uses in-process liblzo2 decompression, following the same pattern as
+ * .gz (zlib) / .bz2 (bzlib) / .zst (zstd). The file format is auto-detected
+ * in lzo_file_open() by probing the first 9 bytes:
+ *
+ * - standard lzop container format: 9-byte magic + full header + block
+ * checksums
+ * - Hadoop Raw LZO format: no magic/header/checksum, plain LZO block
+ * stream (handled as a bonus path; only the standard lzop format is
+ * advertised via the .lzo extension)
+ *
+ * lzop header layout:
+ * magic 9 bytes fixed magic \x89LZO\x00\x0d\x0a\x1a\x0a
+ * version 2 bytes version (big endian)
+ * lib_version 2 bytes library version
+ * ver_needed 2 bytes minimum version required to decompress
+ * method 1 byte compression algorithm
+ * level 1 byte compression level
+ * flags 4 bytes flags (control checksum types etc.)
+ * mode 4 bytes file mode
+ * mtime_low 4 bytes mtime low 32 bits
+ * mtime_high 4 bytes mtime high 32 bits
+ * [extra_ver] 1 byte if F_H_EXTRA_FIELD(0x40) set
+ * [filter] 4 bytes if F_H_FILTER(0x800) set
+ * name_len 1 byte original file name length
+ * name N bytes original file name
+ * [path_len] 4 bytes if F_H_PATH(0x2000) set
+ * [path] N bytes if F_H_PATH(0x2000) set
+ * checksum 4 bytes header checksum
+ *
+ * Data block layout (common to both formats):
+ * uncomp_len 4 bytes decompressed size, big endian (0 = EOF marker)
+ * comp_len 4 bytes compressed size, big endian
+ * [d_adler32] 4 bytes adler32 of decompressed data (F_ADLER32_D=0x01)
+ * [d_crc32] 4 bytes crc32 of decompressed data (F_CRC32_D=0x100)
+ * [c_adler32] 4 bytes adler32 of compressed data (F_ADLER32_C=0x02)
+ * [c_crc32] 4 bytes crc32 of compressed data (F_CRC32_C=0x200)
+ * data comp_len bytes LZO compressed data (or raw data if
+ * incompressible)
+ *
+ * The only difference between the two formats is whether the magic/header
+ * and checksum fields are present. The decompression path is driven by
+ * flags: Raw LZO format has flags=0 so all checksum logic is naturally
+ * skipped. The 9 probe bytes (which belong to the first data block in Raw
+ * LZO) are cached in peek_buf and consumed first by lzo_read_peek /
+ * lzo_read_uint32_peek to keep byte alignment.
+ */
+
+/* LZO block buffer size: standard lzop default block size is 256KB */
+#define LZO_BUFFER_SIZE (256 * 1024)
+
+/*
+ * LZO decompression state structure (complete definition; gfile.h only
+ * declares the pointer). Same heap-allocated double buffer design as
+ * zlib_stuff / bzlib_stuff.
+ */
+struct lzo_stuff
+{
+ int out_size;
+ int out_pos;
+ int eof;
+ unsigned int flags; /* flags from lzop header; 0 for raw
LZO */
+ bool_t has_lzop_header; /* TRUE=standard lzop, FALSE=raw LZO */
+ int peek_size; /* valid bytes in peek_buf */
+ int peek_pos; /* current read offset in peek_buf */
+ char peek_buf[9]; /* probe buffer (at most 9 bytes) */
+ char in[LZO_BUFFER_SIZE];
+ char out[LZO_BUFFER_SIZE];
+};
+
+/* lzop file magic: \x89 L Z O \x00 \x0d \x0a \x1a \x0a */
+static const unsigned char lzop_magic[9] = {
+ 0x89, 0x4c, 0x5a, 0x4f, 0x00, 0x0d, 0x0a, 0x1a, 0x0a
+};
+
+/* lzop header flag bits (from lzop-1.03/src/conf.h) */
+#define LZOP_F_ADLER32_D 0x00000001 /* adler32 checksum of decompressed
data */
+#define LZOP_F_ADLER32_C 0x00000002 /* adler32 checksum of compressed
data */
+#define LZOP_F_CRC32_D 0x00000100 /* crc32 checksum of decompressed
data */
+#define LZOP_F_CRC32_C 0x00000200 /* crc32 checksum of compressed data
*/
+
+/*
+ * Helper: read exactly n bytes from the underlying file descriptor.
+ * Only local files are supported here, so this just wraps read_and_retry
+ * (same as the reads inside gz_file_read / bz_file_read). A short read
+ * means the file was truncated and is reported to the caller.
+ */
+static ssize_t
+read_block_bytes(gfile_t *fd, void *buf, size_t n)
+{
+ size_t total = 0;
+ char *p = (char *) buf;
+
+ while (total < n)
+ {
+ ssize_t r = read_and_retry(fd, p + total, n - total);
+
+ if (r == 0)
+ break; /* EOF, return what we have */
+ if (r < 0)
+ return -1; /* read error */
+ total += r;
+ }
+ return (ssize_t) total;
+}
+
+/*
+ * Helper: read one big-endian uint32 from the file.
+ * All multi-byte integers in the lzop format are big endian.
+ * Returns 0 on success, -1 on error (read failure or truncation).
+ */
+static int
+read_block_uint32(gfile_t *fd, uint32_t *val)
+{
+ unsigned char b[4];
+
+ if (read_block_bytes(fd, b, 4) < 4)
+ return -1;
+ *val = ((uint32_t) b[0] << 24) | ((uint32_t) b[1] << 16) |
+ ((uint32_t) b[2] << 8) | ((uint32_t) b[3]);
+ return 0;
+}
+
+/*
+ * peek helper: read from peek_buf first, then fall back to the file.
+ *
+ * During format probing, the first bytes of the file may already have been
+ * consumed. For the standard lzop path peek_buf is always empty, so these
+ * helpers degenerate to plain file reads. For the Raw LZO path, the probe
+ * bytes belong to the first data block and are consumed gradually here.
+ */
+static ssize_t
+lzo_read_peek(gfile_t *fd, void *buf, size_t n)
+{
+ struct lzo_stuff *z = fd->u.lzo;
+ size_t total = 0;
+ char *p = (char *) buf;
+
+ /* consume the probe bytes first */
+ if (z->peek_pos < z->peek_size)
+ {
+ size_t avail = (size_t)(z->peek_size - z->peek_pos);
Review Comment:
struct lzo_stuff
{
int out_size;
int out_pos;
int eof;
unsigned int flags; /* flags from lzop header; 0 for raw
LZO */
bool_t has_lzop_header; /* TRUE=standard lzop, FALSE=raw LZO */
int peek_size; /* valid bytes in peek_buf */
int peek_pos; /* current read offset in peek_buf */
char peek_buf[9]; /* probe buffer (at most 9 bytes) */
char in[LZO_BUFFER_SIZE];
char out[LZO_BUFFER_SIZE];
};|
Dear reviewer, thank you for your attention. The peek_pos field actually
represents the current read position inside the internal peek_buf, which is
only a small probe buffer with a maximum size of 9 bytes. Therefore, the
potential issue of 32‑bit offset being insufficient for large files does not
exist in this case.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
