[pinot] branch master updated: Adding zstd examples (#11808)

[xiangfu]

This is an automated email from the ASF dual-hosted git repository.

xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 51335a2db1 Adding zstd examples (#11808)
51335a2db1 is described below

commit 51335a2db1720ac18da8818a4b565b557a0c667b
Author: Xiang Fu <xiangfu.1...@gmail.com>
AuthorDate: Mon Oct 16 12:08:01 2023 -0700

    Adding zstd examples (#11808)
---
 .../parquet/ParquetNativeRecordReaderFullTest.java    |   2 ++
 .../inputformat/parquet/ParquetRecordReaderTest.java  |  12 +++++++-----
 .../src/test/resources/airlineStats.zstd.parquet      | Bin 0 -> 891041 bytes
 .../src/test/resources/baseballStats.zstd.parquet     | Bin 0 -> 1609345 bytes
 4 files changed, 9 insertions(+), 5 deletions(-)

diff --git 
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReaderFullTest.java
 
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReaderFullTest.java
index 93edf424dd..f0438a0508 100644
--- 
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReaderFullTest.java
+++ 
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetNativeRecordReaderFullTest.java
@@ -30,6 +30,8 @@ public class ParquetNativeRecordReaderFullTest {
   @Test
   protected void testReadDataSet1()
       throws Exception {
+    testParquetFile("airlineStats.zstd.parquet");
+    testParquetFile("baseballStats.zstd.parquet");
     testParquetFile("test-data/gzip-nation.impala.parquet");
     //testParquetFile("test-data/nation.dict.parquet");
     testParquetFile("test-data/nation.impala.parquet");
diff --git 
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReaderTest.java
 
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReaderTest.java
index 9c861237e1..8133a18850 100644
--- 
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReaderTest.java
+++ 
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/java/org/apache/pinot/plugin/inputformat/parquet/ParquetRecordReaderTest.java
@@ -39,10 +39,6 @@ import org.testng.annotations.Test;
 
 
 public class ParquetRecordReaderTest extends AbstractRecordReaderTest {
-  private final File _testParquetFileWithInt96AndDecimal =
-      new 
File(getClass().getClassLoader().getResource("test-file-with-int96-and-decimal.snappy.parquet").getFile());
-
-  private static final int NUM_RECORDS_TEST_PARQUET_WITH_INT96 = 1965;
 
   @Override
   protected RecordReader createRecordReader(File file)
@@ -99,8 +95,13 @@ public class ParquetRecordReaderTest extends 
AbstractRecordReaderTest {
   public void testParquetNativeRecordReader()
       throws IOException {
     ParquetNativeRecordReader nativeRecordReader = new 
ParquetNativeRecordReader();
-    nativeRecordReader.init(_testParquetFileWithInt96AndDecimal, 
ImmutableSet.of(), new ParquetRecordReaderConfig());
+    nativeRecordReader.init(
+        new 
File(getClass().getClassLoader().getResource("test-file-with-int96-and-decimal.snappy.parquet").getFile()),
+        ImmutableSet.of(), new ParquetRecordReaderConfig());
     testReadParquetFile(nativeRecordReader, 1965);
+    nativeRecordReader.init(new 
File(getClass().getClassLoader().getResource("airlineStats.zstd.parquet").getFile()),
+        ImmutableSet.of(), new ParquetRecordReaderConfig());
+    testReadParquetFile(nativeRecordReader, 19492);
     nativeRecordReader.init(_dataFile, ImmutableSet.of(), new 
ParquetRecordReaderConfig());
     testReadParquetFile(nativeRecordReader, SAMPLE_RECORDS_SIZE);
   }
@@ -129,6 +130,7 @@ public class ParquetRecordReaderTest extends 
AbstractRecordReaderTest {
     testComparison(new 
File(getClass().getClassLoader().getResource("test-comparison.gz.parquet").getFile()),
 363667);
     testComparison(new 
File(getClass().getClassLoader().getResource("test-comparison.snappy.parquet").getFile()),
 2870);
     testComparison(new 
File(getClass().getClassLoader().getResource("baseballStats.snappy.parquet").getFile()),
 97889);
+    testComparison(new 
File(getClass().getClassLoader().getResource("baseballStats.zstd.parquet").getFile()),
 97889);
     testComparison(new 
File(getClass().getClassLoader().getResource("githubEvents.snappy.parquet").getFile()),
 10000);
     testComparison(new 
File(getClass().getClassLoader().getResource("starbucksStores.snappy.parquet").getFile()),
 6443);
     testComparison(new 
File(getClass().getClassLoader().getResource("airlineStats.snappy.parquet").getFile()),
 19492);
diff --git 
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/airlineStats.zstd.parquet
 
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/airlineStats.zstd.parquet
new file mode 100644
index 0000000000..3ec4bee99a
Binary files /dev/null and 
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/airlineStats.zstd.parquet
 differ
diff --git 
a/pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/baseballStats.zstd.parquet
 
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/baseballStats.zstd.parquet
new file mode 100644
index 0000000000..a14ec8a455
Binary files /dev/null and 
b/pinot-plugins/pinot-input-format/pinot-parquet/src/test/resources/baseballStats.zstd.parquet
 differ


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org