xiangfu0 commented on code in PR #20245:
URL: https://github.com/apache/pinot/pull/20245#discussion_r4566170651
##########
pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/tpch/TPCHQueryIntegrationTest.java:
##########
@@ -18,146 +18,316 @@
*/
package org.apache.pinot.integration.tests.tpch;
-import com.google.common.collect.ImmutableSet;
+import com.google.common.primitives.Doubles;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
-import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.sql.ResultSet;
import java.sql.Statement;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.annotation.Nullable;
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.pinot.client.ResultSetGroup;
+import org.apache.pinot.common.utils.config.TagNameUtils;
import org.apache.pinot.integration.tests.BaseClusterIntegrationTest;
import org.apache.pinot.integration.tests.ClusterIntegrationTestUtils;
+import org.apache.pinot.segment.spi.partition.PartitionFunction;
+import org.apache.pinot.segment.spi.partition.PartitionFunctionFactory;
+import org.apache.pinot.spi.config.table.ColumnPartitionConfig;
+import org.apache.pinot.spi.config.table.SegmentPartitionConfig;
import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.assignment.InstanceAssignmentConfig;
+import org.apache.pinot.spi.config.table.assignment.InstanceConstraintConfig;
+import org.apache.pinot.spi.config.table.assignment.InstancePartitionsType;
+import
org.apache.pinot.spi.config.table.assignment.InstanceReplicaGroupPartitionConfig;
+import org.apache.pinot.spi.config.table.assignment.InstanceTagPoolConfig;
import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.apache.pinot.tools.utils.JarUtils;
import org.apache.pinot.util.TestUtils;
import org.testng.Assert;
+import org.testng.SkipException;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
-// TODO: extract common functions from TPCHQueryIntegrationTest and
SSBQueryIntegrationTest
-/// Integration test that tests Pinot using TPCH data.
-/// Data is loaded into Pinot and H2 from /resources/examples/batch/tpch. The
dataset size is very small, please follow
-/// REAME.md to generate a larger dataset for better testing.
-/// Queries are executed against Pinot and H2, and the results are compared.
+
+/// Runs the 22 TPC-H queries on the multi-stage engine and compares the
results with H2.
+///
+/// The data is the TPC-H data at scale factor 0.01, generated with `dbgen`
(see `README.md` in this package). The
+/// data and the queries are in `examples/batch/tpch` of `pinot-tools`, the
same files that the TPC-H quickstart uses.
+///
+/// Each query runs in two forms:
+/// - `queries/<n>.sql` is the query text from the TPC-H specification, with
the default substitution parameters.
+/// - `queries/rewrites/<n>.sql` is the same query, rewritten with a join
order and join hints that suit the
+/// multi-stage engine (see `queries/README.md`). The hints trust the table
layout, so the test uses the layout
+/// that the rewrites need: two servers, a full copy of the small tables on
each server, and `orders` and
+/// `lineitem` partitioned and colocated by order key. When the layout and
the hints disagree, the query fails or
+/// returns wrong results, and the test fails in both cases.
+///
+/// H2 runs the specification text, and both forms must return the same rows
as H2, in the same order.
public class TPCHQueryIntegrationTest extends BaseClusterIntegrationTest {
- private static final int NUM_TPCH_QUERIES = 24;
+ private static final int NUM_TPCH_QUERIES = 22;
+ private static final String QUERY_FOLDER = "examples/batch/tpch/queries/";
+ private static final int NUM_SERVERS = 2;
+ private static final int NUM_ORDERKEY_PARTITIONS = 4;
+ private static final String ORDERKEY_PARTITION_FUNCTION = "Murmur";
+ private static final Set<String> REPLICATED_TABLES = Set.of("customer",
"nation", "part", "region", "supplier");
+ private static final Map<String, String> ORDERKEY_PARTITIONED_TABLES =
+ Map.of("lineitem", "l_orderkey", "orders", "o_orderkey");
+ // Number of rows of each table at scale factor 0.01.
+ private static final Map<String, Long> NUM_ROWS = Map.of("customer", 1_500L,
"lineitem", 60_175L, "nation", 25L,
+ "orders", 15_000L, "part", 2_000L, "partsupp", 8_000L, "region", 5L,
"supplier", 100L);
+ // The rewrites need the planner to keep the partitioning of a table scan
through a local join.
+ private static final String REWRITE_QUERY_OPTIONS = "SET
inferPartitionHint=true; ";
+ // Specification queries that the multi-stage engine cannot run, with the
reason. Their rewrites still run.
+ private static final Map<Integer, String> SKIPPED_SPECIFICATION_QUERIES =
Map.of(21,
+ "the plan for the EXISTS and NOT EXISTS subqueries builds a join with
more than 10M rows, even at scale factor "
+ + "0.01. See queries/rewrites/21.sql");
+ // H2 has no indexes by default, and without them the joins in Q3, Q5, Q9
and Q19 take tens of seconds.
+ private static final Map<String, List<String>> H2_INDEXED_COLUMNS = Map.of(
+ "customer", List.of("c_custkey", "c_nationkey"),
+ "lineitem", List.of("l_orderkey", "l_partkey", "l_suppkey"),
+ "nation", List.of("n_nationkey", "n_regionkey"),
+ "orders", List.of("o_orderkey", "o_custkey"),
+ "part", List.of("p_partkey"),
+ "partsupp", List.of("ps_partkey", "ps_suppkey"),
+ "region", List.of("r_regionkey"),
+ "supplier", List.of("s_suppkey", "s_nationkey"));
- // Pinot queries 15, 16, 17 fail due to lack of support for views.
- // Pinot queries 23 fail due to java heap space problem or timeout.
- private static final Set<Integer> EXEMPT_QUERIES = ImmutableSet.of(15, 16,
17, 23);
+ // Both forms of a query compare with the same H2 result, so each H2 query
runs once.
Review Comment:
Nit: this comment describes `_h2Results`, but the new tolerance constants
now come between the comment and that field. Moving the constants above the
comment keeps the comment next to the field it describes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
