This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new bebd2b43a3 [Multi-stage] Support lookup join (#13966)
bebd2b43a3 is described below
commit bebd2b43a304a3cebbdba3214bde38c101a85422
Author: Xiaotian (Jackie) Jiang <[email protected]>
AuthorDate: Tue Oct 8 15:49:22 2024 -0700
[Multi-stage] Support lookup join (#13966)
---
.gitignore | 3 +-
pinot-common/src/main/proto/plan.proto | 6 +
.../pinot/calcite/rel/hint/PinotHintOptions.java | 11 +-
.../rel/rules/PinotJoinExchangeNodeInsertRule.java | 38 +-
.../rel/rules/PinotJoinToDynamicBroadcastRule.java | 48 +-
.../planner/logical/RelToPlanNodeConverter.java | 70 +-
.../pinot/query/planner/plannode/JoinNode.java | 21 +-
.../pinot/query/planner/plannode/PlanNode.java | 3 +-
.../query/planner/serde/PlanNodeDeserializer.java | 14 +-
.../query/planner/serde/PlanNodeSerializer.java | 14 +-
.../apache/pinot/query/routing/WorkerManager.java | 92 +-
.../query/runtime/InStageStatsTreeBuilder.java | 7 +-
.../query/runtime/operator/HashJoinOperator.java | 45 +-
.../LeafStageTransferableBlockOperator.java | 12 +
.../query/runtime/operator/LookupJoinOperator.java | 260 +++
.../query/runtime/operator/MultiStageOperator.java | 11 +-
.../query/runtime/plan/PlanNodeToOpChain.java | 11 +-
.../plan/server/ServerPlanRequestVisitor.java | 51 +-
.../runtime/operator/HashJoinOperatorTest.java | 20 +-
.../runtime/operator/MultiStageAccountingTest.java | 2 +-
.../plan/pipeline/PipelineBreakerExecutorTest.java | 6 +-
.../pinot/tools/LookupJoinEngineQuickStart.java | 67 +
.../colocated/userGroups/userGroups_schema.json | 6 +-
.../lookup/userGroupsDim/ingestionJobSpec.yaml | 140 ++
.../batch/lookup/userGroupsDim/rawdata/p0.csv | 8 +
.../batch/lookup/userGroupsDim/rawdata/p1.csv | 7 +
.../batch/lookup/userGroupsDim/rawdata/p2.csv | 9 +
.../batch/lookup/userGroupsDim/rawdata/p3.csv | 2455 ++++++++++++++++++++
.../batch/lookup/userGroupsDim/rawdata/p4.csv | 5 +
.../batch/lookup/userGroupsDim/rawdata/p5.csv | 5 +
.../batch/lookup/userGroupsDim/rawdata/p6.csv | 5 +
.../batch/lookup/userGroupsDim/rawdata/p7.csv | 8 +
.../userGroupsDim_offline_table_config.json | 18 +
.../userGroupsDim/userGroupsDim_schema.json} | 7 +-
34 files changed, 3327 insertions(+), 158 deletions(-)
diff --git a/.gitignore b/.gitignore
index 00404492f8..71aa91be6c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,7 +6,8 @@ cscope.*
.externalToolBuilders/
maven-eclipse.xml
target/
-examples/
+/examples/
+/logs/
bin/
*/bin/
.idea
diff --git a/pinot-common/src/main/proto/plan.proto
b/pinot-common/src/main/proto/plan.proto
index 9c5caa5edc..06b2f0910c 100644
--- a/pinot-common/src/main/proto/plan.proto
+++ b/pinot-common/src/main/proto/plan.proto
@@ -83,11 +83,17 @@ enum JoinType {
ANTI = 5;
}
+enum JoinStrategy {
+ HASH = 0;
+ LOOKUP = 1;
+}
+
message JoinNode {
JoinType joinType = 1;
repeated int32 leftKeys = 2;
repeated int32 rightKeys = 3;
repeated Expression nonEquiConditions = 4;
+ JoinStrategy joinStrategy = 5;
}
enum ExchangeType {
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/hint/PinotHintOptions.java
b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/hint/PinotHintOptions.java
index f19fa8a705..431d741e4c 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/hint/PinotHintOptions.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/hint/PinotHintOptions.java
@@ -61,20 +61,27 @@ public class PinotHintOptions {
public static class JoinHintOptions {
public static final String JOIN_STRATEGY = "join_strategy";
+ // "hash" is the default strategy for non-SEMI joins
+ public static final String HASH_JOIN_STRATEGY = "hash";
+ // "dynamic_broadcast" is the default strategy for SEMI joins
public static final String DYNAMIC_BROADCAST_JOIN_STRATEGY =
"dynamic_broadcast";
- public static final String HASH_TABLE_JOIN_STRATEGY = "hash_table";
+ // "lookup" can be used when the right table is a dimension table
replicated to all workers
+ public static final String LOOKUP_JOIN_STRATEGY = "lookup";
+
/**
* Max rows allowed to build the right table hash collection.
*/
public static final String MAX_ROWS_IN_JOIN = "max_rows_in_join";
+
/**
* Mode when join overflow happens, supported values: THROW or BREAK.
* THROW(default): Break right table build process, and throw exception,
no JOIN with left table performed.
* BREAK: Break right table build process, continue to perform JOIN
operation, results might be partial.
*/
public static final String JOIN_OVERFLOW_MODE = "join_overflow_mode";
+
/**
- * Indicat that the join operator(s) within a certain selection scope are
colocated
+ * Indicates that the join operator(s) within a certain selection scope
are colocated
*/
public static final String IS_COLOCATED_BY_JOIN_KEYS =
"is_colocated_by_join_keys";
}
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotJoinExchangeNodeInsertRule.java
b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotJoinExchangeNodeInsertRule.java
index 37f12fbd08..bb7ee8aed7 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotJoinExchangeNodeInsertRule.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotJoinExchangeNodeInsertRule.java
@@ -25,6 +25,8 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinInfo;
import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.pinot.calcite.rel.hint.PinotHintOptions;
+import org.apache.pinot.calcite.rel.hint.PinotHintStrategyTable;
import org.apache.pinot.calcite.rel.logical.PinotLogicalExchange;
@@ -48,24 +50,32 @@ public class PinotJoinExchangeNodeInsertRule extends
RelOptRule {
@Override
public void onMatch(RelOptRuleCall call) {
Join join = call.rel(0);
- RelNode leftInput = join.getInput(0);
- RelNode rightInput = join.getInput(1);
-
- RelNode leftExchange;
- RelNode rightExchange;
+ RelNode left = PinotRuleUtils.unboxRel(join.getInput(0));
+ RelNode right = PinotRuleUtils.unboxRel(join.getInput(1));
JoinInfo joinInfo = join.analyzeCondition();
-
- if (joinInfo.leftKeys.isEmpty()) {
- // when there's no JOIN key, use broadcast.
- leftExchange = PinotLogicalExchange.create(leftInput,
RelDistributions.RANDOM_DISTRIBUTED);
- rightExchange = PinotLogicalExchange.create(rightInput,
RelDistributions.BROADCAST_DISTRIBUTED);
+ String joinStrategy =
PinotHintStrategyTable.getHintOption(join.getHints(),
PinotHintOptions.JOIN_HINT_OPTIONS,
+ PinotHintOptions.JoinHintOptions.JOIN_STRATEGY);
+ RelNode newLeft;
+ RelNode newRight;
+ if
(PinotHintOptions.JoinHintOptions.LOOKUP_JOIN_STRATEGY.equals(joinStrategy)) {
+ // Lookup join - add local exchange on the left side
+ newLeft = PinotLogicalExchange.create(left, RelDistributions.SINGLETON);
+ newRight = right;
} else {
- // when join key exists, use hash distribution.
- leftExchange = PinotLogicalExchange.create(leftInput,
RelDistributions.hash(joinInfo.leftKeys));
- rightExchange = PinotLogicalExchange.create(rightInput,
RelDistributions.hash(joinInfo.rightKeys));
+ // Regular join - add exchange on both sides
+ if (joinInfo.leftKeys.isEmpty()) {
+ // Broadcast the right side if there is no join key
+ newLeft = PinotLogicalExchange.create(left,
RelDistributions.RANDOM_DISTRIBUTED);
+ newRight = PinotLogicalExchange.create(right,
RelDistributions.BROADCAST_DISTRIBUTED);
+ } else {
+ // Use hash exchange when there are join keys
+ newLeft = PinotLogicalExchange.create(left,
RelDistributions.hash(joinInfo.leftKeys));
+ newRight = PinotLogicalExchange.create(right,
RelDistributions.hash(joinInfo.rightKeys));
+ }
}
- call.transformTo(join.copy(join.getTraitSet(), join.getCondition(),
leftExchange, rightExchange, join.getJoinType(),
+ // TODO: Consider creating different JOIN Rel for each join strategy
+ call.transformTo(join.copy(join.getTraitSet(), join.getCondition(),
newLeft, newRight, join.getJoinType(),
join.isSemiJoinDone()));
}
}
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotJoinToDynamicBroadcastRule.java
b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotJoinToDynamicBroadcastRule.java
index ed86a6fcc2..c1924eb1d5 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotJoinToDynamicBroadcastRule.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/calcite/rel/rules/PinotJoinToDynamicBroadcastRule.java
@@ -18,8 +18,6 @@
*/
package org.apache.pinot.calcite.rel.rules;
-import java.util.Collections;
-import java.util.List;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.hep.HepRelVertex;
@@ -35,7 +33,6 @@ import org.apache.pinot.calcite.rel.hint.PinotHintOptions;
import org.apache.pinot.calcite.rel.hint.PinotHintStrategyTable;
import org.apache.pinot.calcite.rel.logical.PinotLogicalExchange;
import org.apache.pinot.calcite.rel.logical.PinotRelExchangeType;
-import org.apache.zookeeper.common.StringUtils;
/**
@@ -125,25 +122,27 @@ public class PinotJoinToDynamicBroadcastRule extends
RelOptRule {
@Override
public boolean matches(RelOptRuleCall call) {
Join join = call.rel(0);
- String joinStrategyString =
- PinotHintStrategyTable.getHintOption(join.getHints(),
PinotHintOptions.JOIN_HINT_OPTIONS,
- PinotHintOptions.JoinHintOptions.JOIN_STRATEGY);
- List<String> joinStrategies =
- joinStrategyString != null ? StringUtils.split(joinStrategyString,
",") : Collections.emptyList();
- boolean explicitOtherStrategy = !joinStrategies.isEmpty() &&
!joinStrategies.contains(
- PinotHintOptions.JoinHintOptions.DYNAMIC_BROADCAST_JOIN_STRATEGY);
+ // Do not apply this rule if join strategy is explicitly set to something
other than dynamic broadcast
+ String joinStrategy =
PinotHintStrategyTable.getHintOption(join.getHints(),
PinotHintOptions.JOIN_HINT_OPTIONS,
+ PinotHintOptions.JoinHintOptions.JOIN_STRATEGY);
+ if (joinStrategy != null && !joinStrategy.equals(
+ PinotHintOptions.JoinHintOptions.DYNAMIC_BROADCAST_JOIN_STRATEGY)) {
+ return false;
+ }
+
+ // Do not apply this rule if it is not a SEMI join
JoinInfo joinInfo = join.analyzeCondition();
+ if (join.getJoinType() != JoinRelType.SEMI ||
!joinInfo.nonEquiConditions.isEmpty()
+ || joinInfo.leftKeys.size() != 1) {
+ return false;
+ }
+
+ // Apply this rule if the left side can be pushed as dynamic exchange
RelNode left = ((HepRelVertex) join.getLeft()).getCurrentRel();
RelNode right = ((HepRelVertex) join.getRight()).getCurrentRel();
- return left instanceof Exchange && right instanceof Exchange
- // left side can be pushed as dynamic exchange
- && PinotRuleUtils.canPushDynamicBroadcastToLeaf(left.getInput(0))
- // default enable dynamic broadcast for SEMI join unless other join
strategy were specified
- && !explicitOtherStrategy
- // condition for SEMI join
- && join.getJoinType() == JoinRelType.SEMI &&
joinInfo.nonEquiConditions.isEmpty()
- && joinInfo.leftKeys.size() == 1;
+ return left instanceof Exchange && right instanceof Exchange &&
PinotRuleUtils.canPushDynamicBroadcastToLeaf(
+ left.getInput(0));
}
@Override
@@ -158,15 +157,10 @@ public class PinotJoinToDynamicBroadcastRule extends
RelOptRule {
boolean isColocatedJoin =
PinotHintStrategyTable.isHintOptionTrue(join.getHints(),
PinotHintOptions.JOIN_HINT_OPTIONS,
PinotHintOptions.JoinHintOptions.IS_COLOCATED_BY_JOIN_KEYS);
- PinotLogicalExchange dynamicBroadcastExchange;
- RelNode rightInput = right.getInput();
- if (isColocatedJoin) {
- RelDistribution dist =
RelDistributions.hash(join.analyzeCondition().rightKeys);
- dynamicBroadcastExchange = PinotLogicalExchange.create(rightInput, dist,
PinotRelExchangeType.PIPELINE_BREAKER);
- } else {
- RelDistribution dist = RelDistributions.BROADCAST_DISTRIBUTED;
- dynamicBroadcastExchange = PinotLogicalExchange.create(rightInput, dist,
PinotRelExchangeType.PIPELINE_BREAKER);
- }
+ RelDistribution relDistribution = isColocatedJoin ?
RelDistributions.hash(join.analyzeCondition().rightKeys)
+ : RelDistributions.BROADCAST_DISTRIBUTED;
+ PinotLogicalExchange dynamicBroadcastExchange =
+ PinotLogicalExchange.create(right.getInput(), relDistribution,
PinotRelExchangeType.PIPELINE_BREAKER);
call.transformTo(join.copy(join.getTraitSet(), join.getCondition(),
left.getInput(), dynamicBroadcastExchange,
join.getJoinType(), join.isSemiJoinDone()));
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java
index cd60856c17..5991b44ed8 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/logical/RelToPlanNodeConverter.java
@@ -19,6 +19,7 @@
package org.apache.pinot.query.planner.logical;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Sets;
import java.util.ArrayList;
import java.util.List;
@@ -32,9 +33,12 @@ import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rel.core.Exchange;
import org.apache.calcite.rel.core.JoinInfo;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.core.SetOp;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.core.Window;
+import org.apache.calcite.rel.hint.RelHint;
import org.apache.calcite.rel.logical.LogicalFilter;
import org.apache.calcite.rel.logical.LogicalJoin;
import org.apache.calcite.rel.logical.LogicalProject;
@@ -45,13 +49,18 @@ import org.apache.calcite.rel.logical.LogicalWindow;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rel.type.RelRecordType;
+import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.pinot.calcite.rel.hint.PinotHintOptions;
+import org.apache.pinot.calcite.rel.hint.PinotHintStrategyTable;
import org.apache.pinot.calcite.rel.logical.PinotLogicalAggregate;
import org.apache.pinot.calcite.rel.logical.PinotLogicalExchange;
import org.apache.pinot.calcite.rel.logical.PinotLogicalSortExchange;
import org.apache.pinot.calcite.rel.logical.PinotRelExchangeType;
+import org.apache.pinot.calcite.rel.rules.PinotRuleUtils;
import org.apache.pinot.common.metrics.BrokerMeter;
import org.apache.pinot.common.metrics.BrokerMetrics;
import org.apache.pinot.common.utils.DataSchema;
@@ -264,11 +273,62 @@ public final class RelToPlanNodeConverter {
convertInputs(node.getInputs()), tableName, columns);
}
- private JoinNode convertLogicalJoin(LogicalJoin node) {
- JoinInfo joinInfo = node.analyzeCondition();
- return new JoinNode(DEFAULT_STAGE_ID, toDataSchema(node.getRowType()),
NodeHint.fromRelHints(node.getHints()),
- convertInputs(node.getInputs()), node.getJoinType(),
joinInfo.leftKeys, joinInfo.rightKeys,
- RexExpressionUtils.fromRexNodes(joinInfo.nonEquiConditions));
+ private JoinNode convertLogicalJoin(LogicalJoin join) {
+ JoinInfo joinInfo = join.analyzeCondition();
+ DataSchema dataSchema = toDataSchema(join.getRowType());
+ List<PlanNode> inputs = convertInputs(join.getInputs());
+ JoinRelType joinType = join.getJoinType();
+
+ // Run some validations for join
+ Preconditions.checkState(inputs.size() == 2, "Join should have exactly 2
inputs, got: %s", inputs.size());
+ PlanNode left = inputs.get(0);
+ PlanNode right = inputs.get(1);
+ int numLeftColumns = left.getDataSchema().size();
+ int numResultColumns = dataSchema.size();
+ if (joinType.projectsRight()) {
+ int numRightColumns = right.getDataSchema().size();
+ Preconditions.checkState(numLeftColumns + numRightColumns ==
numResultColumns,
+ "Invalid number of columns for join type: %s, left: %s, right: %s,
result: %s", joinType, numLeftColumns,
+ numRightColumns, numResultColumns);
+ } else {
+ Preconditions.checkState(numLeftColumns == numResultColumns,
+ "Invalid number of columns for join type: %s, left: %s, result: %s",
joinType, numLeftColumns,
+ numResultColumns);
+ }
+
+ // Check if the join hint specifies the join strategy
+ JoinNode.JoinStrategy joinStrategy;
+ ImmutableList<RelHint> relHints = join.getHints();
+ String joinStrategyHint = PinotHintStrategyTable.getHintOption(relHints,
PinotHintOptions.JOIN_HINT_OPTIONS,
+ PinotHintOptions.JoinHintOptions.JOIN_STRATEGY);
+ if
(PinotHintOptions.JoinHintOptions.LOOKUP_JOIN_STRATEGY.equals(joinStrategyHint))
{
+ joinStrategy = JoinNode.JoinStrategy.LOOKUP;
+
+ // Run some validations for lookup join
+ Preconditions.checkArgument(!joinInfo.leftKeys.isEmpty(), "Lookup join
requires join keys");
+ // Right table should be a dimension table, and the right input should
be an identifier only ProjectNode over
+ // TableScanNode.
+ RelNode rightInput = PinotRuleUtils.unboxRel(join.getRight());
+ Preconditions.checkState(rightInput instanceof Project, "Right input for
lookup join must be a Project, got: %s",
+ rightInput.getClass().getSimpleName());
+ Project project = (Project) rightInput;
+ for (RexNode node : project.getProjects()) {
+ Preconditions.checkState(node instanceof RexInputRef,
+ "Right input for lookup join must be an identifier (RexInputRef)
only Project, got: %s in project",
+ node.getClass().getSimpleName());
+ }
+ RelNode projectInput = PinotRuleUtils.unboxRel(project.getInput());
+ Preconditions.checkState(projectInput instanceof TableScan,
+ "Right input for lookup join must be a Project over TableScan, got
Project over: %s",
+ projectInput.getClass().getSimpleName());
+ } else {
+ // TODO: Consider adding DYNAMIC_BROADCAST as a separate join strategy
+ joinStrategy = JoinNode.JoinStrategy.HASH;
+ }
+
+ return new JoinNode(DEFAULT_STAGE_ID, dataSchema,
NodeHint.fromRelHints(relHints), inputs, joinType,
+ joinInfo.leftKeys, joinInfo.rightKeys,
RexExpressionUtils.fromRexNodes(joinInfo.nonEquiConditions),
+ joinStrategy);
}
private List<PlanNode> convertInputs(List<RelNode> inputs) {
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/JoinNode.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/JoinNode.java
index ea15b7c715..c07392c298 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/JoinNode.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/JoinNode.java
@@ -30,14 +30,17 @@ public class JoinNode extends BasePlanNode {
private final List<Integer> _leftKeys;
private final List<Integer> _rightKeys;
private final List<RexExpression> _nonEquiConditions;
+ private final JoinStrategy _joinStrategy;
public JoinNode(int stageId, DataSchema dataSchema, NodeHint nodeHint,
List<PlanNode> inputs, JoinRelType joinType,
- List<Integer> leftKeys, List<Integer> rightKeys, List<RexExpression>
nonEquiConditions) {
+ List<Integer> leftKeys, List<Integer> rightKeys, List<RexExpression>
nonEquiConditions,
+ JoinStrategy joinStrategy) {
super(stageId, dataSchema, nodeHint, inputs);
_joinType = joinType;
_leftKeys = leftKeys;
_rightKeys = rightKeys;
_nonEquiConditions = nonEquiConditions;
+ _joinStrategy = joinStrategy;
}
public JoinRelType getJoinType() {
@@ -56,6 +59,10 @@ public class JoinNode extends BasePlanNode {
return _nonEquiConditions;
}
+ public JoinStrategy getJoinStrategy() {
+ return _joinStrategy;
+ }
+
@Override
public String explain() {
return "JOIN";
@@ -68,7 +75,8 @@ public class JoinNode extends BasePlanNode {
@Override
public PlanNode withInputs(List<PlanNode> inputs) {
- return new JoinNode(_stageId, _dataSchema, _nodeHint, inputs, _joinType,
_leftKeys, _rightKeys, _nonEquiConditions);
+ return new JoinNode(_stageId, _dataSchema, _nodeHint, inputs, _joinType,
_leftKeys, _rightKeys, _nonEquiConditions,
+ _joinStrategy);
}
@Override
@@ -84,11 +92,16 @@ public class JoinNode extends BasePlanNode {
}
JoinNode joinNode = (JoinNode) o;
return _joinType == joinNode._joinType && Objects.equals(_leftKeys,
joinNode._leftKeys) && Objects.equals(
- _rightKeys, joinNode._rightKeys) && Objects.equals(_nonEquiConditions,
joinNode._nonEquiConditions);
+ _rightKeys, joinNode._rightKeys) && Objects.equals(_nonEquiConditions,
joinNode._nonEquiConditions)
+ && _joinStrategy == joinNode._joinStrategy;
}
@Override
public int hashCode() {
- return Objects.hash(super.hashCode(), _joinType, _leftKeys, _rightKeys,
_nonEquiConditions);
+ return Objects.hash(super.hashCode(), _joinType, _leftKeys, _rightKeys,
_nonEquiConditions, _joinStrategy);
+ }
+
+ public enum JoinStrategy {
+ HASH, LOOKUP
}
}
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/PlanNode.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/PlanNode.java
index cadcb899bb..3994b82c67 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/PlanNode.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/plannode/PlanNode.java
@@ -81,7 +81,8 @@ public interface PlanNode {
} else {
hintOptions = Maps.newHashMapWithExpectedSize(numHints);
for (RelHint relHint : relHints) {
- hintOptions.put(relHint.hintName, relHint.kvOptions);
+ // Put the first matching hint to match the behavior of
PinotHintStrategyTable
+ hintOptions.putIfAbsent(relHint.hintName, relHint.kvOptions);
}
}
return new NodeHint(hintOptions);
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeDeserializer.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeDeserializer.java
index 026e9737f0..d90f03c73c 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeDeserializer.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeDeserializer.java
@@ -99,7 +99,8 @@ public class PlanNodeDeserializer {
Plan.JoinNode protoJoinNode = protoNode.getJoinNode();
return new JoinNode(protoNode.getStageId(), extractDataSchema(protoNode),
extractNodeHint(protoNode),
extractInputs(protoNode),
convertJoinType(protoJoinNode.getJoinType()), protoJoinNode.getLeftKeysList(),
- protoJoinNode.getRightKeysList(),
convertExpressions(protoJoinNode.getNonEquiConditionsList()));
+ protoJoinNode.getRightKeysList(),
convertExpressions(protoJoinNode.getNonEquiConditionsList()),
+ convertJoinStrategy(protoJoinNode.getJoinStrategy()));
}
private static MailboxReceiveNode
deserializeMailboxReceiveNode(Plan.PlanNode protoNode) {
@@ -274,6 +275,17 @@ public class PlanNodeDeserializer {
}
}
+ private static JoinNode.JoinStrategy convertJoinStrategy(Plan.JoinStrategy
joinStrategy) {
+ switch (joinStrategy) {
+ case HASH:
+ return JoinNode.JoinStrategy.HASH;
+ case LOOKUP:
+ return JoinNode.JoinStrategy.LOOKUP;
+ default:
+ throw new IllegalStateException("Unsupported JoinStrategy: " +
joinStrategy);
+ }
+ }
+
private static PinotRelExchangeType convertExchangeType(Plan.ExchangeType
exchangeType) {
switch (exchangeType) {
case STREAMING:
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeSerializer.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeSerializer.java
index be631c3733..00a21c05e9 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeSerializer.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/serde/PlanNodeSerializer.java
@@ -112,7 +112,8 @@ public class PlanNodeSerializer {
Plan.JoinNode joinNode =
Plan.JoinNode.newBuilder().setJoinType(convertJoinType(node.getJoinType())).addAllLeftKeys(node.getLeftKeys())
.addAllRightKeys(node.getRightKeys())
-
.addAllNonEquiConditions(convertExpressions(node.getNonEquiConditions())).build();
+
.addAllNonEquiConditions(convertExpressions(node.getNonEquiConditions()))
+
.setJoinStrategy(convertJoinStrategy(node.getJoinStrategy())).build();
builder.setJoinNode(joinNode);
return null;
}
@@ -265,6 +266,17 @@ public class PlanNodeSerializer {
}
}
+ private static Plan.JoinStrategy convertJoinStrategy(JoinNode.JoinStrategy
joinStrategy) {
+ switch (joinStrategy) {
+ case HASH:
+ return Plan.JoinStrategy.HASH;
+ case LOOKUP:
+ return Plan.JoinStrategy.LOOKUP;
+ default:
+ throw new IllegalStateException("Unsupported JoinStrategy: " +
joinStrategy);
+ }
+ }
+
private static Plan.ExchangeType convertExchangeType(PinotRelExchangeType
exchangeType) {
switch (exchangeType) {
case STREAMING:
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/routing/WorkerManager.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/routing/WorkerManager.java
index d256a19a00..ba3a709fed 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/routing/WorkerManager.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/routing/WorkerManager.java
@@ -19,6 +19,7 @@
package org.apache.pinot.query.routing;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -29,6 +30,7 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.calcite.rel.RelDistribution;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.calcite.rel.hint.PinotHintOptions;
import org.apache.pinot.core.routing.RoutingManager;
@@ -39,6 +41,8 @@ import org.apache.pinot.core.transport.ServerInstance;
import org.apache.pinot.query.planner.PlanFragment;
import org.apache.pinot.query.planner.physical.DispatchablePlanContext;
import org.apache.pinot.query.planner.physical.DispatchablePlanMetadata;
+import org.apache.pinot.query.planner.plannode.MailboxSendNode;
+import org.apache.pinot.query.planner.plannode.PlanNode;
import org.apache.pinot.query.planner.plannode.TableScanNode;
import org.apache.pinot.spi.config.table.TableType;
import
org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey;
@@ -84,16 +88,46 @@ public class WorkerManager {
}
private void assignWorkersToNonRootFragment(PlanFragment fragment,
DispatchablePlanContext context) {
- for (PlanFragment child : fragment.getChildren()) {
+ List<PlanFragment> children = fragment.getChildren();
+ for (PlanFragment child : children) {
assignWorkersToNonRootFragment(child, context);
}
- if
(isLeafPlan(context.getDispatchablePlanMetadataMap().get(fragment.getFragmentId())))
{
+ Map<Integer, DispatchablePlanMetadata> metadataMap =
context.getDispatchablePlanMetadataMap();
+ DispatchablePlanMetadata metadata =
metadataMap.get(fragment.getFragmentId());
+ boolean leafPlan = isLeafPlan(metadata);
+ if (isLocalExchange(children)) {
+ // If it is a local exchange (single child with SINGLETON distribution),
use the same worker assignment to avoid
+ // shuffling data.
+ // TODO: Support partition parallelism
+ DispatchablePlanMetadata childMetadata =
metadataMap.get(children.get(0).getFragmentId());
+
metadata.setWorkerIdToServerInstanceMap(childMetadata.getWorkerIdToServerInstanceMap());
+ metadata.setPartitionFunction(childMetadata.getPartitionFunction());
+ if (leafPlan) {
+ // Fake segments map for leaf plan
+ Set<Integer> workerIds =
metadata.getWorkerIdToServerInstanceMap().keySet();
+ Map<Integer, Map<String, List<String>>> workerIdToSegmentsMap =
+ Maps.newHashMapWithExpectedSize(workerIds.size());
+ for (Integer workerId : workerIds) {
+ workerIdToSegmentsMap.put(workerId, Map.of(TableType.OFFLINE.name(),
List.of()));
+ }
+ metadata.setWorkerIdToSegmentsMap(workerIdToSegmentsMap);
+ }
+ } else if (leafPlan) {
assignWorkersToLeafFragment(fragment, context);
} else {
assignWorkersToIntermediateFragment(fragment, context);
}
}
+ private boolean isLocalExchange(List<PlanFragment> children) {
+ if (children.size() != 1) {
+ return false;
+ }
+ PlanNode childPlanNode = children.get(0).getFragmentRoot();
+ return childPlanNode instanceof MailboxSendNode
+ && ((MailboxSendNode) childPlanNode).getDistributionType() ==
RelDistribution.Type.SINGLETON;
+ }
+
private static boolean isLeafPlan(DispatchablePlanMetadata metadata) {
return metadata.getScannedTables().size() == 1;
}
@@ -102,21 +136,22 @@ public class WorkerManager {
// Intermediate stage assign logic
// --------------------------------------------------------------------------
private void assignWorkersToIntermediateFragment(PlanFragment fragment,
DispatchablePlanContext context) {
+ List<PlanFragment> children = fragment.getChildren();
Map<Integer, DispatchablePlanMetadata> metadataMap =
context.getDispatchablePlanMetadataMap();
DispatchablePlanMetadata metadata =
metadataMap.get(fragment.getFragmentId());
- // If the first child is partitioned and can be inherent from this
intermediate stage, use the same worker
- // assignment to avoid shuffling data.
- // When partition parallelism is configured,
- // 1. create multiple intermediate stage workers on the same instance for
each worker in the first child if the
- // first child is a table scan. this is b/c we cannot pre-config
parallelism on leaf stage thus needs fan-out.
- // 2. ignore partition parallelism when first child is NOT table scan b/c
it would've done fan-out already.
- if (isPrePartitionAssignment(fragment, metadataMap)) {
- DispatchablePlanMetadata firstChildMetadata =
metadataMap.get(fragment.getChildren().get(0).getFragmentId());
+ if (isPrePartitionAssignment(children, metadataMap)) {
+ // If the first child is partitioned and can be inherent from this
intermediate stage, use the same worker
+ // assignment to avoid shuffling data.
+ // When partition parallelism is configured,
+ // 1. Create multiple intermediate stage workers on the same instance
for each worker in the first child if the
+ // first child is a table scan. this is b/c we cannot pre-config
parallelism on leaf stage thus needs fan-out.
+ // 2. Ignore partition parallelism when first child is NOT table scan
b/c it would've done fan-out already.
+ DispatchablePlanMetadata firstChildMetadata =
metadataMap.get(children.get(0).getFragmentId());
int partitionParallelism = firstChildMetadata.getPartitionParallelism();
Map<Integer, QueryServerInstance> childWorkerIdToServerInstanceMap =
firstChildMetadata.getWorkerIdToServerInstanceMap();
- if (partitionParallelism == 1 ||
firstChildMetadata.getScannedTables().size() == 0) {
+ if (partitionParallelism == 1 ||
firstChildMetadata.getScannedTables().isEmpty()) {
metadata.setWorkerIdToServerInstanceMap(childWorkerIdToServerInstanceMap);
} else {
int numChildWorkers = childWorkerIdToServerInstanceMap.size();
@@ -158,17 +193,18 @@ public class WorkerManager {
}
}
- private boolean isPrePartitionAssignment(PlanFragment fragment, Map<Integer,
DispatchablePlanMetadata> metadataMap) {
- List<PlanFragment> children = fragment.getChildren();
+ private boolean isPrePartitionAssignment(List<PlanFragment> children,
+ Map<Integer, DispatchablePlanMetadata> metadataMap) {
if (children.isEmpty()) {
return false;
}
// Now, is all children needs to be pre-partitioned by the same function
and size to allow pre-partition assignment
- // TODO1: when partition function is allowed to be configured in exchange
we can relax this condition
- // TODO2: pick the most colocate assignment instead of picking the first
children
+ // TODO:
+ // 1. When partition function is allowed to be configured in exchange we
can relax this condition
+ // 2. Pick the most colocate assignment instead of picking the first
children
String partitionFunction = null;
int partitionCount = 0;
- for (PlanFragment child : fragment.getChildren()) {
+ for (PlanFragment child : children) {
DispatchablePlanMetadata childMetadata =
metadataMap.get(child.getFragmentId());
if (!childMetadata.isPrePartitioned()) {
return false;
@@ -178,8 +214,9 @@ public class WorkerManager {
} else if
(!partitionFunction.equalsIgnoreCase(childMetadata.getPartitionFunction())) {
return false;
}
- int childComputedPartitionCount =
childMetadata.getWorkerIdToServerInstanceMap().size()
- * (isLeafPlan(childMetadata) ?
childMetadata.getPartitionParallelism() : 1);
+ int childComputedPartitionCount =
+ childMetadata.getWorkerIdToServerInstanceMap().size() *
(isLeafPlan(childMetadata)
+ ? childMetadata.getPartitionParallelism() : 1);
if (partitionCount == 0) {
partitionCount = childComputedPartitionCount;
} else if (childComputedPartitionCount != partitionCount) {
@@ -193,7 +230,7 @@ public class WorkerManager {
List<ServerInstance> serverInstances;
Set<String> tableNames = context.getTableNames();
Map<String, ServerInstance> enabledServerInstanceMap =
_routingManager.getEnabledServerInstanceMap();
- if (tableNames.size() == 0) {
+ if (tableNames.isEmpty()) {
// TODO: Short circuit it when no table needs to be scanned
// This could be the case from queries that don't actually fetch values
from the tables. In such cases the
// routing need not be tenant aware.
@@ -435,23 +472,20 @@ public class WorkerManager {
}
if (offlinePartitionInfo == null) {
partitionInfoMap[i] =
- new
PartitionInfo(realtimePartitionInfo._fullyReplicatedServers, null,
- realtimePartitionInfo._segments);
+ new
PartitionInfo(realtimePartitionInfo._fullyReplicatedServers, null,
realtimePartitionInfo._segments);
continue;
}
if (realtimePartitionInfo == null) {
partitionInfoMap[i] =
- new
PartitionInfo(offlinePartitionInfo._fullyReplicatedServers,
offlinePartitionInfo._segments,
- null);
+ new
PartitionInfo(offlinePartitionInfo._fullyReplicatedServers,
offlinePartitionInfo._segments, null);
continue;
}
Set<String> fullyReplicatedServers = new
HashSet<>(offlinePartitionInfo._fullyReplicatedServers);
fullyReplicatedServers.retainAll(realtimePartitionInfo._fullyReplicatedServers);
Preconditions.checkState(!fullyReplicatedServers.isEmpty(),
"Failed to find fully replicated server for partition: %s in
hybrid table: %s", i, tableName);
- partitionInfoMap[i] =
- new PartitionInfo(fullyReplicatedServers,
offlinePartitionInfo._segments,
- realtimePartitionInfo._segments);
+ partitionInfoMap[i] = new PartitionInfo(fullyReplicatedServers,
offlinePartitionInfo._segments,
+ realtimePartitionInfo._segments);
}
return new PartitionTableInfo(partitionInfoMap, timeBoundaryInfo);
} else if (offlineRoutingExists) {
@@ -496,8 +530,7 @@ public class WorkerManager {
for (int i = 0; i < numPartitions; i++) {
TablePartitionInfo.PartitionInfo partitionInfo =
tablePartitionInfoArr[i];
if (partitionInfo != null) {
- partitionInfoMap[i] =
- new PartitionInfo(partitionInfo._fullyReplicatedServers,
partitionInfo._segments, null);
+ partitionInfoMap[i] = new
PartitionInfo(partitionInfo._fullyReplicatedServers, partitionInfo._segments,
null);
}
}
return new PartitionTableInfo(partitionInfoMap, null);
@@ -511,8 +544,7 @@ public class WorkerManager {
for (int i = 0; i < numPartitions; i++) {
TablePartitionInfo.PartitionInfo partitionInfo =
tablePartitionInfoArr[i];
if (partitionInfo != null) {
- partitionInfoMap[i] =
- new PartitionInfo(partitionInfo._fullyReplicatedServers, null,
partitionInfo._segments);
+ partitionInfoMap[i] = new
PartitionInfo(partitionInfo._fullyReplicatedServers, null,
partitionInfo._segments);
}
}
return new PartitionTableInfo(partitionInfoMap, null);
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/InStageStatsTreeBuilder.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/InStageStatsTreeBuilder.java
index 389e96f3e8..048af39253 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/InStageStatsTreeBuilder.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/InStageStatsTreeBuilder.java
@@ -138,7 +138,12 @@ public class InStageStatsTreeBuilder implements
PlanNodeVisitor<ObjectNode, Void
@Override
public ObjectNode visitJoin(JoinNode node, Void context) {
- return recursiveCase(node, MultiStageOperator.Type.HASH_JOIN);
+ if (node.getJoinStrategy() == JoinNode.JoinStrategy.HASH) {
+ return recursiveCase(node, MultiStageOperator.Type.HASH_JOIN);
+ } else {
+ assert node.getJoinStrategy() == JoinNode.JoinStrategy.LOOKUP;
+ return recursiveCase(node, MultiStageOperator.Type.LOOKUP_JOIN);
+ }
}
@Override
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/HashJoinOperator.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/HashJoinOperator.java
index 9d845561c8..28cebdbcd3 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/HashJoinOperator.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/HashJoinOperator.java
@@ -53,16 +53,12 @@ import org.slf4j.LoggerFactory;
/**
- * This basic {@code BroadcastJoinOperator} implement a basic broadcast join
algorithm.
- * This algorithm assumes that the broadcast table has to fit in memory since
we are not supporting any spilling.
- *
- * For left join, inner join, right join and full join,
- * <p>It takes the right table as the broadcast side and materialize a hash
table. Then for each of the left table row,
- * it looks up for the corresponding row(s) from the hash table and create a
joint row.
- *
- * <p>For each of the data block received from the left table, it will
generate a joint data block.
- * We currently support left join, inner join, right join and full join.
- * The output is in the format of [left_row, right_row]
+ * This {@code HashJoinOperator} implements the hash join algorithm.
+ * <p>This algorithm assumes that the right table has to fit in memory since
we are not supporting any spilling. It
+ * reads the complete hash partitioned right table and materialize the data
into a hash table. Then for each of the left
+ * table row, it looks up for the corresponding row(s) from the hash table and
create a joint row.
+ * <p>For each of the data block received from the left table, it generates a
joint data block. The output is in the
+ * format of [left_row, right_row].
*/
// TODO: Move inequi out of hashjoin.
(https://github.com/apache/pinot/issues/9728)
// TODO: Support memory size based resource limit.
@@ -120,19 +116,17 @@ public class HashJoinOperator extends MultiStageOperator {
public HashJoinOperator(OpChainExecutionContext context, MultiStageOperator
leftInput, DataSchema leftSchema,
MultiStageOperator rightInput, JoinNode node) {
super(context);
- Preconditions.checkState(SUPPORTED_JOIN_TYPES.contains(node.getJoinType()),
- "Join type: " + node.getJoinType() + " is not supported!");
+ _leftInput = leftInput;
+ _rightInput = rightInput;
_joinType = node.getJoinType();
+ Preconditions.checkState(SUPPORTED_JOIN_TYPES.contains(_joinType), "Join
type: % is not supported for hash join",
+ _joinType);
+
_leftKeySelector = KeySelectorFactory.getKeySelector(node.getLeftKeys());
_rightKeySelector = KeySelectorFactory.getKeySelector(node.getRightKeys());
_leftColumnSize = leftSchema.size();
_resultSchema = node.getDataSchema();
_resultColumnSize = _resultSchema.size();
- Preconditions.checkState(_resultColumnSize >= _leftColumnSize,
- "Result column size: %s has to be greater than or equal to left column
size: %s", _resultColumnSize,
- _leftColumnSize);
- _leftInput = leftInput;
- _rightInput = rightInput;
List<RexExpression> nonEquiConditions = node.getNonEquiConditions();
_nonEquiEvaluators = new ArrayList<>(nonEquiConditions.size());
for (RexExpression nonEquiCondition : nonEquiConditions) {
@@ -292,7 +286,7 @@ public class HashJoinOperator extends MultiStageOperator {
return new TransferableBlock(rows, _resultSchema,
DataBlock.Type.ROW);
}
}
- return
TransferableBlockUtils.getEndOfStreamTransferableBlock(_leftSideStats);
+ return leftBlock;
}
assert leftBlock.isDataBlock();
List<Object[]> rows = buildJoinedRows(leftBlock);
@@ -377,7 +371,7 @@ public class HashJoinOperator extends MultiStageOperator {
Object key = _leftKeySelector.getKey(leftRow);
// SEMI-JOIN only checks existence of the key
if (_broadcastRightTable.containsKey(key)) {
- rows.add(joinRow(leftRow, null));
+ rows.add(leftRow);
}
}
@@ -392,7 +386,7 @@ public class HashJoinOperator extends MultiStageOperator {
Object key = _leftKeySelector.getKey(leftRow);
// ANTI-JOIN only checks non-existence of the key
if (!_broadcastRightTable.containsKey(key)) {
- rows.add(joinRow(leftRow, null));
+ rows.add(leftRow);
}
}
@@ -421,18 +415,11 @@ public class HashJoinOperator extends MultiStageOperator {
private Object[] joinRow(@Nullable Object[] leftRow, @Nullable Object[]
rightRow) {
Object[] resultRow = new Object[_resultColumnSize];
- int idx = 0;
if (leftRow != null) {
- for (Object obj : leftRow) {
- resultRow[idx++] = obj;
- }
+ System.arraycopy(leftRow, 0, resultRow, 0, leftRow.length);
}
- // This is needed since left row can be null and we need to advance the
idx to the beginning of right row.
- idx = _leftColumnSize;
if (rightRow != null) {
- for (Object obj : rightRow) {
- resultRow[idx++] = obj;
- }
+ System.arraycopy(rightRow, 0, resultRow, _leftColumnSize,
rightRow.length);
}
return resultRow;
}
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/LeafStageTransferableBlockOperator.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/LeafStageTransferableBlockOperator.java
index 3baab8f536..c185d63e56 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/LeafStageTransferableBlockOperator.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/LeafStageTransferableBlockOperator.java
@@ -120,6 +120,18 @@ public class LeafStageTransferableBlockOperator extends
MultiStageOperator {
_statMap.merge(StatKey.TABLE, tableName);
}
+ public List<ServerQueryRequest> getRequests() {
+ return _requests;
+ }
+
+ public DataSchema getDataSchema() {
+ return _dataSchema;
+ }
+
+ public MultiStageQueryStats getQueryStats() {
+ return MultiStageQueryStats.createLeaf(_context.getStageId(), _statMap);
+ }
+
@Override
public void registerExecution(long time, int numRows) {
_statMap.merge(StatKey.EXECUTION_TIME_MS, time);
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/LookupJoinOperator.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/LookupJoinOperator.java
new file mode 100644
index 0000000000..1f43543018
--- /dev/null
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/LookupJoinOperator.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.query.runtime.operator;
+
+import com.google.common.base.Preconditions;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import javax.annotation.Nullable;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.pinot.common.datablock.DataBlock;
+import org.apache.pinot.common.datatable.StatMap;
+import org.apache.pinot.common.utils.DataSchema;
+import org.apache.pinot.core.data.manager.offline.DimensionTableDataManager;
+import org.apache.pinot.core.query.request.ServerQueryRequest;
+import org.apache.pinot.core.query.request.context.QueryContext;
+import org.apache.pinot.query.planner.logical.RexExpression;
+import org.apache.pinot.query.planner.plannode.JoinNode;
+import org.apache.pinot.query.runtime.blocks.TransferableBlock;
+import org.apache.pinot.query.runtime.operator.operands.TransformOperand;
+import
org.apache.pinot.query.runtime.operator.operands.TransformOperandFactory;
+import org.apache.pinot.query.runtime.plan.MultiStageQueryStats;
+import org.apache.pinot.query.runtime.plan.OpChainExecutionContext;
+import org.apache.pinot.spi.data.readers.PrimaryKey;
+import org.apache.pinot.spi.utils.BooleanUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * This {@code LookupJoinOperator} implements the lookup join algorithm.
+ * <p>This algorithm assumes that the right table is a dimension table which
is preloaded. For each of the left table
+ * row, it looks up for the corresponding row from the dimension table and
create a joint row.
+ * <p>For each of the data block received from the left table, it generates a
joint data block. The output is in the
+ * format of [left_row, right_row].
+ * <p>Since right table is a dimension table which is replicated across all
servers, RIGHT and FULL join are not
+ * supported to avoid duplication.
+ */
+public class LookupJoinOperator extends MultiStageOperator {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(LookupJoinOperator.class);
+ private static final String EXPLAIN_NAME = "LOOKUP_JOIN";
+ private static final Set<JoinRelType> SUPPORTED_JOIN_TYPES =
+ Set.of(JoinRelType.INNER, JoinRelType.LEFT, JoinRelType.SEMI,
JoinRelType.ANTI);
+
+ private final MultiStageOperator _leftInput;
+ private final LeafStageTransferableBlockOperator _rightInput;
+ private final JoinRelType _joinType;
+ private final int[] _leftKeyIds;
+ private final DimensionTableDataManager _rightTable;
+ private final String[] _rightColumns;
+ private final DataSchema _resultSchema;
+ private final int _resultColumnSize;
+ private final List<TransformOperand> _nonEquiEvaluators;
+ private final StatMap<StatKey> _statMap = new StatMap<>(StatKey.class);
+
+ public LookupJoinOperator(OpChainExecutionContext context,
MultiStageOperator leftInput,
+ MultiStageOperator rightInput, JoinNode node) {
+ super(context);
+ _leftInput = leftInput;
+ Preconditions.checkState(rightInput instanceof
LeafStageTransferableBlockOperator,
+ "Right input must be leaf stage operator");
+ _rightInput = (LeafStageTransferableBlockOperator) rightInput;
+ _joinType = node.getJoinType();
+ Preconditions.checkState(SUPPORTED_JOIN_TYPES.contains(_joinType), "Join
type: % is not supported for lookup join",
+ _joinType);
+
+ List<Integer> leftKeys = node.getLeftKeys();
+ _leftKeyIds = new int[leftKeys.size()];
+ for (int i = 0; i < leftKeys.size(); i++) {
+ _leftKeyIds[i] = leftKeys.get(i);
+ }
+ List<ServerQueryRequest> leafStageRequests = _rightInput.getRequests();
+ Preconditions.checkState(leafStageRequests.size() == 1, "Lookup join
cannot be applied to hybrid tables");
+ QueryContext queryContext = leafStageRequests.get(0).getQueryContext();
+ String rightTableName = queryContext.getTableName();
+ _rightTable =
DimensionTableDataManager.getInstanceByTableName(rightTableName);
+ Preconditions.checkState(_rightTable != null, "Failed to find dimension
table for name: %s", rightTableName);
+ _rightColumns = _rightInput.getDataSchema().getColumnNames();
+ _resultSchema = node.getDataSchema();
+ _resultColumnSize = _resultSchema.size();
+ List<RexExpression> nonEquiConditions = node.getNonEquiConditions();
+ _nonEquiEvaluators = new ArrayList<>(nonEquiConditions.size());
+ for (RexExpression nonEquiCondition : nonEquiConditions) {
+
_nonEquiEvaluators.add(TransformOperandFactory.getTransformOperand(nonEquiCondition,
_resultSchema));
+ }
+ }
+
+ @Override
+ public void registerExecution(long time, int numRows) {
+ _statMap.merge(LookupJoinOperator.StatKey.EXECUTION_TIME_MS, time);
+ _statMap.merge(LookupJoinOperator.StatKey.EMITTED_ROWS, numRows);
+ }
+
+ @Override
+ public Type getOperatorType() {
+ return Type.LOOKUP_JOIN;
+ }
+
+ @Override
+ protected Logger logger() {
+ return LOGGER;
+ }
+
+ @Override
+ public List<MultiStageOperator> getChildOperators() {
+ return List.of(_leftInput, _rightInput);
+ }
+
+ @Override
+ public String toExplainString() {
+ return EXPLAIN_NAME;
+ }
+
+ @Override
+ protected TransferableBlock getNextBlock() {
+ // Keep reading the input blocks until we find a match row or all blocks
are processed.
+ // TODO: Consider batching the rows to improve performance.
+ while (true) {
+ TransferableBlock leftBlock = _leftInput.nextBlock();
+ if (leftBlock.isErrorBlock()) {
+ return leftBlock;
+ }
+ if (leftBlock.isSuccessfulEndOfStreamBlock()) {
+ MultiStageQueryStats leftStats = leftBlock.getQueryStats();
+ assert leftStats != null;
+ leftStats.mergeInOrder(_rightInput.getQueryStats(), getOperatorType(),
_statMap);
+ return leftBlock;
+ }
+ assert leftBlock.isDataBlock();
+ List<Object[]> rows = buildJoinedRows(leftBlock);
+ sampleAndCheckInterruption();
+ if (!rows.isEmpty()) {
+ return new TransferableBlock(rows, _resultSchema, DataBlock.Type.ROW);
+ }
+ }
+ }
+
+ private List<Object[]> buildJoinedRows(TransferableBlock leftBlock) {
+ switch (_joinType) {
+ case SEMI:
+ return buildJoinedDataBlockSemi(leftBlock);
+ case ANTI:
+ return buildJoinedDataBlockAnti(leftBlock);
+ default: { // INNER, LEFT, RIGHT, FULL
+ return buildJoinedDataBlockDefault(leftBlock);
+ }
+ }
+ }
+
+ private List<Object[]> buildJoinedDataBlockDefault(TransferableBlock
leftBlock) {
+ List<Object[]> container = leftBlock.getContainer();
+ ArrayList<Object[]> rows = new ArrayList<>(container.size());
+
+ for (Object[] leftRow : container) {
+ PrimaryKey key = getKey(leftRow);
+ Object[] rightRow = _rightTable.lookupValues(key, _rightColumns);
+ if (rightRow != null) {
+ // TODO: Optimize this to avoid unnecessary object copy.
+ Object[] resultRow = joinRow(leftRow, rightRow);
+ if (_nonEquiEvaluators.isEmpty() || _nonEquiEvaluators.stream()
+ .allMatch(evaluator ->
BooleanUtils.isTrueInternalValue(evaluator.apply(resultRow)))) {
+ rows.add(resultRow);
+ continue;
+ }
+ }
+ if (needUnmatchedLeftRows()) {
+ rows.add(joinRow(leftRow, null));
+ }
+ }
+
+ return rows;
+ }
+
+ private List<Object[]> buildJoinedDataBlockSemi(TransferableBlock leftBlock)
{
+ List<Object[]> container = leftBlock.getContainer();
+ List<Object[]> rows = new ArrayList<>(container.size());
+ for (Object[] leftRow : container) {
+ if (_rightTable.containsKey(getKey(leftRow))) {
+ rows.add(leftRow);
+ }
+ }
+ return rows;
+ }
+
+ private List<Object[]> buildJoinedDataBlockAnti(TransferableBlock leftBlock)
{
+ List<Object[]> container = leftBlock.getContainer();
+ List<Object[]> rows = new ArrayList<>(container.size());
+ for (Object[] leftRow : container) {
+ if (!_rightTable.containsKey(getKey(leftRow))) {
+ rows.add(leftRow);
+ }
+ }
+ return rows;
+ }
+
+ private PrimaryKey getKey(Object[] row) {
+ Object[] values = new Object[_leftKeyIds.length];
+ for (int i = 0; i < _leftKeyIds.length; i++) {
+ values[i] = row[_leftKeyIds[i]];
+ }
+ return new PrimaryKey(values);
+ }
+
+ private Object[] joinRow(Object[] leftRow, @Nullable Object[] rightRow) {
+ Object[] resultRow = new Object[_resultColumnSize];
+ System.arraycopy(leftRow, 0, resultRow, 0, leftRow.length);
+ if (rightRow != null) {
+ System.arraycopy(rightRow, 0, resultRow, leftRow.length,
rightRow.length);
+ }
+ return resultRow;
+ }
+
+ private boolean needUnmatchedLeftRows() {
+ return _joinType == JoinRelType.LEFT;
+ }
+
+ public enum StatKey implements StatMap.Key {
+ //@formatter:off
+ EXECUTION_TIME_MS(StatMap.Type.LONG) {
+ @Override
+ public boolean includeDefaultInJson() {
+ return true;
+ }
+ },
+ EMITTED_ROWS(StatMap.Type.LONG) {
+ @Override
+ public boolean includeDefaultInJson() {
+ return true;
+ }
+ };
+ //@formatter:on
+
+ private final StatMap.Type _type;
+
+ StatKey(StatMap.Type type) {
+ _type = type;
+ }
+
+ @Override
+ public StatMap.Type getType() {
+ return _type;
+ }
+ }
+}
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MultiStageOperator.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MultiStageOperator.java
index bfb02a7006..300f474fcc 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MultiStageOperator.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/operator/MultiStageOperator.java
@@ -207,6 +207,7 @@ public abstract class MultiStageOperator
* <p>
* This is mostly used in the context of stats collection, where we use this
enum in the serialization form in order
* to identify the type of the stats in an efficient way.
+ * DO NOT change the order of the enum values, as the ordinal is used in
serialization.
*/
public enum Type {
AGGREGATE(AggregateOperator.StatKey.class) {
@@ -391,7 +392,15 @@ public abstract class MultiStageOperator
serverMetrics.addMeteredGlobalValue(ServerMeter.WINDOW_TIMES_MAX_ROWS_REACHED,
1);
}
}
- },;
+ },
+ LOOKUP_JOIN(LookupJoinOperator.StatKey.class) {
+ @Override
+ public void mergeInto(BrokerResponseNativeV2 response, StatMap<?> map) {
+ @SuppressWarnings("unchecked")
+ StatMap<LookupJoinOperator.StatKey> stats =
(StatMap<LookupJoinOperator.StatKey>) map;
+
response.mergeMaxRowsInOperator(stats.getLong(LookupJoinOperator.StatKey.EMITTED_ROWS));
+ }
+ };
private final Class _statKeyClass;
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/PlanNodeToOpChain.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/PlanNodeToOpChain.java
index 742419d696..9570d77b47 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/PlanNodeToOpChain.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/PlanNodeToOpChain.java
@@ -43,6 +43,7 @@ import
org.apache.pinot.query.runtime.operator.IntersectAllOperator;
import org.apache.pinot.query.runtime.operator.IntersectOperator;
import
org.apache.pinot.query.runtime.operator.LeafStageTransferableBlockOperator;
import org.apache.pinot.query.runtime.operator.LiteralValueOperator;
+import org.apache.pinot.query.runtime.operator.LookupJoinOperator;
import org.apache.pinot.query.runtime.operator.MailboxReceiveOperator;
import org.apache.pinot.query.runtime.operator.MailboxSendOperator;
import org.apache.pinot.query.runtime.operator.MinusAllOperator;
@@ -174,8 +175,16 @@ public class PlanNodeToOpChain {
public MultiStageOperator visitJoin(JoinNode node, OpChainExecutionContext
context) {
List<PlanNode> inputs = node.getInputs();
PlanNode left = inputs.get(0);
+ MultiStageOperator leftOperator = visit(left, context);
PlanNode right = inputs.get(1);
- return new HashJoinOperator(context, visit(left, context),
left.getDataSchema(), visit(right, context), node);
+ MultiStageOperator rightOperator = visit(right, context);
+ JoinNode.JoinStrategy joinStrategy = node.getJoinStrategy();
+ if (joinStrategy == JoinNode.JoinStrategy.HASH) {
+ return new HashJoinOperator(context, leftOperator,
left.getDataSchema(), rightOperator, node);
+ } else {
+ assert joinStrategy == JoinNode.JoinStrategy.LOOKUP;
+ return new LookupJoinOperator(context, leftOperator, rightOperator,
node);
+ }
}
@Override
diff --git
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/server/ServerPlanRequestVisitor.java
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/server/ServerPlanRequestVisitor.java
index 87dbd27a71..9de79d15e2 100644
---
a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/server/ServerPlanRequestVisitor.java
+++
b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/plan/server/ServerPlanRequestVisitor.java
@@ -18,9 +18,11 @@
*/
package org.apache.pinot.query.runtime.plan.server;
+import com.google.common.base.Preconditions;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.apache.pinot.calcite.rel.logical.PinotRelExchangeType;
import org.apache.pinot.common.datablock.DataBlock;
import org.apache.pinot.common.request.DataSource;
import org.apache.pinot.common.request.Expression;
@@ -125,28 +127,39 @@ public class ServerPlanRequestVisitor implements
PlanNodeVisitor<Void, ServerPla
@Override
public Void visitJoin(JoinNode node, ServerPlanRequestContext context) {
- // visit only the static side, turn the dynamic side into a lookup from
the pipeline breaker resultDataContainer
- PlanNode staticSide = node.getInputs().get(0);
- PlanNode dynamicSide = node.getInputs().get(1);
- if (staticSide instanceof MailboxReceiveNode) {
- dynamicSide = node.getInputs().get(0);
- staticSide = node.getInputs().get(1);
- }
- if (visit(staticSide, context)) {
- PipelineBreakerResult pipelineBreakerResult =
context.getPipelineBreakerResult();
- int resultMapId = pipelineBreakerResult.getNodeIdMap().get(dynamicSide);
- List<TransferableBlock> transferableBlocks =
- pipelineBreakerResult.getResultMap().getOrDefault(resultMapId,
Collections.emptyList());
- List<Object[]> resultDataContainer = new ArrayList<>();
- DataSchema dataSchema = dynamicSide.getDataSchema();
- for (TransferableBlock block : transferableBlocks) {
- if (block.getType() == DataBlock.Type.ROW) {
- resultDataContainer.addAll(block.getContainer());
+ // We can reach here for dynamic broadcast SEMI join and lookup join.
+ List<PlanNode> inputs = node.getInputs();
+ PlanNode left = inputs.get(0);
+ PlanNode right = inputs.get(1);
+
+ if (right instanceof MailboxReceiveNode
+ && ((MailboxReceiveNode) right).getExchangeType() ==
PinotRelExchangeType.PIPELINE_BREAKER) {
+ // For dynamic broadcast SEMI join, right child should be a
PIPELINE_BREAKER exchange. Visit the left child and
+ // attach the dynamic filter to the query.
+ if (visit(left, context)) {
+ PipelineBreakerResult pipelineBreakerResult =
context.getPipelineBreakerResult();
+ int resultMapId = pipelineBreakerResult.getNodeIdMap().get(right);
+ List<TransferableBlock> transferableBlocks =
+ pipelineBreakerResult.getResultMap().getOrDefault(resultMapId,
Collections.emptyList());
+ List<Object[]> resultDataContainer = new ArrayList<>();
+ DataSchema dataSchema = right.getDataSchema();
+ for (TransferableBlock block : transferableBlocks) {
+ if (block.getType() == DataBlock.Type.ROW) {
+ resultDataContainer.addAll(block.getContainer());
+ }
}
+ ServerPlanRequestUtils.attachDynamicFilter(context.getPinotQuery(),
node.getLeftKeys(), node.getRightKeys(),
+ resultDataContainer, dataSchema);
+ }
+ } else {
+ // For lookup join, visit the right child and set it as the leaf
boundary.
+ Preconditions.checkState(node.getJoinStrategy() ==
JoinNode.JoinStrategy.LOOKUP,
+ "Leaf stage should not visit regular JoinNode");
+ if (visit(right, context)) {
+ context.setLeafStageBoundaryNode(right);
}
- ServerPlanRequestUtils.attachDynamicFilter(context.getPinotQuery(),
node.getLeftKeys(), node.getRightKeys(),
- resultDataContainer, dataSchema);
}
+
return null;
}
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/HashJoinOperatorTest.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/HashJoinOperatorTest.java
index 983b2bf937..5b29817b7b 100644
---
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/HashJoinOperatorTest.java
+++
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/HashJoinOperatorTest.java
@@ -349,15 +349,15 @@ public class HashJoinOperatorTest {
when(_rightInput.nextBlock()).thenReturn(
OperatorTestUtil.block(rightSchema, new Object[]{2, "Aa"}, new
Object[]{2, "BB"}, new Object[]{3, "BB"}))
.thenReturn(TransferableBlockTestUtils.getEndOfStreamTransferableBlock(0));
- DataSchema resultSchema = new DataSchema(new String[]{"foo", "bar", "foo",
"bar"}, new ColumnDataType[]{
- ColumnDataType.INT, ColumnDataType.STRING, ColumnDataType.INT,
ColumnDataType.STRING
+ DataSchema resultSchema = new DataSchema(new String[]{"foo", "bar"}, new
ColumnDataType[]{
+ ColumnDataType.INT, ColumnDataType.STRING
});
HashJoinOperator operator =
getOperator(leftSchema, resultSchema, JoinRelType.SEMI, List.of(1),
List.of(1), List.of());
List<Object[]> resultRows = operator.nextBlock().getContainer();
assertEquals(resultRows.size(), 2);
- assertEquals(resultRows.get(0), new Object[]{1, "Aa", null, null});
- assertEquals(resultRows.get(1), new Object[]{2, "BB", null, null});
+ assertEquals(resultRows.get(0), new Object[]{1, "Aa"});
+ assertEquals(resultRows.get(1), new Object[]{2, "BB"});
assertTrue(operator.nextBlock().isSuccessfulEndOfStreamBlock());
}
@@ -408,14 +408,14 @@ public class HashJoinOperatorTest {
when(_rightInput.nextBlock()).thenReturn(
OperatorTestUtil.block(rightSchema, new Object[]{2, "Aa"}, new
Object[]{2, "BB"}, new Object[]{3, "BB"}))
.thenReturn(TransferableBlockTestUtils.getEndOfStreamTransferableBlock(0));
- DataSchema resultSchema = new DataSchema(new String[]{"foo", "bar", "foo",
"bar"}, new ColumnDataType[]{
- ColumnDataType.INT, ColumnDataType.STRING, ColumnDataType.INT,
ColumnDataType.STRING
+ DataSchema resultSchema = new DataSchema(new String[]{"foo", "bar"}, new
ColumnDataType[]{
+ ColumnDataType.INT, ColumnDataType.STRING
});
HashJoinOperator operator =
getOperator(leftSchema, resultSchema, JoinRelType.ANTI, List.of(1),
List.of(1), List.of());
List<Object[]> resultRows = operator.nextBlock().getContainer();
assertEquals(resultRows.size(), 1);
- assertEquals(resultRows.get(0), new Object[]{4, "CC", null, null});
+ assertEquals(resultRows.get(0), new Object[]{4, "CC"});
assertTrue(operator.nextBlock().isSuccessfulEndOfStreamBlock());
}
@@ -574,8 +574,7 @@ public class HashJoinOperatorTest {
OperatorTestUtil.block(leftSchema, new Object[]{1, "Aa"}, new
Object[]{2, "Aa"}, new Object[]{3, "Aa"}))
.thenReturn(OperatorTestUtil.block(leftSchema, new Object[]{4, "Aa"},
new Object[]{5, "Aa"}))
.thenReturn(TransferableBlockTestUtils.getEndOfStreamTransferableBlock(0));
- when(_rightInput.nextBlock()).thenReturn(
- OperatorTestUtil.block(rightSchema, new Object[]{2, "Aa"}))
+
when(_rightInput.nextBlock()).thenReturn(OperatorTestUtil.block(rightSchema,
new Object[]{2, "Aa"}))
.thenReturn(TransferableBlockTestUtils.getEndOfStreamTransferableBlock(0));
DataSchema resultSchema =
new DataSchema(new String[]{"int_col1", "string_col1", "int_co2",
"string_col2"}, new ColumnDataType[]{
@@ -600,7 +599,8 @@ public class HashJoinOperatorTest {
List<Integer> leftKeys, List<Integer> rightKeys, List<RexExpression>
nonEquiConditions,
PlanNode.NodeHint nodeHint) {
return new HashJoinOperator(OperatorTestUtil.getTracingContext(),
_leftInput, leftSchema, _rightInput,
- new JoinNode(-1, resultSchema, nodeHint, List.of(), joinType,
leftKeys, rightKeys, nonEquiConditions));
+ new JoinNode(-1, resultSchema, nodeHint, List.of(), joinType,
leftKeys, rightKeys, nonEquiConditions,
+ JoinNode.JoinStrategy.HASH));
}
private HashJoinOperator getOperator(DataSchema leftSchema, DataSchema
resultSchema, JoinRelType joinType,
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/MultiStageAccountingTest.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/MultiStageAccountingTest.java
index 2821de5e73..5b38dcdab5 100644
---
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/MultiStageAccountingTest.java
+++
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/MultiStageAccountingTest.java
@@ -178,7 +178,7 @@ public class MultiStageAccountingTest implements ITest {
});
return new HashJoinOperator(OperatorTestUtil.getTracingContext(),
leftInput, leftSchema, rightInput,
new JoinNode(-1, resultSchema, PlanNode.NodeHint.EMPTY, List.of(),
JoinRelType.INNER, List.of(0), List.of(0),
- List.of()));
+ List.of(), JoinNode.JoinStrategy.HASH));
}
private static MultiStageOperator getSortOperator() {
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/plan/pipeline/PipelineBreakerExecutorTest.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/plan/pipeline/PipelineBreakerExecutorTest.java
index 8c71bd00a8..26635cb8f8 100644
---
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/plan/pipeline/PipelineBreakerExecutorTest.java
+++
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/plan/pipeline/PipelineBreakerExecutorTest.java
@@ -142,7 +142,7 @@ public class PipelineBreakerExecutorTest {
MailboxReceiveNode mailboxReceiveNode2 = getPBReceiveNode(2);
JoinNode joinNode =
new JoinNode(0, DATA_SCHEMA, PlanNode.NodeHint.EMPTY,
List.of(mailboxReceiveNode1, mailboxReceiveNode2),
- JoinRelType.INNER, List.of(0), List.of(0), List.of());
+ JoinRelType.INNER, List.of(0), List.of(0), List.of(),
JoinNode.JoinStrategy.HASH);
StagePlan stagePlan = new StagePlan(joinNode, _stageMetadata);
// when
@@ -231,7 +231,7 @@ public class PipelineBreakerExecutorTest {
MailboxReceiveNode incorrectlyConfiguredMailboxNode = getPBReceiveNode(3);
JoinNode joinNode = new JoinNode(0, DATA_SCHEMA, PlanNode.NodeHint.EMPTY,
List.of(mailboxReceiveNode1, incorrectlyConfiguredMailboxNode),
JoinRelType.INNER, List.of(0), List.of(0),
- List.of());
+ List.of(), JoinNode.JoinStrategy.HASH);
StagePlan stagePlan = new StagePlan(joinNode, _stageMetadata);
// when
@@ -264,7 +264,7 @@ public class PipelineBreakerExecutorTest {
MailboxReceiveNode incorrectlyConfiguredMailboxNode = getPBReceiveNode(2);
JoinNode joinNode = new JoinNode(0, DATA_SCHEMA, PlanNode.NodeHint.EMPTY,
List.of(mailboxReceiveNode1, incorrectlyConfiguredMailboxNode),
JoinRelType.INNER, List.of(0), List.of(0),
- List.of());
+ List.of(), JoinNode.JoinStrategy.HASH);
StagePlan stagePlan = new StagePlan(joinNode, _stageMetadata);
// when
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/LookupJoinEngineQuickStart.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/LookupJoinEngineQuickStart.java
new file mode 100644
index 0000000000..a6fb1a417d
--- /dev/null
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/LookupJoinEngineQuickStart.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.tools;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.tools.admin.PinotAdministrator;
+
+
+public class LookupJoinEngineQuickStart extends MultistageEngineQuickStart {
+ private static final String QUICKSTART_IDENTIFIER = "LOOKUP_JOIN";
+ // Reuse userAttributes from ColocatedJoinEngineQuickStart
+ private static final String[] LOOKUP_JOIN_DIRECTORIES = new String[]{
+ "examples/batch/colocated/userAttributes",
"examples/batch/lookup/userGroupsDim"
+ };
+
+ @Override
+ public List<String> types() {
+ return Collections.singletonList(QUICKSTART_IDENTIFIER);
+ }
+
+ @Override
+ public Map<String, Object> getConfigOverrides() {
+ Map<String, Object> overrides = new HashMap<>(super.getConfigOverrides());
+
overrides.put(CommonConstants.Broker.CONFIG_OF_ENABLE_PARTITION_METADATA_MANAGER,
"true");
+ return overrides;
+ }
+
+ @Override
+ public String[] getDefaultBatchTableDirectories() {
+ return LOOKUP_JOIN_DIRECTORIES;
+ }
+
+ @Override
+ protected int getNumQuickstartRunnerServers() {
+ return 4;
+ }
+
+ public static void main(String[] args)
+ throws Exception {
+ List<String> arguments = new ArrayList<>();
+ arguments.addAll(Arrays.asList("QuickStart", "-type",
QUICKSTART_IDENTIFIER));
+ arguments.addAll(Arrays.asList(args));
+ PinotAdministrator.main(arguments.toArray(new String[0]));
+ }
+}
diff --git
a/pinot-tools/src/main/resources/examples/batch/colocated/userGroups/userGroups_schema.json
b/pinot-tools/src/main/resources/examples/batch/colocated/userGroups/userGroups_schema.json
index c4772df7a2..7dad5360b2 100644
---
a/pinot-tools/src/main/resources/examples/batch/colocated/userGroups/userGroups_schema.json
+++
b/pinot-tools/src/main/resources/examples/batch/colocated/userGroups/userGroups_schema.json
@@ -1,6 +1,5 @@
{
- "metricFieldSpecs": [
- ],
+ "schemaName": "userGroups",
"dimensionFieldSpecs": [
{
"dataType": "STRING",
@@ -10,6 +9,5 @@
"dataType": "STRING",
"name": "userUUID"
}
- ],
- "schemaName": "userGroups"
+ ]
}
diff --git
a/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/ingestionJobSpec.yaml
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/ingestionJobSpec.yaml
new file mode 100644
index 0000000000..93712178be
--- /dev/null
+++
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/ingestionJobSpec.yaml
@@ -0,0 +1,140 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# executionFrameworkSpec: Defines ingestion jobs to be running.
+executionFrameworkSpec:
+
+ # name: execution framework name
+ name: 'standalone'
+
+ # Class to use for segment generation and different push types.
+ segmentGenerationJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentGenerationJobRunner'
+ segmentTarPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentTarPushJobRunner'
+ segmentUriPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentUriPushJobRunner'
+ segmentMetadataPushJobRunnerClassName:
'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'
+
+
+# jobType: Pinot ingestion job type.
+# Supported job types are defined in PinotIngestionJobType class.
+# 'SegmentCreation'
+# 'SegmentTarPush'
+# 'SegmentUriPush'
+# 'SegmentMetadataPush'
+# 'SegmentCreationAndTarPush'
+# 'SegmentCreationAndUriPush'
+# 'SegmentCreationAndMetadataPush'
+jobType: SegmentCreationAndTarPush
+
+# inputDirURI: Root directory of input data, expected to have scheme
configured in PinotFS.
+inputDirURI: 'examples/batch/lookup/userGroupsDim/rawdata'
+
+# includeFileNamePattern: include file name pattern, supported glob pattern.
+# Sample usage:
+# 'glob:*.avro' will include all avro files just under the inputDirURI, not
sub directories;
+# 'glob:**/*.avro' will include all the avro files under inputDirURI
recursively.
+includeFileNamePattern: 'glob:**/*.csv'
+
+# excludeFileNamePattern: exclude file name pattern, supported glob pattern.
+# Sample usage:
+# 'glob:*.avro' will exclude all avro files just under the inputDirURI, not
sub directories;
+# 'glob:**/*.avro' will exclude all the avro files under inputDirURI
recursively.
+# _excludeFileNamePattern: ''
+
+# outputDirURI: Root directory of output segments, expected to have scheme
configured in PinotFS.
+outputDirURI: 'examples/batch/lookup/userGroupsDim/segments'
+
+# overwriteOutput: Overwrite output segments if existed.
+overwriteOutput: true
+
+# pinotFSSpecs: defines all related Pinot file systems.
+pinotFSSpecs:
+
+ - # scheme: used to identify a PinotFS.
+ # E.g. local, hdfs, dbfs, etc
+ scheme: file
+
+ # className: Class name used to create the PinotFS instance.
+ # E.g.
+ # org.apache.pinot.spi.filesystem.LocalPinotFS is used for local
filesystem
+ # org.apache.pinot.plugin.filesystem.AzurePinotFS is used for Azure Data
Lake
+ # org.apache.pinot.plugin.filesystem.HadoopPinotFS is used for HDFS
+ className: org.apache.pinot.spi.filesystem.LocalPinotFS
+
+# recordReaderSpec: defines all record reader
+recordReaderSpec:
+
+ # dataFormat: Record data format, e.g. 'avro', 'parquet', 'orc', 'csv',
'json', 'thrift' etc.
+ dataFormat: 'csv'
+
+ # className: Corresponding RecordReader class name.
+ # E.g.
+ # org.apache.pinot.plugin.inputformat.avro.AvroRecordReader
+ # org.apache.pinot.plugin.inputformat.csv.CSVRecordReader
+ # org.apache.pinot.plugin.inputformat.parquet.ParquetRecordReader
+ # org.apache.pinot.plugin.inputformat.parquet.ParquetNativeRecordReader
+ # org.apache.pinot.plugin.inputformat.json.JSONRecordReader
+ # org.apache.pinot.plugin.inputformat.orc.ORCRecordReader
+ # org.apache.pinot.plugin.inputformat.thrift.ThriftRecordReader
+ className: 'org.apache.pinot.plugin.inputformat.csv.CSVRecordReader'
+
+ # configClassName: Corresponding RecordReaderConfig class name, it's
mandatory for CSV and Thrift file format.
+ # E.g.
+ # org.apache.pinot.plugin.inputformat.csv.CSVRecordReaderConfig
+ # org.apache.pinot.plugin.inputformat.thrift.ThriftRecordReaderConfig
+ configClassName:
'org.apache.pinot.plugin.inputformat.csv.CSVRecordReaderConfig'
+
+ # configs: Used to init RecordReaderConfig class name, this config is
required for CSV and Thrift data format.
+ configs:
+
+
+# tableSpec: defines table name and where to fetch corresponding table config
and table schema.
+tableSpec:
+
+ # tableName: Table name
+ tableName: 'userGroupsDim'
+
+ # schemaURI: defines where to read the table schema, supports PinotFS or
HTTP.
+ # E.g.
+ # hdfs://path/to/table_schema.json
+ # http://localhost:9000/tables/myTable/schema
+ schemaURI: 'http://localhost:9000/tables/userGroupsDim/schema'
+
+ # tableConfigURI: defines where to reade the table config.
+ # Supports using PinotFS or HTTP.
+ # E.g.
+ # hdfs://path/to/table_config.json
+ # http://localhost:9000/tables/myTable
+ # Note that the API to read Pinot table config directly from pinot
controller contains a JSON wrapper.
+ # The real table config is the object under the field 'OFFLINE'.
+ tableConfigURI: 'http://localhost:9000/tables/userGroupsDim'
+
+# pinotClusterSpecs: defines the Pinot Cluster Access Point.
+pinotClusterSpecs:
+ - # controllerURI: used to fetch table/schema information and data push.
+ # E.g. http://localhost:9000
+ controllerURI: 'http://localhost:9000'
+
+# pushJobSpec: defines segment push job related configuration.
+pushJobSpec:
+
+ # pushAttempts: number of attempts for push job, default is 1, which means
no retry.
+ pushAttempts: 2
+
+ # pushRetryIntervalMillis: retry wait Ms, default to 1 second.
+ pushRetryIntervalMillis: 1000
diff --git
a/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p0.csv
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p0.csv
new file mode 100644
index 0000000000..7708a2c3a6
--- /dev/null
+++
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p0.csv
@@ -0,0 +1,8 @@
+userUUID,groupUUID
+user-1,group-0
+user-1,group-1
+user-2,group-0
+user-2,group-1
+user-7,group-1
+user-7,group-2
+user-15,group-1
diff --git
a/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p1.csv
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p1.csv
new file mode 100644
index 0000000000..675175d70c
--- /dev/null
+++
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p1.csv
@@ -0,0 +1,7 @@
+userUUID,groupUUID
+user-6,group-0
+user-6,group-1
+user-6,group-2
+user-8,group-0
+user-14,group-0
+user-24,group-0
diff --git
a/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p2.csv
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p2.csv
new file mode 100644
index 0000000000..4afa948eaf
--- /dev/null
+++
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p2.csv
@@ -0,0 +1,9 @@
+userUUID,groupUUID
+user-5,group-0
+user-10,group-0
+user-11,group-0
+user-12,group-0
+user-16,group-0
+user-18,group-0
+user-19,group-0
+user-25,group-0
diff --git
a/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p3.csv
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p3.csv
new file mode 100644
index 0000000000..3a9e76c651
--- /dev/null
+++
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p3.csv
@@ -0,0 +1,2455 @@
+userUUID,groupUUID
+user-0,group-0
+user-3,group-0
+user-4,group-0
+user-9,group-0
+user-13,group-0
+user-20,group-0
+user-22,group-0
+user-29,group-0
+user-30,group-0
+user-36,group-0
+user-37,group-0
+user-40,group-0
+user-51,group-0
+user-55,group-0
+user-63,group-0
+user-64,group-0
+user-76,group-0
+user-77,group-0
+user-81,group-0
+user-82,group-0
+user-84,group-0
+user-86,group-0
+user-102,group-1
+user-104,group-1
+user-109,group-1
+user-111,group-1
+user-112,group-1
+user-114,group-1
+user-116,group-1
+user-122,group-1
+user-133,group-1
+user-136,group-1
+user-141,group-1
+user-142,group-1
+user-147,group-1
+user-152,group-1
+user-154,group-1
+user-155,group-1
+user-156,group-1
+user-158,group-1
+user-167,group-1
+user-168,group-1
+user-171,group-1
+user-176,group-1
+user-180,group-1
+user-187,group-1
+user-191,group-1
+user-194,group-1
+user-195,group-1
+user-204,group-2
+user-205,group-2
+user-207,group-2
+user-213,group-2
+user-220,group-2
+user-221,group-2
+user-223,group-2
+user-230,group-2
+user-238,group-2
+user-239,group-2
+user-246,group-2
+user-247,group-2
+user-252,group-2
+user-253,group-2
+user-256,group-2
+user-259,group-2
+user-262,group-2
+user-264,group-2
+user-268,group-2
+user-269,group-2
+user-277,group-2
+user-287,group-2
+user-293,group-2
+user-296,group-2
+user-299,group-2
+user-302,group-3
+user-304,group-3
+user-309,group-3
+user-310,group-3
+user-320,group-3
+user-321,group-3
+user-322,group-3
+user-326,group-3
+user-332,group-3
+user-334,group-3
+user-337,group-3
+user-338,group-3
+user-352,group-3
+user-357,group-3
+user-359,group-3
+user-361,group-3
+user-364,group-3
+user-367,group-3
+user-369,group-3
+user-370,group-3
+user-371,group-3
+user-376,group-3
+user-379,group-3
+user-383,group-3
+user-387,group-3
+user-390,group-3
+user-391,group-3
+user-392,group-3
+user-394,group-3
+user-399,group-3
+user-417,group-4
+user-420,group-4
+user-423,group-4
+user-424,group-4
+user-428,group-4
+user-430,group-4
+user-432,group-4
+user-434,group-4
+user-440,group-4
+user-443,group-4
+user-444,group-4
+user-448,group-4
+user-449,group-4
+user-450,group-4
+user-457,group-4
+user-458,group-4
+user-471,group-4
+user-472,group-4
+user-489,group-4
+user-504,group-5
+user-505,group-5
+user-515,group-5
+user-520,group-5
+user-523,group-5
+user-524,group-5
+user-531,group-5
+user-533,group-5
+user-535,group-5
+user-537,group-5
+user-542,group-5
+user-543,group-5
+user-557,group-5
+user-574,group-5
+user-576,group-5
+user-582,group-5
+user-589,group-5
+user-591,group-5
+user-592,group-5
+user-593,group-5
+user-594,group-5
+user-596,group-5
+user-597,group-5
+user-600,group-6
+user-602,group-6
+user-604,group-6
+user-612,group-6
+user-615,group-6
+user-621,group-6
+user-622,group-6
+user-623,group-6
+user-629,group-6
+user-638,group-6
+user-640,group-6
+user-652,group-6
+user-655,group-6
+user-656,group-6
+user-659,group-6
+user-661,group-6
+user-668,group-6
+user-669,group-6
+user-686,group-6
+user-690,group-6
+user-695,group-6
+user-698,group-6
+user-702,group-7
+user-703,group-7
+user-704,group-7
+user-706,group-7
+user-710,group-7
+user-716,group-7
+user-718,group-7
+user-725,group-7
+user-726,group-7
+user-728,group-7
+user-730,group-7
+user-731,group-7
+user-737,group-7
+user-738,group-7
+user-747,group-7
+user-748,group-7
+user-753,group-7
+user-755,group-7
+user-761,group-7
+user-766,group-7
+user-769,group-7
+user-773,group-7
+user-774,group-7
+user-775,group-7
+user-776,group-7
+user-777,group-7
+user-778,group-7
+user-782,group-7
+user-784,group-7
+user-803,group-8
+user-804,group-8
+user-805,group-8
+user-812,group-8
+user-818,group-8
+user-825,group-8
+user-826,group-8
+user-827,group-8
+user-829,group-8
+user-830,group-8
+user-831,group-8
+user-842,group-8
+user-844,group-8
+user-845,group-8
+user-849,group-8
+user-855,group-8
+user-863,group-8
+user-866,group-8
+user-871,group-8
+user-877,group-8
+user-880,group-8
+user-882,group-8
+user-884,group-8
+user-885,group-8
+user-897,group-8
+user-909,group-9
+user-916,group-9
+user-919,group-9
+user-921,group-9
+user-922,group-9
+user-923,group-9
+user-927,group-9
+user-928,group-9
+user-931,group-9
+user-932,group-9
+user-937,group-9
+user-939,group-9
+user-947,group-9
+user-949,group-9
+user-951,group-9
+user-954,group-9
+user-963,group-9
+user-965,group-9
+user-972,group-9
+user-974,group-9
+user-982,group-9
+user-984,group-9
+user-991,group-9
+user-992,group-9
+user-993,group-9
+user-1000,group-10
+user-1012,group-10
+user-1013,group-10
+user-1019,group-10
+user-1022,group-10
+user-1025,group-10
+user-1026,group-10
+user-1027,group-10
+user-1032,group-10
+user-1033,group-10
+user-1035,group-10
+user-1036,group-10
+user-1038,group-10
+user-1040,group-10
+user-1048,group-10
+user-1061,group-10
+user-1063,group-10
+user-1066,group-10
+user-1068,group-10
+user-1070,group-10
+user-1071,group-10
+user-1072,group-10
+user-1073,group-10
+user-1078,group-10
+user-1079,group-10
+user-1080,group-10
+user-1081,group-10
+user-1082,group-10
+user-1090,group-10
+user-1094,group-10
+user-1096,group-10
+user-1099,group-10
+user-1101,group-11
+user-1103,group-11
+user-1110,group-11
+user-1113,group-11
+user-1117,group-11
+user-1118,group-11
+user-1119,group-11
+user-1122,group-11
+user-1131,group-11
+user-1133,group-11
+user-1142,group-11
+user-1146,group-11
+user-1147,group-11
+user-1148,group-11
+user-1153,group-11
+user-1154,group-11
+user-1157,group-11
+user-1159,group-11
+user-1160,group-11
+user-1161,group-11
+user-1166,group-11
+user-1170,group-11
+user-1174,group-11
+user-1175,group-11
+user-1182,group-11
+user-1188,group-11
+user-1192,group-11
+user-1196,group-11
+user-1201,group-12
+user-1202,group-12
+user-1204,group-12
+user-1211,group-12
+user-1214,group-12
+user-1219,group-12
+user-1220,group-12
+user-1223,group-12
+user-1224,group-12
+user-1228,group-12
+user-1230,group-12
+user-1233,group-12
+user-1238,group-12
+user-1240,group-12
+user-1242,group-12
+user-1247,group-12
+user-1259,group-12
+user-1261,group-12
+user-1262,group-12
+user-1264,group-12
+user-1268,group-12
+user-1272,group-12
+user-1281,group-12
+user-1284,group-12
+user-1285,group-12
+user-1298,group-12
+user-1299,group-12
+user-1301,group-13
+user-1305,group-13
+user-1306,group-13
+user-1307,group-13
+user-1308,group-13
+user-1313,group-13
+user-1314,group-13
+user-1318,group-13
+user-1322,group-13
+user-1327,group-13
+user-1331,group-13
+user-1337,group-13
+user-1339,group-13
+user-1350,group-13
+user-1354,group-13
+user-1356,group-13
+user-1360,group-13
+user-1363,group-13
+user-1365,group-13
+user-1367,group-13
+user-1371,group-13
+user-1373,group-13
+user-1375,group-13
+user-1381,group-13
+user-1383,group-13
+user-1392,group-13
+user-1393,group-13
+user-1401,group-14
+user-1403,group-14
+user-1406,group-14
+user-1409,group-14
+user-1413,group-14
+user-1414,group-14
+user-1415,group-14
+user-1419,group-14
+user-1421,group-14
+user-1423,group-14
+user-1425,group-14
+user-1426,group-14
+user-1427,group-14
+user-1428,group-14
+user-1430,group-14
+user-1433,group-14
+user-1434,group-14
+user-1447,group-14
+user-1451,group-14
+user-1454,group-14
+user-1458,group-14
+user-1464,group-14
+user-1468,group-14
+user-1469,group-14
+user-1470,group-14
+user-1478,group-14
+user-1485,group-14
+user-1487,group-14
+user-1490,group-14
+user-1491,group-14
+user-1492,group-14
+user-1493,group-14
+user-1494,group-14
+user-1496,group-14
+user-1498,group-14
+user-1503,group-15
+user-1504,group-15
+user-1509,group-15
+user-1516,group-15
+user-1518,group-15
+user-1519,group-15
+user-1522,group-15
+user-1525,group-15
+user-1526,group-15
+user-1529,group-15
+user-1532,group-15
+user-1533,group-15
+user-1536,group-15
+user-1538,group-15
+user-1545,group-15
+user-1548,group-15
+user-1553,group-15
+user-1557,group-15
+user-1560,group-15
+user-1562,group-15
+user-1566,group-15
+user-1568,group-15
+user-1574,group-15
+user-1575,group-15
+user-1578,group-15
+user-1582,group-15
+user-1587,group-15
+user-1590,group-15
+user-1591,group-15
+user-1595,group-15
+user-1596,group-15
+user-1609,group-16
+user-1614,group-16
+user-1617,group-16
+user-1622,group-16
+user-1623,group-16
+user-1628,group-16
+user-1632,group-16
+user-1633,group-16
+user-1636,group-16
+user-1642,group-16
+user-1643,group-16
+user-1649,group-16
+user-1652,group-16
+user-1656,group-16
+user-1663,group-16
+user-1666,group-16
+user-1673,group-16
+user-1675,group-16
+user-1676,group-16
+user-1678,group-16
+user-1684,group-16
+user-1687,group-16
+user-1697,group-16
+user-1698,group-16
+user-1699,group-16
+user-1704,group-17
+user-1707,group-17
+user-1710,group-17
+user-1711,group-17
+user-1718,group-17
+user-1723,group-17
+user-1727,group-17
+user-1729,group-17
+user-1733,group-17
+user-1734,group-17
+user-1759,group-17
+user-1765,group-17
+user-1775,group-17
+user-1779,group-17
+user-1782,group-17
+user-1789,group-17
+user-1795,group-17
+user-1801,group-18
+user-1806,group-18
+user-1819,group-18
+user-1821,group-18
+user-1822,group-18
+user-1823,group-18
+user-1827,group-18
+user-1832,group-18
+user-1836,group-18
+user-1838,group-18
+user-1839,group-18
+user-1850,group-18
+user-1854,group-18
+user-1862,group-18
+user-1866,group-18
+user-1868,group-18
+user-1872,group-18
+user-1874,group-18
+user-1879,group-18
+user-1880,group-18
+user-1882,group-18
+user-1888,group-18
+user-1889,group-18
+user-1896,group-18
+user-1898,group-18
+user-1908,group-19
+user-1920,group-19
+user-1921,group-19
+user-1922,group-19
+user-1928,group-19
+user-1937,group-19
+user-1940,group-19
+user-1942,group-19
+user-1943,group-19
+user-1949,group-19
+user-1951,group-19
+user-1952,group-19
+user-1954,group-19
+user-1966,group-19
+user-1967,group-19
+user-1972,group-19
+user-1979,group-19
+user-1984,group-19
+user-1985,group-19
+user-1988,group-19
+user-1990,group-19
+user-2002,group-20
+user-2010,group-20
+user-2014,group-20
+user-2021,group-20
+user-2024,group-20
+user-2027,group-20
+user-2028,group-20
+user-2030,group-20
+user-2039,group-20
+user-2044,group-20
+user-2058,group-20
+user-2061,group-20
+user-2064,group-20
+user-2069,group-20
+user-2078,group-20
+user-2084,group-20
+user-2087,group-20
+user-2092,group-20
+user-2096,group-20
+user-2110,group-21
+user-2113,group-21
+user-2117,group-21
+user-2124,group-21
+user-2129,group-21
+user-2130,group-21
+user-2139,group-21
+user-2148,group-21
+user-2150,group-21
+user-2151,group-21
+user-2159,group-21
+user-2160,group-21
+user-2168,group-21
+user-2177,group-21
+user-2182,group-21
+user-2189,group-21
+user-2194,group-21
+user-2202,group-22
+user-2207,group-22
+user-2217,group-22
+user-2221,group-22
+user-2223,group-22
+user-2224,group-22
+user-2226,group-22
+user-2237,group-22
+user-2241,group-22
+user-2242,group-22
+user-2244,group-22
+user-2255,group-22
+user-2264,group-22
+user-2265,group-22
+user-2268,group-22
+user-2269,group-22
+user-2273,group-22
+user-2286,group-22
+user-2288,group-22
+user-2302,group-23
+user-2310,group-23
+user-2312,group-23
+user-2315,group-23
+user-2320,group-23
+user-2325,group-23
+user-2327,group-23
+user-2334,group-23
+user-2340,group-23
+user-2355,group-23
+user-2356,group-23
+user-2364,group-23
+user-2365,group-23
+user-2368,group-23
+user-2386,group-23
+user-2390,group-23
+user-2394,group-23
+user-2399,group-23
+user-2400,group-24
+user-2402,group-24
+user-2407,group-24
+user-2409,group-24
+user-2414,group-24
+user-2415,group-24
+user-2416,group-24
+user-2417,group-24
+user-2418,group-24
+user-2420,group-24
+user-2423,group-24
+user-2426,group-24
+user-2432,group-24
+user-2435,group-24
+user-2436,group-24
+user-2437,group-24
+user-2442,group-24
+user-2443,group-24
+user-2444,group-24
+user-2446,group-24
+user-2448,group-24
+user-2449,group-24
+user-2454,group-24
+user-2458,group-24
+user-2467,group-24
+user-2472,group-24
+user-2474,group-24
+user-2475,group-24
+user-2478,group-24
+user-2482,group-24
+user-2489,group-24
+user-2494,group-24
+user-2496,group-24
+user-2498,group-24
+user-2499,group-24
+user-2500,group-25
+user-2501,group-25
+user-2512,group-25
+user-2513,group-25
+user-2514,group-25
+user-2517,group-25
+user-2522,group-25
+user-2524,group-25
+user-2526,group-25
+user-2529,group-25
+user-2530,group-25
+user-2532,group-25
+user-2546,group-25
+user-2551,group-25
+user-2552,group-25
+user-2564,group-25
+user-2570,group-25
+user-2580,group-25
+user-2583,group-25
+user-2584,group-25
+user-2588,group-25
+user-2589,group-25
+user-2590,group-25
+user-2591,group-25
+user-2594,group-25
+user-2596,group-25
+user-2597,group-25
+user-2601,group-26
+user-2606,group-26
+user-2607,group-26
+user-2608,group-26
+user-2610,group-26
+user-2616,group-26
+user-2619,group-26
+user-2620,group-26
+user-2625,group-26
+user-2634,group-26
+user-2648,group-26
+user-2656,group-26
+user-2664,group-26
+user-2665,group-26
+user-2677,group-26
+user-2681,group-26
+user-2682,group-26
+user-2684,group-26
+user-2686,group-26
+user-2690,group-26
+user-2693,group-26
+user-2694,group-26
+user-2700,group-27
+user-2707,group-27
+user-2709,group-27
+user-2713,group-27
+user-2716,group-27
+user-2718,group-27
+user-2736,group-27
+user-2740,group-27
+user-2745,group-27
+user-2750,group-27
+user-2756,group-27
+user-2759,group-27
+user-2760,group-27
+user-2763,group-27
+user-2764,group-27
+user-2768,group-27
+user-2769,group-27
+user-2777,group-27
+user-2778,group-27
+user-2782,group-27
+user-2783,group-27
+user-2785,group-27
+user-2789,group-27
+user-2790,group-27
+user-2798,group-27
+user-2807,group-28
+user-2816,group-28
+user-2817,group-28
+user-2830,group-28
+user-2832,group-28
+user-2833,group-28
+user-2834,group-28
+user-2836,group-28
+user-2838,group-28
+user-2845,group-28
+user-2849,group-28
+user-2850,group-28
+user-2865,group-28
+user-2878,group-28
+user-2879,group-28
+user-2880,group-28
+user-2890,group-28
+user-2894,group-28
+user-2901,group-29
+user-2908,group-29
+user-2909,group-29
+user-2910,group-29
+user-2912,group-29
+user-2913,group-29
+user-2914,group-29
+user-2922,group-29
+user-2924,group-29
+user-2928,group-29
+user-2932,group-29
+user-2940,group-29
+user-2942,group-29
+user-2944,group-29
+user-2947,group-29
+user-2948,group-29
+user-2949,group-29
+user-2953,group-29
+user-2958,group-29
+user-2961,group-29
+user-2967,group-29
+user-2977,group-29
+user-2983,group-29
+user-2988,group-29
+user-3004,group-30
+user-3007,group-30
+user-3010,group-30
+user-3011,group-30
+user-3013,group-30
+user-3016,group-30
+user-3018,group-30
+user-3019,group-30
+user-3021,group-30
+user-3025,group-30
+user-3026,group-30
+user-3029,group-30
+user-3037,group-30
+user-3049,group-30
+user-3052,group-30
+user-3057,group-30
+user-3059,group-30
+user-3060,group-30
+user-3062,group-30
+user-3091,group-30
+user-3094,group-30
+user-3100,group-31
+user-3101,group-31
+user-3107,group-31
+user-3119,group-31
+user-3120,group-31
+user-3125,group-31
+user-3144,group-31
+user-3148,group-31
+user-3149,group-31
+user-3159,group-31
+user-3163,group-31
+user-3167,group-31
+user-3181,group-31
+user-3183,group-31
+user-3189,group-31
+user-3190,group-31
+user-3198,group-31
+user-3199,group-31
+user-3205,group-32
+user-3208,group-32
+user-3212,group-32
+user-3213,group-32
+user-3218,group-32
+user-3219,group-32
+user-3221,group-32
+user-3223,group-32
+user-3226,group-32
+user-3228,group-32
+user-3229,group-32
+user-3231,group-32
+user-3243,group-32
+user-3245,group-32
+user-3246,group-32
+user-3249,group-32
+user-3264,group-32
+user-3268,group-32
+user-3270,group-32
+user-3272,group-32
+user-3274,group-32
+user-3275,group-32
+user-3277,group-32
+user-3279,group-32
+user-3286,group-32
+user-3288,group-32
+user-3290,group-32
+user-3291,group-32
+user-3294,group-32
+user-3301,group-33
+user-3302,group-33
+user-3319,group-33
+user-3324,group-33
+user-3325,group-33
+user-3329,group-33
+user-3331,group-33
+user-3336,group-33
+user-3337,group-33
+user-3344,group-33
+user-3346,group-33
+user-3359,group-33
+user-3373,group-33
+user-3376,group-33
+user-3380,group-33
+user-3381,group-33
+user-3382,group-33
+user-3386,group-33
+user-3394,group-33
+user-3396,group-33
+user-3398,group-33
+user-3401,group-34
+user-3403,group-34
+user-3404,group-34
+user-3408,group-34
+user-3410,group-34
+user-3422,group-34
+user-3426,group-34
+user-3428,group-34
+user-3434,group-34
+user-3439,group-34
+user-3448,group-34
+user-3449,group-34
+user-3450,group-34
+user-3451,group-34
+user-3455,group-34
+user-3456,group-34
+user-3459,group-34
+user-3460,group-34
+user-3461,group-34
+user-3463,group-34
+user-3466,group-34
+user-3470,group-34
+user-3480,group-34
+user-3481,group-34
+user-3484,group-34
+user-3495,group-34
+user-3497,group-34
+user-3498,group-34
+user-3499,group-34
+user-3500,group-35
+user-3501,group-35
+user-3512,group-35
+user-3515,group-35
+user-3523,group-35
+user-3527,group-35
+user-3529,group-35
+user-3532,group-35
+user-3534,group-35
+user-3540,group-35
+user-3545,group-35
+user-3546,group-35
+user-3549,group-35
+user-3550,group-35
+user-3554,group-35
+user-3557,group-35
+user-3562,group-35
+user-3565,group-35
+user-3571,group-35
+user-3573,group-35
+user-3580,group-35
+user-3584,group-35
+user-3585,group-35
+user-3586,group-35
+user-3588,group-35
+user-3589,group-35
+user-3593,group-35
+user-3600,group-36
+user-3603,group-36
+user-3616,group-36
+user-3618,group-36
+user-3625,group-36
+user-3637,group-36
+user-3641,group-36
+user-3642,group-36
+user-3647,group-36
+user-3649,group-36
+user-3650,group-36
+user-3657,group-36
+user-3658,group-36
+user-3667,group-36
+user-3674,group-36
+user-3675,group-36
+user-3678,group-36
+user-3686,group-36
+user-3694,group-36
+user-3695,group-36
+user-3705,group-37
+user-3712,group-37
+user-3716,group-37
+user-3717,group-37
+user-3722,group-37
+user-3723,group-37
+user-3727,group-37
+user-3745,group-37
+user-3746,group-37
+user-3756,group-37
+user-3758,group-37
+user-3759,group-37
+user-3762,group-37
+user-3767,group-37
+user-3772,group-37
+user-3777,group-37
+user-3778,group-37
+user-3781,group-37
+user-3782,group-37
+user-3787,group-37
+user-3794,group-37
+user-3807,group-38
+user-3812,group-38
+user-3814,group-38
+user-3815,group-38
+user-3821,group-38
+user-3823,group-38
+user-3829,group-38
+user-3831,group-38
+user-3832,group-38
+user-3835,group-38
+user-3838,group-38
+user-3839,group-38
+user-3840,group-38
+user-3841,group-38
+user-3843,group-38
+user-3844,group-38
+user-3845,group-38
+user-3849,group-38
+user-3854,group-38
+user-3859,group-38
+user-3865,group-38
+user-3867,group-38
+user-3872,group-38
+user-3884,group-38
+user-3885,group-38
+user-3889,group-38
+user-3891,group-38
+user-3897,group-38
+user-3902,group-39
+user-3906,group-39
+user-3907,group-39
+user-3915,group-39
+user-3920,group-39
+user-3921,group-39
+user-3929,group-39
+user-3933,group-39
+user-3935,group-39
+user-3943,group-39
+user-3957,group-39
+user-3959,group-39
+user-3960,group-39
+user-3974,group-39
+user-3979,group-39
+user-3988,group-39
+user-4001,group-40
+user-4003,group-40
+user-4009,group-40
+user-4013,group-40
+user-4017,group-40
+user-4022,group-40
+user-4023,group-40
+user-4024,group-40
+user-4029,group-40
+user-4030,group-40
+user-4031,group-40
+user-4032,group-40
+user-4035,group-40
+user-4042,group-40
+user-4058,group-40
+user-4060,group-40
+user-4061,group-40
+user-4064,group-40
+user-4079,group-40
+user-4080,group-40
+user-4083,group-40
+user-4086,group-40
+user-4091,group-40
+user-4093,group-40
+user-4095,group-40
+user-4098,group-40
+user-4103,group-41
+user-4107,group-41
+user-4119,group-41
+user-4123,group-41
+user-4124,group-41
+user-4126,group-41
+user-4127,group-41
+user-4133,group-41
+user-4135,group-41
+user-4144,group-41
+user-4150,group-41
+user-4153,group-41
+user-4155,group-41
+user-4156,group-41
+user-4167,group-41
+user-4173,group-41
+user-4176,group-41
+user-4177,group-41
+user-4181,group-41
+user-4186,group-41
+user-4197,group-41
+user-4199,group-41
+user-4204,group-42
+user-4207,group-42
+user-4215,group-42
+user-4217,group-42
+user-4218,group-42
+user-4219,group-42
+user-4221,group-42
+user-4222,group-42
+user-4230,group-42
+user-4234,group-42
+user-4239,group-42
+user-4243,group-42
+user-4246,group-42
+user-4247,group-42
+user-4250,group-42
+user-4251,group-42
+user-4258,group-42
+user-4260,group-42
+user-4261,group-42
+user-4268,group-42
+user-4278,group-42
+user-4279,group-42
+user-4283,group-42
+user-4284,group-42
+user-4287,group-42
+user-4296,group-42
+user-4297,group-42
+user-4304,group-43
+user-4316,group-43
+user-4321,group-43
+user-4323,group-43
+user-4331,group-43
+user-4335,group-43
+user-4337,group-43
+user-4344,group-43
+user-4346,group-43
+user-4347,group-43
+user-4349,group-43
+user-4351,group-43
+user-4362,group-43
+user-4369,group-43
+user-4374,group-43
+user-4378,group-43
+user-4381,group-43
+user-4385,group-43
+user-4387,group-43
+user-4396,group-43
+user-4400,group-44
+user-4403,group-44
+user-4405,group-44
+user-4408,group-44
+user-4420,group-44
+user-4422,group-44
+user-4428,group-44
+user-4430,group-44
+user-4432,group-44
+user-4438,group-44
+user-4439,group-44
+user-4440,group-44
+user-4441,group-44
+user-4445,group-44
+user-4446,group-44
+user-4449,group-44
+user-4451,group-44
+user-4461,group-44
+user-4462,group-44
+user-4464,group-44
+user-4466,group-44
+user-4468,group-44
+user-4474,group-44
+user-4476,group-44
+user-4481,group-44
+user-4487,group-44
+user-4488,group-44
+user-4489,group-44
+user-4494,group-44
+user-4501,group-45
+user-4506,group-45
+user-4509,group-45
+user-4516,group-45
+user-4527,group-45
+user-4529,group-45
+user-4532,group-45
+user-4540,group-45
+user-4547,group-45
+user-4549,group-45
+user-4554,group-45
+user-4556,group-45
+user-4572,group-45
+user-4575,group-45
+user-4576,group-45
+user-4581,group-45
+user-4586,group-45
+user-4597,group-45
+user-4600,group-46
+user-4601,group-46
+user-4602,group-46
+user-4604,group-46
+user-4607,group-46
+user-4611,group-46
+user-4615,group-46
+user-4629,group-46
+user-4636,group-46
+user-4642,group-46
+user-4644,group-46
+user-4645,group-46
+user-4647,group-46
+user-4650,group-46
+user-4653,group-46
+user-4655,group-46
+user-4667,group-46
+user-4668,group-46
+user-4673,group-46
+user-4676,group-46
+user-4679,group-46
+user-4683,group-46
+user-4696,group-46
+user-4700,group-47
+user-4701,group-47
+user-4705,group-47
+user-4708,group-47
+user-4711,group-47
+user-4716,group-47
+user-4722,group-47
+user-4725,group-47
+user-4726,group-47
+user-4735,group-47
+user-4737,group-47
+user-4748,group-47
+user-4753,group-47
+user-4755,group-47
+user-4757,group-47
+user-4761,group-47
+user-4762,group-47
+user-4768,group-47
+user-4770,group-47
+user-4771,group-47
+user-4773,group-47
+user-4774,group-47
+user-4778,group-47
+user-4779,group-47
+user-4786,group-47
+user-4787,group-47
+user-4791,group-47
+user-4792,group-47
+user-4794,group-47
+user-4797,group-47
+user-4801,group-48
+user-4803,group-48
+user-4812,group-48
+user-4814,group-48
+user-4821,group-48
+user-4832,group-48
+user-4834,group-48
+user-4836,group-48
+user-4847,group-48
+user-4854,group-48
+user-4856,group-48
+user-4862,group-48
+user-4863,group-48
+user-4864,group-48
+user-4877,group-48
+user-4882,group-48
+user-4883,group-48
+user-4886,group-48
+user-4889,group-48
+user-4890,group-48
+user-4893,group-48
+user-4895,group-48
+user-4902,group-49
+user-4908,group-49
+user-4923,group-49
+user-4929,group-49
+user-4934,group-49
+user-4940,group-49
+user-4946,group-49
+user-4952,group-49
+user-4953,group-49
+user-4954,group-49
+user-4957,group-49
+user-4961,group-49
+user-4965,group-49
+user-4970,group-49
+user-4977,group-49
+user-4980,group-49
+user-5000,group-50
+user-5001,group-50
+user-5004,group-50
+user-5006,group-50
+user-5009,group-50
+user-5017,group-50
+user-5023,group-50
+user-5024,group-50
+user-5025,group-50
+user-5028,group-50
+user-5033,group-50
+user-5034,group-50
+user-5040,group-50
+user-5046,group-50
+user-5052,group-50
+user-5057,group-50
+user-5060,group-50
+user-5062,group-50
+user-5067,group-50
+user-5071,group-50
+user-5077,group-50
+user-5078,group-50
+user-5079,group-50
+user-5084,group-50
+user-5087,group-50
+user-5097,group-50
+user-5100,group-51
+user-5103,group-51
+user-5104,group-51
+user-5107,group-51
+user-5112,group-51
+user-5114,group-51
+user-5118,group-51
+user-5121,group-51
+user-5122,group-51
+user-5123,group-51
+user-5124,group-51
+user-5128,group-51
+user-5132,group-51
+user-5135,group-51
+user-5136,group-51
+user-5137,group-51
+user-5139,group-51
+user-5143,group-51
+user-5144,group-51
+user-5147,group-51
+user-5156,group-51
+user-5157,group-51
+user-5163,group-51
+user-5164,group-51
+user-5166,group-51
+user-5169,group-51
+user-5171,group-51
+user-5172,group-51
+user-5180,group-51
+user-5181,group-51
+user-5186,group-51
+user-5190,group-51
+user-5191,group-51
+user-5192,group-51
+user-5201,group-52
+user-5209,group-52
+user-5218,group-52
+user-5220,group-52
+user-5228,group-52
+user-5230,group-52
+user-5231,group-52
+user-5236,group-52
+user-5241,group-52
+user-5251,group-52
+user-5258,group-52
+user-5264,group-52
+user-5270,group-52
+user-5274,group-52
+user-5278,group-52
+user-5279,group-52
+user-5282,group-52
+user-5286,group-52
+user-5290,group-52
+user-5293,group-52
+user-5299,group-52
+user-5304,group-53
+user-5315,group-53
+user-5316,group-53
+user-5318,group-53
+user-5319,group-53
+user-5320,group-53
+user-5322,group-53
+user-5324,group-53
+user-5325,group-53
+user-5329,group-53
+user-5333,group-53
+user-5345,group-53
+user-5351,group-53
+user-5354,group-53
+user-5357,group-53
+user-5360,group-53
+user-5361,group-53
+user-5363,group-53
+user-5367,group-53
+user-5368,group-53
+user-5370,group-53
+user-5371,group-53
+user-5376,group-53
+user-5377,group-53
+user-5388,group-53
+user-5389,group-53
+user-5396,group-53
+user-5407,group-54
+user-5409,group-54
+user-5422,group-54
+user-5423,group-54
+user-5424,group-54
+user-5434,group-54
+user-5438,group-54
+user-5439,group-54
+user-5441,group-54
+user-5446,group-54
+user-5454,group-54
+user-5457,group-54
+user-5471,group-54
+user-5481,group-54
+user-5482,group-54
+user-5485,group-54
+user-5489,group-54
+user-5501,group-55
+user-5503,group-55
+user-5504,group-55
+user-5505,group-55
+user-5508,group-55
+user-5509,group-55
+user-5514,group-55
+user-5516,group-55
+user-5517,group-55
+user-5518,group-55
+user-5521,group-55
+user-5522,group-55
+user-5523,group-55
+user-5524,group-55
+user-5528,group-55
+user-5532,group-55
+user-5535,group-55
+user-5536,group-55
+user-5539,group-55
+user-5540,group-55
+user-5542,group-55
+user-5549,group-55
+user-5550,group-55
+user-5554,group-55
+user-5555,group-55
+user-5559,group-55
+user-5561,group-55
+user-5562,group-55
+user-5566,group-55
+user-5569,group-55
+user-5574,group-55
+user-5579,group-55
+user-5580,group-55
+user-5582,group-55
+user-5592,group-55
+user-5594,group-55
+user-5595,group-55
+user-5599,group-55
+user-5606,group-56
+user-5607,group-56
+user-5608,group-56
+user-5614,group-56
+user-5622,group-56
+user-5629,group-56
+user-5638,group-56
+user-5639,group-56
+user-5644,group-56
+user-5661,group-56
+user-5663,group-56
+user-5670,group-56
+user-5674,group-56
+user-5675,group-56
+user-5677,group-56
+user-5684,group-56
+user-5685,group-56
+user-5689,group-56
+user-5694,group-56
+user-5698,group-56
+user-5703,group-57
+user-5723,group-57
+user-5731,group-57
+user-5733,group-57
+user-5736,group-57
+user-5738,group-57
+user-5747,group-57
+user-5753,group-57
+user-5757,group-57
+user-5759,group-57
+user-5769,group-57
+user-5774,group-57
+user-5777,group-57
+user-5782,group-57
+user-5788,group-57
+user-5793,group-57
+user-5797,group-57
+user-5799,group-57
+user-5804,group-58
+user-5805,group-58
+user-5811,group-58
+user-5812,group-58
+user-5813,group-58
+user-5818,group-58
+user-5826,group-58
+user-5828,group-58
+user-5830,group-58
+user-5831,group-58
+user-5832,group-58
+user-5834,group-58
+user-5840,group-58
+user-5841,group-58
+user-5847,group-58
+user-5853,group-58
+user-5854,group-58
+user-5856,group-58
+user-5858,group-58
+user-5860,group-58
+user-5862,group-58
+user-5866,group-58
+user-5870,group-58
+user-5875,group-58
+user-5876,group-58
+user-5882,group-58
+user-5885,group-58
+user-5886,group-58
+user-5893,group-58
+user-5895,group-58
+user-5897,group-58
+user-5908,group-59
+user-5909,group-59
+user-5914,group-59
+user-5917,group-59
+user-5919,group-59
+user-5924,group-59
+user-5926,group-59
+user-5933,group-59
+user-5939,group-59
+user-5940,group-59
+user-5948,group-59
+user-5955,group-59
+user-5956,group-59
+user-5958,group-59
+user-5959,group-59
+user-5961,group-59
+user-5987,group-59
+user-5991,group-59
+user-5993,group-59
+user-5994,group-59
+user-6000,group-60
+user-6002,group-60
+user-6004,group-60
+user-6013,group-60
+user-6017,group-60
+user-6024,group-60
+user-6033,group-60
+user-6042,group-60
+user-6043,group-60
+user-6055,group-60
+user-6062,group-60
+user-6064,group-60
+user-6071,group-60
+user-6072,group-60
+user-6076,group-60
+user-6080,group-60
+user-6085,group-60
+user-6088,group-60
+user-6096,group-60
+user-6104,group-61
+user-6105,group-61
+user-6109,group-61
+user-6113,group-61
+user-6114,group-61
+user-6124,group-61
+user-6127,group-61
+user-6129,group-61
+user-6130,group-61
+user-6136,group-61
+user-6143,group-61
+user-6144,group-61
+user-6147,group-61
+user-6153,group-61
+user-6158,group-61
+user-6162,group-61
+user-6167,group-61
+user-6170,group-61
+user-6172,group-61
+user-6175,group-61
+user-6176,group-61
+user-6177,group-61
+user-6194,group-61
+user-6196,group-61
+user-6209,group-62
+user-6211,group-62
+user-6222,group-62
+user-6223,group-62
+user-6225,group-62
+user-6226,group-62
+user-6230,group-62
+user-6233,group-62
+user-6234,group-62
+user-6241,group-62
+user-6249,group-62
+user-6250,group-62
+user-6251,group-62
+user-6255,group-62
+user-6258,group-62
+user-6259,group-62
+user-6261,group-62
+user-6268,group-62
+user-6283,group-62
+user-6284,group-62
+user-6290,group-62
+user-6299,group-62
+user-6301,group-63
+user-6302,group-63
+user-6304,group-63
+user-6305,group-63
+user-6311,group-63
+user-6316,group-63
+user-6317,group-63
+user-6318,group-63
+user-6320,group-63
+user-6321,group-63
+user-6328,group-63
+user-6331,group-63
+user-6334,group-63
+user-6335,group-63
+user-6337,group-63
+user-6343,group-63
+user-6347,group-63
+user-6351,group-63
+user-6359,group-63
+user-6360,group-63
+user-6361,group-63
+user-6365,group-63
+user-6368,group-63
+user-6371,group-63
+user-6372,group-63
+user-6380,group-63
+user-6385,group-63
+user-6386,group-63
+user-6390,group-63
+user-6398,group-63
+user-6399,group-63
+user-6403,group-64
+user-6404,group-64
+user-6405,group-64
+user-6407,group-64
+user-6411,group-64
+user-6412,group-64
+user-6415,group-64
+user-6418,group-64
+user-6419,group-64
+user-6422,group-64
+user-6428,group-64
+user-6436,group-64
+user-6441,group-64
+user-6443,group-64
+user-6449,group-64
+user-6450,group-64
+user-6460,group-64
+user-6471,group-64
+user-6476,group-64
+user-6482,group-64
+user-6483,group-64
+user-6485,group-64
+user-6486,group-64
+user-6490,group-64
+user-6513,group-65
+user-6515,group-65
+user-6518,group-65
+user-6522,group-65
+user-6530,group-65
+user-6531,group-65
+user-6544,group-65
+user-6545,group-65
+user-6556,group-65
+user-6562,group-65
+user-6563,group-65
+user-6566,group-65
+user-6567,group-65
+user-6578,group-65
+user-6580,group-65
+user-6582,group-65
+user-6583,group-65
+user-6585,group-65
+user-6588,group-65
+user-6590,group-65
+user-6599,group-65
+user-6601,group-66
+user-6604,group-66
+user-6610,group-66
+user-6612,group-66
+user-6613,group-66
+user-6614,group-66
+user-6616,group-66
+user-6618,group-66
+user-6619,group-66
+user-6621,group-66
+user-6625,group-66
+user-6626,group-66
+user-6627,group-66
+user-6631,group-66
+user-6639,group-66
+user-6642,group-66
+user-6647,group-66
+user-6649,group-66
+user-6652,group-66
+user-6656,group-66
+user-6659,group-66
+user-6666,group-66
+user-6678,group-66
+user-6681,group-66
+user-6692,group-66
+user-6694,group-66
+user-6696,group-66
+user-6704,group-67
+user-6709,group-67
+user-6710,group-67
+user-6713,group-67
+user-6715,group-67
+user-6717,group-67
+user-6727,group-67
+user-6731,group-67
+user-6734,group-67
+user-6738,group-67
+user-6740,group-67
+user-6742,group-67
+user-6743,group-67
+user-6755,group-67
+user-6756,group-67
+user-6757,group-67
+user-6768,group-67
+user-6771,group-67
+user-6772,group-67
+user-6774,group-67
+user-6776,group-67
+user-6779,group-67
+user-6792,group-67
+user-6794,group-67
+user-6797,group-67
+user-6800,group-68
+user-6803,group-68
+user-6804,group-68
+user-6805,group-68
+user-6808,group-68
+user-6811,group-68
+user-6819,group-68
+user-6823,group-68
+user-6827,group-68
+user-6834,group-68
+user-6836,group-68
+user-6837,group-68
+user-6839,group-68
+user-6849,group-68
+user-6851,group-68
+user-6859,group-68
+user-6861,group-68
+user-6863,group-68
+user-6868,group-68
+user-6877,group-68
+user-6885,group-68
+user-6887,group-68
+user-6894,group-68
+user-6899,group-68
+user-6902,group-69
+user-6903,group-69
+user-6905,group-69
+user-6909,group-69
+user-6912,group-69
+user-6913,group-69
+user-6914,group-69
+user-6916,group-69
+user-6919,group-69
+user-6921,group-69
+user-6923,group-69
+user-6924,group-69
+user-6925,group-69
+user-6927,group-69
+user-6935,group-69
+user-6936,group-69
+user-6937,group-69
+user-6940,group-69
+user-6941,group-69
+user-6944,group-69
+user-6952,group-69
+user-6957,group-69
+user-6958,group-69
+user-6959,group-69
+user-6965,group-69
+user-6966,group-69
+user-6970,group-69
+user-6972,group-69
+user-6984,group-69
+user-6985,group-69
+user-6990,group-69
+user-6996,group-69
+user-6999,group-69
+user-7003,group-70
+user-7010,group-70
+user-7015,group-70
+user-7016,group-70
+user-7018,group-70
+user-7020,group-70
+user-7021,group-70
+user-7022,group-70
+user-7023,group-70
+user-7028,group-70
+user-7031,group-70
+user-7037,group-70
+user-7038,group-70
+user-7039,group-70
+user-7041,group-70
+user-7045,group-70
+user-7046,group-70
+user-7050,group-70
+user-7052,group-70
+user-7058,group-70
+user-7060,group-70
+user-7068,group-70
+user-7073,group-70
+user-7074,group-70
+user-7076,group-70
+user-7078,group-70
+user-7082,group-70
+user-7085,group-70
+user-7086,group-70
+user-7087,group-70
+user-7089,group-70
+user-7095,group-70
+user-7097,group-70
+user-7102,group-71
+user-7111,group-71
+user-7114,group-71
+user-7118,group-71
+user-7119,group-71
+user-7121,group-71
+user-7124,group-71
+user-7129,group-71
+user-7137,group-71
+user-7140,group-71
+user-7150,group-71
+user-7164,group-71
+user-7165,group-71
+user-7167,group-71
+user-7172,group-71
+user-7183,group-71
+user-7185,group-71
+user-7191,group-71
+user-7192,group-71
+user-7195,group-71
+user-7201,group-72
+user-7212,group-72
+user-7228,group-72
+user-7234,group-72
+user-7236,group-72
+user-7246,group-72
+user-7253,group-72
+user-7256,group-72
+user-7260,group-72
+user-7267,group-72
+user-7270,group-72
+user-7272,group-72
+user-7273,group-72
+user-7282,group-72
+user-7285,group-72
+user-7288,group-72
+user-7289,group-72
+user-7290,group-72
+user-7299,group-72
+user-7306,group-73
+user-7309,group-73
+user-7311,group-73
+user-7312,group-73
+user-7314,group-73
+user-7316,group-73
+user-7320,group-73
+user-7327,group-73
+user-7329,group-73
+user-7338,group-73
+user-7340,group-73
+user-7351,group-73
+user-7364,group-73
+user-7365,group-73
+user-7367,group-73
+user-7381,group-73
+user-7388,group-73
+user-7390,group-73
+user-7391,group-73
+user-7396,group-73
+user-7403,group-74
+user-7407,group-74
+user-7408,group-74
+user-7410,group-74
+user-7415,group-74
+user-7417,group-74
+user-7420,group-74
+user-7426,group-74
+user-7427,group-74
+user-7430,group-74
+user-7431,group-74
+user-7433,group-74
+user-7447,group-74
+user-7449,group-74
+user-7450,group-74
+user-7454,group-74
+user-7461,group-74
+user-7472,group-74
+user-7477,group-74
+user-7479,group-74
+user-7486,group-74
+user-7494,group-74
+user-7498,group-74
+user-7506,group-75
+user-7510,group-75
+user-7513,group-75
+user-7515,group-75
+user-7516,group-75
+user-7518,group-75
+user-7519,group-75
+user-7522,group-75
+user-7532,group-75
+user-7536,group-75
+user-7543,group-75
+user-7544,group-75
+user-7546,group-75
+user-7553,group-75
+user-7554,group-75
+user-7555,group-75
+user-7558,group-75
+user-7560,group-75
+user-7567,group-75
+user-7568,group-75
+user-7569,group-75
+user-7571,group-75
+user-7572,group-75
+user-7576,group-75
+user-7583,group-75
+user-7586,group-75
+user-7591,group-75
+user-7592,group-75
+user-7593,group-75
+user-7595,group-75
+user-7600,group-76
+user-7605,group-76
+user-7606,group-76
+user-7607,group-76
+user-7609,group-76
+user-7611,group-76
+user-7629,group-76
+user-7630,group-76
+user-7633,group-76
+user-7639,group-76
+user-7642,group-76
+user-7644,group-76
+user-7651,group-76
+user-7652,group-76
+user-7657,group-76
+user-7659,group-76
+user-7665,group-76
+user-7676,group-76
+user-7678,group-76
+user-7685,group-76
+user-7686,group-76
+user-7687,group-76
+user-7688,group-76
+user-7689,group-76
+user-7695,group-76
+user-7696,group-76
+user-7698,group-76
+user-7700,group-77
+user-7703,group-77
+user-7705,group-77
+user-7707,group-77
+user-7713,group-77
+user-7716,group-77
+user-7718,group-77
+user-7719,group-77
+user-7725,group-77
+user-7731,group-77
+user-7732,group-77
+user-7734,group-77
+user-7737,group-77
+user-7739,group-77
+user-7761,group-77
+user-7766,group-77
+user-7772,group-77
+user-7773,group-77
+user-7785,group-77
+user-7786,group-77
+user-7795,group-77
+user-7798,group-77
+user-7803,group-78
+user-7805,group-78
+user-7813,group-78
+user-7815,group-78
+user-7822,group-78
+user-7833,group-78
+user-7834,group-78
+user-7838,group-78
+user-7839,group-78
+user-7855,group-78
+user-7861,group-78
+user-7865,group-78
+user-7868,group-78
+user-7870,group-78
+user-7872,group-78
+user-7873,group-78
+user-7876,group-78
+user-7878,group-78
+user-7879,group-78
+user-7882,group-78
+user-7888,group-78
+user-7891,group-78
+user-7894,group-78
+user-7895,group-78
+user-7898,group-78
+user-7901,group-79
+user-7902,group-79
+user-7907,group-79
+user-7908,group-79
+user-7913,group-79
+user-7914,group-79
+user-7923,group-79
+user-7927,group-79
+user-7930,group-79
+user-7936,group-79
+user-7938,group-79
+user-7941,group-79
+user-7944,group-79
+user-7954,group-79
+user-7959,group-79
+user-7965,group-79
+user-7968,group-79
+user-7970,group-79
+user-7972,group-79
+user-7975,group-79
+user-7998,group-79
+user-8002,group-80
+user-8004,group-80
+user-8006,group-80
+user-8009,group-80
+user-8011,group-80
+user-8013,group-80
+user-8014,group-80
+user-8017,group-80
+user-8020,group-80
+user-8021,group-80
+user-8024,group-80
+user-8027,group-80
+user-8029,group-80
+user-8036,group-80
+user-8038,group-80
+user-8039,group-80
+user-8045,group-80
+user-8050,group-80
+user-8053,group-80
+user-8054,group-80
+user-8057,group-80
+user-8064,group-80
+user-8074,group-80
+user-8081,group-80
+user-8084,group-80
+user-8096,group-80
+user-8110,group-81
+user-8111,group-81
+user-8116,group-81
+user-8117,group-81
+user-8118,group-81
+user-8119,group-81
+user-8123,group-81
+user-8129,group-81
+user-8131,group-81
+user-8136,group-81
+user-8144,group-81
+user-8145,group-81
+user-8147,group-81
+user-8162,group-81
+user-8164,group-81
+user-8166,group-81
+user-8174,group-81
+user-8181,group-81
+user-8184,group-81
+user-8187,group-81
+user-8195,group-81
+user-8200,group-82
+user-8203,group-82
+user-8207,group-82
+user-8208,group-82
+user-8209,group-82
+user-8213,group-82
+user-8233,group-82
+user-8234,group-82
+user-8245,group-82
+user-8248,group-82
+user-8256,group-82
+user-8260,group-82
+user-8262,group-82
+user-8268,group-82
+user-8272,group-82
+user-8276,group-82
+user-8277,group-82
+user-8281,group-82
+user-8282,group-82
+user-8284,group-82
+user-8288,group-82
+user-8290,group-82
+user-8294,group-82
+user-8295,group-82
+user-8296,group-82
+user-8297,group-82
+user-8300,group-83
+user-8302,group-83
+user-8305,group-83
+user-8308,group-83
+user-8309,group-83
+user-8310,group-83
+user-8313,group-83
+user-8319,group-83
+user-8322,group-83
+user-8323,group-83
+user-8324,group-83
+user-8329,group-83
+user-8337,group-83
+user-8345,group-83
+user-8359,group-83
+user-8365,group-83
+user-8373,group-83
+user-8374,group-83
+user-8375,group-83
+user-8377,group-83
+user-8383,group-83
+user-8385,group-83
+user-8391,group-83
+user-8392,group-83
+user-8393,group-83
+user-8395,group-83
+user-8403,group-84
+user-8405,group-84
+user-8411,group-84
+user-8413,group-84
+user-8414,group-84
+user-8417,group-84
+user-8421,group-84
+user-8428,group-84
+user-8431,group-84
+user-8432,group-84
+user-8436,group-84
+user-8439,group-84
+user-8443,group-84
+user-8444,group-84
+user-8446,group-84
+user-8467,group-84
+user-8468,group-84
+user-8474,group-84
+user-8477,group-84
+user-8488,group-84
+user-8492,group-84
+user-8500,group-85
+user-8503,group-85
+user-8505,group-85
+user-8507,group-85
+user-8509,group-85
+user-8511,group-85
+user-8516,group-85
+user-8523,group-85
+user-8529,group-85
+user-8537,group-85
+user-8545,group-85
+user-8549,group-85
+user-8554,group-85
+user-8560,group-85
+user-8570,group-85
+user-8574,group-85
+user-8575,group-85
+user-8576,group-85
+user-8578,group-85
+user-8581,group-85
+user-8583,group-85
+user-8585,group-85
+user-8596,group-85
+user-8597,group-85
+user-8610,group-86
+user-8612,group-86
+user-8627,group-86
+user-8635,group-86
+user-8637,group-86
+user-8642,group-86
+user-8644,group-86
+user-8647,group-86
+user-8649,group-86
+user-8650,group-86
+user-8651,group-86
+user-8657,group-86
+user-8660,group-86
+user-8675,group-86
+user-8678,group-86
+user-8680,group-86
+user-8683,group-86
+user-8685,group-86
+user-8687,group-86
+user-8695,group-86
+user-8699,group-86
+user-8713,group-87
+user-8714,group-87
+user-8723,group-87
+user-8726,group-87
+user-8730,group-87
+user-8731,group-87
+user-8734,group-87
+user-8738,group-87
+user-8739,group-87
+user-8743,group-87
+user-8746,group-87
+user-8751,group-87
+user-8752,group-87
+user-8754,group-87
+user-8759,group-87
+user-8766,group-87
+user-8767,group-87
+user-8775,group-87
+user-8779,group-87
+user-8781,group-87
+user-8787,group-87
+user-8789,group-87
+user-8795,group-87
+user-8802,group-88
+user-8803,group-88
+user-8805,group-88
+user-8811,group-88
+user-8814,group-88
+user-8815,group-88
+user-8816,group-88
+user-8819,group-88
+user-8822,group-88
+user-8824,group-88
+user-8831,group-88
+user-8840,group-88
+user-8848,group-88
+user-8855,group-88
+user-8858,group-88
+user-8860,group-88
+user-8868,group-88
+user-8882,group-88
+user-8886,group-88
+user-8887,group-88
+user-8889,group-88
+user-8890,group-88
+user-8893,group-88
+user-8898,group-88
+user-8903,group-89
+user-8906,group-89
+user-8913,group-89
+user-8914,group-89
+user-8916,group-89
+user-8922,group-89
+user-8932,group-89
+user-8936,group-89
+user-8938,group-89
+user-8939,group-89
+user-8940,group-89
+user-8941,group-89
+user-8942,group-89
+user-8947,group-89
+user-8950,group-89
+user-8956,group-89
+user-8957,group-89
+user-8959,group-89
+user-8960,group-89
+user-8963,group-89
+user-8967,group-89
+user-8970,group-89
+user-8971,group-89
+user-8977,group-89
+user-8980,group-89
+user-8981,group-89
+user-8982,group-89
+user-8985,group-89
+user-8987,group-89
+user-8988,group-89
+user-8994,group-89
+user-9004,group-90
+user-9008,group-90
+user-9009,group-90
+user-9014,group-90
+user-9016,group-90
+user-9031,group-90
+user-9032,group-90
+user-9038,group-90
+user-9045,group-90
+user-9047,group-90
+user-9051,group-90
+user-9058,group-90
+user-9064,group-90
+user-9065,group-90
+user-9069,group-90
+user-9070,group-90
+user-9073,group-90
+user-9075,group-90
+user-9077,group-90
+user-9078,group-90
+user-9082,group-90
+user-9086,group-90
+user-9090,group-90
+user-9094,group-90
+user-9098,group-90
+user-9102,group-91
+user-9108,group-91
+user-9119,group-91
+user-9121,group-91
+user-9126,group-91
+user-9131,group-91
+user-9135,group-91
+user-9137,group-91
+user-9144,group-91
+user-9150,group-91
+user-9159,group-91
+user-9163,group-91
+user-9166,group-91
+user-9168,group-91
+user-9169,group-91
+user-9170,group-91
+user-9173,group-91
+user-9175,group-91
+user-9176,group-91
+user-9177,group-91
+user-9182,group-91
+user-9187,group-91
+user-9188,group-91
+user-9193,group-91
+user-9194,group-91
+user-9197,group-91
+user-9206,group-92
+user-9208,group-92
+user-9215,group-92
+user-9223,group-92
+user-9224,group-92
+user-9225,group-92
+user-9227,group-92
+user-9228,group-92
+user-9230,group-92
+user-9232,group-92
+user-9234,group-92
+user-9238,group-92
+user-9239,group-92
+user-9245,group-92
+user-9251,group-92
+user-9253,group-92
+user-9255,group-92
+user-9262,group-92
+user-9263,group-92
+user-9266,group-92
+user-9267,group-92
+user-9274,group-92
+user-9278,group-92
+user-9284,group-92
+user-9288,group-92
+user-9291,group-92
+user-9292,group-92
+user-9293,group-92
+user-9297,group-92
+user-9298,group-92
+user-9303,group-93
+user-9308,group-93
+user-9309,group-93
+user-9310,group-93
+user-9311,group-93
+user-9315,group-93
+user-9319,group-93
+user-9321,group-93
+user-9324,group-93
+user-9327,group-93
+user-9329,group-93
+user-9330,group-93
+user-9332,group-93
+user-9333,group-93
+user-9334,group-93
+user-9337,group-93
+user-9339,group-93
+user-9342,group-93
+user-9347,group-93
+user-9354,group-93
+user-9358,group-93
+user-9365,group-93
+user-9369,group-93
+user-9374,group-93
+user-9375,group-93
+user-9378,group-93
+user-9379,group-93
+user-9382,group-93
+user-9383,group-93
+user-9386,group-93
+user-9387,group-93
+user-9388,group-93
+user-9397,group-93
+user-9400,group-94
+user-9401,group-94
+user-9403,group-94
+user-9404,group-94
+user-9407,group-94
+user-9409,group-94
+user-9412,group-94
+user-9413,group-94
+user-9425,group-94
+user-9432,group-94
+user-9434,group-94
+user-9435,group-94
+user-9447,group-94
+user-9456,group-94
+user-9463,group-94
+user-9468,group-94
+user-9474,group-94
+user-9477,group-94
+user-9479,group-94
+user-9489,group-94
+user-9490,group-94
+user-9493,group-94
+user-9500,group-95
+user-9508,group-95
+user-9516,group-95
+user-9518,group-95
+user-9520,group-95
+user-9523,group-95
+user-9526,group-95
+user-9531,group-95
+user-9532,group-95
+user-9535,group-95
+user-9541,group-95
+user-9542,group-95
+user-9543,group-95
+user-9544,group-95
+user-9549,group-95
+user-9555,group-95
+user-9564,group-95
+user-9572,group-95
+user-9573,group-95
+user-9596,group-95
+user-9598,group-95
+user-9603,group-96
+user-9607,group-96
+user-9608,group-96
+user-9612,group-96
+user-9615,group-96
+user-9617,group-96
+user-9620,group-96
+user-9626,group-96
+user-9627,group-96
+user-9629,group-96
+user-9632,group-96
+user-9634,group-96
+user-9636,group-96
+user-9637,group-96
+user-9639,group-96
+user-9645,group-96
+user-9649,group-96
+user-9650,group-96
+user-9652,group-96
+user-9654,group-96
+user-9656,group-96
+user-9657,group-96
+user-9662,group-96
+user-9664,group-96
+user-9671,group-96
+user-9672,group-96
+user-9682,group-96
+user-9698,group-96
+user-9701,group-97
+user-9703,group-97
+user-9710,group-97
+user-9717,group-97
+user-9723,group-97
+user-9728,group-97
+user-9734,group-97
+user-9736,group-97
+user-9737,group-97
+user-9740,group-97
+user-9751,group-97
+user-9752,group-97
+user-9754,group-97
+user-9758,group-97
+user-9762,group-97
+user-9765,group-97
+user-9767,group-97
+user-9775,group-97
+user-9780,group-97
+user-9784,group-97
+user-9787,group-97
+user-9788,group-97
+user-9789,group-97
+user-9790,group-97
+user-9805,group-98
+user-9806,group-98
+user-9815,group-98
+user-9816,group-98
+user-9821,group-98
+user-9822,group-98
+user-9827,group-98
+user-9829,group-98
+user-9830,group-98
+user-9831,group-98
+user-9838,group-98
+user-9839,group-98
+user-9841,group-98
+user-9845,group-98
+user-9857,group-98
+user-9858,group-98
+user-9859,group-98
+user-9860,group-98
+user-9862,group-98
+user-9863,group-98
+user-9867,group-98
+user-9873,group-98
+user-9878,group-98
+user-9880,group-98
+user-9882,group-98
+user-9885,group-98
+user-9896,group-98
+user-9897,group-98
+user-9912,group-99
+user-9917,group-99
+user-9922,group-99
+user-9926,group-99
+user-9929,group-99
+user-9933,group-99
+user-9937,group-99
+user-9939,group-99
+user-9942,group-99
+user-9945,group-99
+user-9947,group-99
+user-9951,group-99
+user-9952,group-99
+user-9953,group-99
+user-9957,group-99
+user-9959,group-99
+user-9960,group-99
+user-9963,group-99
+user-9966,group-99
+user-9971,group-99
+user-9974,group-99
+user-9975,group-99
+user-9981,group-99
+user-9984,group-99
+user-9987,group-99
+user-9991,group-99
+user-9992,group-99
+user-9996,group-99
+user-9997,group-99
+user-9998,group-99
\ No newline at end of file
diff --git
a/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p4.csv
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p4.csv
new file mode 100644
index 0000000000..6f4f597ef0
--- /dev/null
+++
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p4.csv
@@ -0,0 +1,5 @@
+userUUID,groupUUID
+user-1,group-2
+user-2,group-2
+user-7,group-0
+user-15,group-2
diff --git
a/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p5.csv
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p5.csv
new file mode 100644
index 0000000000..e135aeea68
--- /dev/null
+++
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p5.csv
@@ -0,0 +1,5 @@
+userUUID,groupUUID
+user-6,group-3
+user-8,group-1
+user-14,group-1
+user-24,group-1
diff --git
a/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p6.csv
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p6.csv
new file mode 100644
index 0000000000..889b1be1d3
--- /dev/null
+++
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p6.csv
@@ -0,0 +1,5 @@
+userUUID,groupUUID
+user-5,group-1
+user-10,group-1
+user-19,group-1
+user-25,group-1
diff --git
a/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p7.csv
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p7.csv
new file mode 100644
index 0000000000..47afb26791
--- /dev/null
+++
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/rawdata/p7.csv
@@ -0,0 +1,8 @@
+userUUID,groupUUID
+user-0,group-1
+user-3,group-1
+user-4,group-2
+user-9,group-2
+user-13,group-2
+user-20,group-1
+user-22,group-1
diff --git
a/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/userGroupsDim_offline_table_config.json
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/userGroupsDim_offline_table_config.json
new file mode 100644
index 0000000000..004f1cfe05
--- /dev/null
+++
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/userGroupsDim_offline_table_config.json
@@ -0,0 +1,18 @@
+{
+ "tableName": "userGroupsDim",
+ "tableType": "OFFLINE",
+ "isDimTable": true,
+ "segmentsConfig": {
+ "replication": "2"
+ },
+ "tenants": {
+ "broker": "DefaultTenant",
+ "server": "DefaultTenant"
+ },
+ "tableIndexConfig": {
+ },
+ "metadata": {
+ "customConfigs": {
+ }
+ }
+}
diff --git
a/pinot-tools/src/main/resources/examples/batch/colocated/userGroups/userGroups_schema.json
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/userGroupsDim_schema.json
similarity index 68%
copy from
pinot-tools/src/main/resources/examples/batch/colocated/userGroups/userGroups_schema.json
copy to
pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/userGroupsDim_schema.json
index c4772df7a2..98aa39ac06 100644
---
a/pinot-tools/src/main/resources/examples/batch/colocated/userGroups/userGroups_schema.json
+++
b/pinot-tools/src/main/resources/examples/batch/lookup/userGroupsDim/userGroupsDim_schema.json
@@ -1,6 +1,5 @@
{
- "metricFieldSpecs": [
- ],
+ "schemaName": "userGroupsDim",
"dimensionFieldSpecs": [
{
"dataType": "STRING",
@@ -11,5 +10,7 @@
"name": "userUUID"
}
],
- "schemaName": "userGroups"
+ "primaryKeyColumns": [
+ "userUUID"
+ ]
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
