This is an automated email from the ASF dual-hosted git repository.
fjy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new f09e718 Implement MapVirtualColumn.makeDimensionSelector properly
(#6396)
f09e718 is described below
commit f09e718c6813c866591501236285166655c85bf8
Author: Jihoon Son <[email protected]>
AuthorDate: Sat Sep 29 14:13:05 2018 -0700
Implement MapVirtualColumn.makeDimensionSelector properly (#6396)
* Implement MapVirtualColumn.makeDimensionSelector properly
* address comments
---
.../MapTypeMapVirtualColumnDimensionSelector.java | 142 +++++++++++++++
.../org/apache/druid/segment/MapVirtualColumn.java | 60 ++-----
.../segment/MapVirtualColumnDimensionSelector.java | 75 ++++++++
.../segment/MapVirtualColumnValueSelector.java | 68 +++++++
...tringTypeMapVirtualColumnDimensionSelector.java | 197 +++++++++++++++++++++
.../druid/segment/MapVirtualColumnGroupByTest.java | 174 ++++++++++++++++++
...mnTest.java => MapVirtualColumnSelectTest.java} | 28 +--
.../druid/segment/MapVirtualColumnTestBase.java | 81 +++++++++
.../druid/segment/MapVirtualColumnTopNTest.java | 145 +++++++++++++++
.../query/groupby/GroupByQueryMergeBufferTest.java | 1 -
.../java/org/apache/druid/segment/TestIndex.java | 6 +-
11 files changed, 905 insertions(+), 72 deletions(-)
diff --git
a/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapTypeMapVirtualColumnDimensionSelector.java
b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapTypeMapVirtualColumnDimensionSelector.java
new file mode 100644
index 0000000..45d20cb
--- /dev/null
+++
b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapTypeMapVirtualColumnDimensionSelector.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.druid.segment;
+
+import com.google.common.base.Predicate;
+import org.apache.druid.query.filter.ValueMatcher;
+import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
+import org.apache.druid.segment.data.IndexedInts;
+
+import javax.annotation.Nullable;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/**
+ * {@link DimensionSelector} for {@link Map} type {@link MapVirtualColumn}.
This dimensionSelector only supports
+ * {@link #getObject()} currently.
+ */
+final class MapTypeMapVirtualColumnDimensionSelector extends
MapVirtualColumnDimensionSelector
+{
+ MapTypeMapVirtualColumnDimensionSelector(
+ DimensionSelector keySelector,
+ DimensionSelector valueSelector
+ )
+ {
+ super(keySelector, valueSelector);
+ }
+
+ @Override
+ public IndexedInts getRow()
+ {
+ throw new UnsupportedOperationException("Map column doesn't support
getRow()");
+ }
+
+ @Override
+ public ValueMatcher makeValueMatcher(@Nullable String value)
+ {
+ return new ValueMatcher()
+ {
+ @Override
+ public boolean matches()
+ {
+ // Map column doesn't match with any string
+ return false;
+ }
+
+ @Override
+ public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+ {
+
+ }
+ };
+ }
+
+ @Override
+ public ValueMatcher makeValueMatcher(Predicate<String> predicate)
+ {
+ return new ValueMatcher()
+ {
+ @Override
+ public boolean matches()
+ {
+ return false;
+ }
+
+ @Override
+ public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+ {
+
+ }
+ };
+ }
+
+ @Override
+ public int getValueCardinality()
+ {
+ return CARDINALITY_UNKNOWN;
+ }
+
+ @Nullable
+ @Override
+ public String lookupName(int id)
+ {
+ throw new UnsupportedOperationException("Map column doesn't support
lookupName()");
+ }
+
+ @Override
+ public boolean nameLookupPossibleInAdvance()
+ {
+ return false;
+ }
+
+ @Nullable
+ @Override
+ public IdLookup idLookup()
+ {
+ throw new UnsupportedOperationException("Map column doesn't support
idLookup()");
+ }
+
+ @Override
+ public Object getObject()
+ {
+ final DimensionSelector keySelector = getKeySelector();
+ final DimensionSelector valueSelector = getValueSelector();
+
+ final IndexedInts keyIndices = keySelector.getRow();
+ final IndexedInts valueIndices = valueSelector.getRow();
+
+ final int limit = Math.min(keyIndices.size(), valueIndices.size());
+ return IntStream
+ .range(0, limit)
+ .boxed()
+ .collect(
+ Collectors.toMap(
+ i -> keySelector.lookupName(keyIndices.get(i)),
+ i -> valueSelector.lookupName(valueIndices.get(i))
+ )
+ );
+ }
+
+ @Override
+ public Class classOfObject()
+ {
+ return Map.class;
+ }
+}
diff --git
a/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumn.java
b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumn.java
index fdded43..7451398 100644
---
a/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumn.java
+++
b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumn.java
@@ -24,12 +24,10 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
-import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.query.filter.DimFilterUtils;
-import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ColumnCapabilitiesImpl;
import org.apache.druid.segment.column.ValueType;
@@ -69,8 +67,16 @@ public class MapVirtualColumn implements VirtualColumn
@Override
public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec,
ColumnSelectorFactory factory)
{
- // Could probably do something useful here if the column name is
dot-style. But for now just return nothing.
- return
dimensionSpec.decorate(DimensionSelectorUtils.constantSelector(null,
dimensionSpec.getExtractionFn()));
+ final DimensionSelector keySelector =
factory.makeDimensionSelector(DefaultDimensionSpec.of(keyDimension));
+ final DimensionSelector valueSelector =
factory.makeDimensionSelector(DefaultDimensionSpec.of(valueDimension));
+ final String subColumnName =
VirtualColumns.splitColumnName(dimensionSpec.getDimension()).rhs;
+ if (subColumnName == null) {
+ return dimensionSpec.decorate(new
MapTypeMapVirtualColumnDimensionSelector(keySelector, valueSelector));
+ } else {
+ return dimensionSpec.decorate(
+ new StringTypeMapVirtualColumnDimensionSelector(keySelector,
valueSelector, subColumnName)
+ );
+ }
}
@Override
@@ -164,52 +170,6 @@ public class MapVirtualColumn implements VirtualColumn
}
}
- private abstract static class MapVirtualColumnValueSelector<T> implements
ColumnValueSelector<T>
- {
- final DimensionSelector keySelector;
- final DimensionSelector valueSelector;
-
- private MapVirtualColumnValueSelector(DimensionSelector keySelector,
DimensionSelector valueSelector)
- {
- this.keySelector = keySelector;
- this.valueSelector = valueSelector;
- }
-
- @Override
- public double getDouble()
- {
- assert NullHandling.replaceWithDefault();
- return 0.0;
- }
-
- @Override
- public float getFloat()
- {
- assert NullHandling.replaceWithDefault();
- return 0.0f;
- }
-
- @Override
- public long getLong()
- {
- assert NullHandling.replaceWithDefault();
- return 0L;
- }
-
- @Override
- public boolean isNull()
- {
- return false;
- }
-
- @Override
- public void inspectRuntimeShape(RuntimeShapeInspector inspector)
- {
- inspector.visit("keySelector", keySelector);
- inspector.visit("valueSelector", valueSelector);
- }
- }
-
@Override
public ColumnCapabilities capabilities(String columnName)
{
diff --git
a/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumnDimensionSelector.java
b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumnDimensionSelector.java
new file mode 100644
index 0000000..2c56df4
--- /dev/null
+++
b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumnDimensionSelector.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.druid.segment;
+
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
+
+abstract class MapVirtualColumnDimensionSelector implements DimensionSelector
+{
+ private final DimensionSelector keySelector;
+ private final DimensionSelector valueSelector;
+
+ MapVirtualColumnDimensionSelector(
+ DimensionSelector keySelector,
+ DimensionSelector valueSelector
+ )
+ {
+ this.keySelector = keySelector;
+ this.valueSelector = valueSelector;
+ }
+
+ protected DimensionSelector getKeySelector()
+ {
+ return keySelector;
+ }
+
+ protected DimensionSelector getValueSelector()
+ {
+ return valueSelector;
+ }
+
+ @Override
+ public double getDouble()
+ {
+ assert NullHandling.replaceWithDefault();
+ return 0.0;
+ }
+
+ @Override
+ public float getFloat()
+ {
+ assert NullHandling.replaceWithDefault();
+ return 0.0f;
+ }
+
+ @Override
+ public long getLong()
+ {
+ assert NullHandling.replaceWithDefault();
+ return 0L;
+ }
+
+ @Override
+ public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+ {
+ inspector.visit("keySelector", keySelector);
+ inspector.visit("valueSelector", valueSelector);
+ }
+}
diff --git
a/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumnValueSelector.java
b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumnValueSelector.java
new file mode 100644
index 0000000..ba25bb0
--- /dev/null
+++
b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumnValueSelector.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.druid.segment;
+
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
+
+abstract class MapVirtualColumnValueSelector<T> implements
ColumnValueSelector<T>
+{
+ private final DimensionSelector keySelector;
+ private final DimensionSelector valueSelector;
+
+ MapVirtualColumnValueSelector(DimensionSelector keySelector,
DimensionSelector valueSelector)
+ {
+ this.keySelector = keySelector;
+ this.valueSelector = valueSelector;
+ }
+
+ @Override
+ public double getDouble()
+ {
+ assert NullHandling.replaceWithDefault();
+ return 0.0;
+ }
+
+ @Override
+ public float getFloat()
+ {
+ assert NullHandling.replaceWithDefault();
+ return 0.0f;
+ }
+
+ @Override
+ public long getLong()
+ {
+ assert NullHandling.replaceWithDefault();
+ return 0L;
+ }
+
+ @Override
+ public boolean isNull()
+ {
+ return false;
+ }
+
+ @Override
+ public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+ {
+ inspector.visit("keySelector", keySelector);
+ inspector.visit("valueSelector", valueSelector);
+ }
+}
diff --git
a/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/StringTypeMapVirtualColumnDimensionSelector.java
b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/StringTypeMapVirtualColumnDimensionSelector.java
new file mode 100644
index 0000000..9b15a29
--- /dev/null
+++
b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/StringTypeMapVirtualColumnDimensionSelector.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.druid.segment;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import org.apache.druid.query.filter.ValueMatcher;
+import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
+import org.apache.druid.segment.data.IndexedInts;
+import org.apache.druid.segment.data.SingleIndexedInt;
+import org.apache.druid.segment.data.ZeroIndexedInts;
+
+import javax.annotation.Nullable;
+import java.util.Objects;
+import java.util.stream.IntStream;
+
+/**
+ * {@link DimensionSelector} for String type {@link MapVirtualColumn}. The
performance has not considered yet and so
+ * it may need to be improved later.
+ */
+final class StringTypeMapVirtualColumnDimensionSelector extends
MapVirtualColumnDimensionSelector
+{
+ private final String subColumnName;
+ private final SingleIndexedInt indexedInt = new SingleIndexedInt();
+
+ StringTypeMapVirtualColumnDimensionSelector(
+ DimensionSelector keySelector,
+ DimensionSelector valueSelector,
+ String subColumnName
+ )
+ {
+ super(keySelector, valueSelector);
+ this.subColumnName = Preconditions.checkNotNull(subColumnName,
"subColumnName");
+ }
+
+ @Override
+ public IndexedInts getRow()
+ {
+ final int valueIndex = findValueIndicesIndexForSubColumn();
+ if (valueIndex < 0) {
+ return ZeroIndexedInts.instance();
+ } else {
+ indexedInt.setValue(valueIndex);
+ return indexedInt;
+ }
+ }
+
+ @Override
+ public ValueMatcher makeValueMatcher(@Nullable String value)
+ {
+ return new ValueMatcher()
+ {
+ @Override
+ public boolean matches()
+ {
+ return Objects.equals(value, getObject());
+ }
+
+ @Override
+ public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+ {
+ inspector.visit("keySelector", getKeySelector());
+ inspector.visit("valueSelector", getValueSelector());
+ inspector.visit("subColumnName", subColumnName);
+ }
+ };
+ }
+
+ @Override
+ public ValueMatcher makeValueMatcher(Predicate<String> predicate)
+ {
+ return new ValueMatcher()
+ {
+ @Override
+ public boolean matches()
+ {
+ return predicate.apply((String) getObject());
+ }
+
+ @Override
+ public void inspectRuntimeShape(RuntimeShapeInspector inspector)
+ {
+ inspector.visit("keySelector", getKeySelector());
+ inspector.visit("valueSelector", getValueSelector());
+ inspector.visit("subColumnName", subColumnName);
+ }
+ };
+ }
+
+ @Override
+ public int getValueCardinality()
+ {
+ // To get the value cardinarlity, we need to first check all keys and
values to find valid pairs, and then find the
+ // number of distinct values among them.
+ return CARDINALITY_UNKNOWN;
+ }
+
+ @Nullable
+ @Override
+ public String lookupName(int id)
+ {
+ final int valueIndex = findValueIndicesIndexForSubColumn();
+
+ if (valueIndex == id) {
+ return getValueSelector().lookupName(id);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public boolean nameLookupPossibleInAdvance()
+ {
+ return false;
+ }
+
+ @Nullable
+ @Override
+ public IdLookup idLookup()
+ {
+ final DimensionSelector valueSelector = getValueSelector();
+ final IdLookup valueLookup = valueSelector.idLookup();
+
+ if (valueLookup != null) {
+ final int valueIndex = findValueIndicesIndexForSubColumn();
+ return name -> {
+ final int candidate = valueLookup.lookupId(name);
+ if (candidate == valueIndex) {
+ return candidate;
+ }
+ return -1;
+ };
+ } else {
+ return null;
+ }
+ }
+
+ @Nullable
+ @Override
+ public Object getObject()
+ {
+ final int valueIndex = findValueIndicesIndexForSubColumn();
+
+ if (valueIndex < 0) {
+ return null;
+ } else {
+ final DimensionSelector valueSelector = getValueSelector();
+ final IndexedInts valueIndices = valueSelector.getRow();
+ return valueSelector.lookupName(valueIndices.get(valueIndex));
+ }
+ }
+
+ /**
+ * Find the index of valueIndices which is {@link IndexedInts} returned from
{@link #getValueSelector()#getRow()}
+ * corresponding to the {@link #subColumnName}.
+ *
+ * @return index for valueIndices if found. -1 otherwise.
+ */
+ private int findValueIndicesIndexForSubColumn()
+ {
+ final DimensionSelector keySelector = getKeySelector();
+ final DimensionSelector valueSelector = getValueSelector();
+
+ final IndexedInts keyIndices = keySelector.getRow();
+ final IndexedInts valueIndices = valueSelector.getRow();
+
+ final int limit = Math.min(keyIndices.size(), valueIndices.size());
+
+ return IntStream
+ .range(0, limit)
+ .filter(i ->
subColumnName.equals(keySelector.lookupName(keyIndices.get(i)))) //
subColumnName is never null
+ .findAny()
+ .orElse(-1);
+ }
+
+ @Override
+ public Class classOfObject()
+ {
+ return String.class;
+ }
+}
diff --git
a/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnGroupByTest.java
b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnGroupByTest.java
new file mode 100644
index 0000000..43d6809
--- /dev/null
+++
b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnGroupByTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.druid.segment;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.druid.collections.DefaultBlockingPool;
+import org.apache.druid.collections.StupidPool;
+import org.apache.druid.data.input.MapBasedRow;
+import org.apache.druid.data.input.Row;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.query.DruidProcessingConfig;
+import org.apache.druid.query.QueryPlus;
+import org.apache.druid.query.QueryRunner;
+import org.apache.druid.query.QueryRunnerTestHelper;
+import org.apache.druid.query.TableDataSource;
+import org.apache.druid.query.aggregation.CountAggregatorFactory;
+import org.apache.druid.query.dimension.DefaultDimensionSpec;
+import org.apache.druid.query.groupby.GroupByQuery;
+import org.apache.druid.query.groupby.GroupByQueryConfig;
+import org.apache.druid.query.groupby.GroupByQueryQueryToolChest;
+import org.apache.druid.query.groupby.GroupByQueryRunnerFactory;
+import org.apache.druid.query.groupby.strategy.GroupByStrategySelector;
+import org.apache.druid.query.groupby.strategy.GroupByStrategyV2;
+import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
+import org.apache.druid.segment.incremental.IncrementalIndex;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.List;
+
+public class MapVirtualColumnGroupByTest
+{
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ private QueryRunner<Row> runner;
+
+ @Before
+ public void setup() throws IOException
+ {
+ final IncrementalIndex incrementalIndex =
MapVirtualColumnTestBase.generateIndex();
+
+ final GroupByStrategySelector strategySelector = new
GroupByStrategySelector(
+ GroupByQueryConfig::new,
+ null,
+ new GroupByStrategyV2(
+ new DruidProcessingConfig()
+ {
+ @Override
+ public String getFormatString()
+ {
+ return null;
+ }
+
+ @Override
+ public int intermediateComputeSizeBytes()
+ {
+ return 10 * 1024 * 1024;
+ }
+
+ @Override
+ public int getNumMergeBuffers()
+ {
+ return 1;
+ }
+
+ @Override
+ public int getNumThreads()
+ {
+ return 1;
+ }
+ },
+ GroupByQueryConfig::new,
+ new StupidPool<>("map-virtual-column-groupby-test", () ->
ByteBuffer.allocate(1024)),
+ new DefaultBlockingPool<>(() -> ByteBuffer.allocate(1024), 1),
+ new DefaultObjectMapper(),
+ QueryRunnerTestHelper.NOOP_QUERYWATCHER
+ )
+ );
+
+ final GroupByQueryRunnerFactory factory = new GroupByQueryRunnerFactory(
+ strategySelector,
+ new GroupByQueryQueryToolChest(
+ strategySelector,
+ QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator()
+ )
+ );
+
+ runner = QueryRunnerTestHelper.makeQueryRunner(
+ factory,
+ "index",
+ new IncrementalIndexSegment(incrementalIndex, "index"),
+ "incremental"
+ );
+ }
+
+ @Test
+ public void testWithMapColumn()
+ {
+ final GroupByQuery query = new GroupByQuery(
+ new TableDataSource(QueryRunnerTestHelper.dataSource),
+ new
MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("2011/2012"))),
+ VirtualColumns.create(ImmutableList.of(new MapVirtualColumn("keys",
"values", "params"))),
+ null,
+ Granularities.ALL,
+ ImmutableList.of(new DefaultDimensionSpec("params", "params")),
+ ImmutableList.of(new CountAggregatorFactory("count")),
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ expectedException.expect(UnsupportedOperationException.class);
+ expectedException.expectMessage("Map column doesn't support getRow()");
+ runner.run(QueryPlus.wrap(query), new HashMap<>()).toList();
+ }
+
+ @Test
+ public void testWithSubColumn()
+ {
+ final GroupByQuery query = new GroupByQuery(
+ new TableDataSource(QueryRunnerTestHelper.dataSource),
+ new
MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("2011/2012"))),
+ VirtualColumns.create(ImmutableList.of(new MapVirtualColumn("keys",
"values", "params"))),
+ null,
+ Granularities.ALL,
+ ImmutableList.of(new DefaultDimensionSpec("params.key3",
"params.key3")),
+ ImmutableList.of(new CountAggregatorFactory("count")),
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ final List<Row> result = runner.run(QueryPlus.wrap(query), new
HashMap<>()).toList();
+ final List<Row> expected = ImmutableList.of(
+ new MapBasedRow(
+ DateTimes.of("2011-01-12T00:00:00.000Z"),
+ MapVirtualColumnTestBase.mapOf("count", 1L, "params.key3",
"value3")
+ ),
+ new MapBasedRow(DateTimes.of("2011-01-12T00:00:00.000Z"),
MapVirtualColumnTestBase.mapOf("count", 2L))
+ );
+
+ Assert.assertEquals(expected, result);
+ }
+}
diff --git
a/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTest.java
b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnSelectTest.java
similarity index 91%
rename from
extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTest.java
rename to
extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnSelectTest.java
index 7f7c438..9ff2f94 100644
---
a/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTest.java
+++
b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnSelectTest.java
@@ -23,7 +23,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
import com.google.common.io.CharSource;
import org.apache.druid.data.input.impl.DelimitedParseSpec;
import org.apache.druid.data.input.impl.DimensionsSpec;
@@ -60,7 +59,7 @@ import java.util.Map;
/**
*/
@RunWith(Parameterized.class)
-public class MapVirtualColumnTest
+public class MapVirtualColumnSelectTest
{
@Parameterized.Parameters
public static Iterable<Object[]> constructorFeeder() throws IOException
@@ -104,7 +103,7 @@ public class MapVirtualColumnTest
"2011-01-12T00:00:00.000Z\tc\tkey1,key5\tvalue1,value5,value9\n"
);
- IncrementalIndex index1 = TestIndex.loadIncrementalIndex(index, input,
parser);
+ IncrementalIndex index1 = TestIndex.loadIncrementalIndex(() -> index,
input, parser);
QueryableIndex index2 = TestIndex.persistRealtimeAndLoadMMapped(index1);
return QueryRunnerTestHelper.transformToConstructionFeeder(
@@ -127,7 +126,7 @@ public class MapVirtualColumnTest
private final QueryRunner runner;
- public MapVirtualColumnTest(QueryRunner runner)
+ public MapVirtualColumnSelectTest(QueryRunner runner)
{
this.runner = runner;
}
@@ -159,26 +158,26 @@ public class MapVirtualColumnTest
Druids.SelectQueryBuilder builder = testBuilder();
List<Map> expectedResults = Arrays.asList(
- mapOf(
+ MapVirtualColumnTestBase.mapOf(
"dim", "a",
"params.key1", "value1",
"params.key3", "value3",
"params.key5", null,
- "params", mapOf("key1", "value1", "key2", "value2", "key3",
"value3")
+ "params", MapVirtualColumnTestBase.mapOf("key1", "value1", "key2",
"value2", "key3", "value3")
),
- mapOf(
+ MapVirtualColumnTestBase.mapOf(
"dim", "b",
"params.key1", null,
"params.key3", null,
"params.key5", null,
- "params", mapOf("key4", "value4")
+ "params", MapVirtualColumnTestBase.mapOf("key4", "value4")
),
- mapOf(
+ MapVirtualColumnTestBase.mapOf(
"dim", "c",
"params.key1", "value1",
"params.key3", null,
"params.key5", "value5",
- "params", mapOf("key1", "value1", "key5", "value5")
+ "params", MapVirtualColumnTestBase.mapOf("key1", "value1", "key5",
"value5")
)
);
List<VirtualColumn> virtualColumns = Collections.singletonList(new
MapVirtualColumn("keys", "values", "params"));
@@ -189,15 +188,6 @@ public class MapVirtualColumnTest
checkSelectQuery(selectQuery, expectedResults);
}
- private Map mapOf(Object... elements)
- {
- Map map = Maps.newHashMap();
- for (int i = 0; i < elements.length; i += 2) {
- map.put(elements[i], elements[i + 1]);
- }
- return map;
- }
-
private void checkSelectQuery(SelectQuery searchQuery, List<Map> expected)
{
List<Result<SelectResultValue>> results =
runner.run(QueryPlus.wrap(searchQuery), ImmutableMap.of()).toList();
diff --git
a/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTestBase.java
b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTestBase.java
new file mode 100644
index 0000000..7736ee2
--- /dev/null
+++
b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTestBase.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.druid.segment;
+
+import com.google.common.io.CharSource;
+import org.apache.druid.data.input.impl.DelimitedParseSpec;
+import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.data.input.impl.StringInputRowParser;
+import org.apache.druid.data.input.impl.TimestampSpec;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.segment.incremental.IncrementalIndex;
+import org.apache.druid.segment.incremental.IncrementalIndexSchema;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+public class MapVirtualColumnTestBase
+{
+ static IncrementalIndex generateIndex() throws IOException
+ {
+ final CharSource input = CharSource.wrap(
+ "2011-01-12T00:00:00.000Z\ta\tkey1,key2,key3\tvalue1,value2,value3\n" +
+ "2011-01-12T00:00:00.000Z\tb\tkey4,key5,key6\tvalue4\n" +
+ "2011-01-12T00:00:00.000Z\tc\tkey1,key5\tvalue1,value5,value9\n"
+ );
+
+ final StringInputRowParser parser = new StringInputRowParser(
+ new DelimitedParseSpec(
+ new TimestampSpec("ts", "auto", null),
+ new
DimensionsSpec(DimensionsSpec.getDefaultSchemas(Arrays.asList("dim", "keys",
"values")), null, null),
+ "\t",
+ ",",
+ Arrays.asList("ts", "dim", "keys", "values"),
+ false,
+ 0
+ ),
+ "utf8"
+ );
+
+ final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder()
+ .withMinTimestamp(DateTimes.of("2011-01-12T00:00:00.000Z").getMillis())
+ .build();
+
+ return TestIndex.loadIncrementalIndex(
+ () -> new IncrementalIndex.Builder()
+ .setIndexSchema(schema)
+ .setMaxRowCount(10000)
+ .buildOnheap(),
+ input,
+ parser
+ );
+ }
+
+ static <K, V> Map<K, V> mapOf(Object... elements)
+ {
+ final Map<K, V> map = new HashMap<>();
+ for (int i = 0; i < elements.length; i += 2) {
+ //noinspection unchecked
+ map.put((K) elements[i], (V) elements[i + 1]);
+ }
+ return map;
+ }
+}
diff --git
a/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTopNTest.java
b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTopNTest.java
new file mode 100644
index 0000000..e223681
--- /dev/null
+++
b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTopNTest.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.druid.segment;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.druid.collections.StupidPool;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.query.QueryPlus;
+import org.apache.druid.query.QueryRunner;
+import org.apache.druid.query.QueryRunnerTestHelper;
+import org.apache.druid.query.Result;
+import org.apache.druid.query.TableDataSource;
+import org.apache.druid.query.aggregation.CountAggregatorFactory;
+import org.apache.druid.query.dimension.DefaultDimensionSpec;
+import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
+import org.apache.druid.query.topn.DimensionAndMetricValueExtractor;
+import org.apache.druid.query.topn.NumericTopNMetricSpec;
+import org.apache.druid.query.topn.TopNQuery;
+import org.apache.druid.query.topn.TopNQueryConfig;
+import org.apache.druid.query.topn.TopNQueryQueryToolChest;
+import org.apache.druid.query.topn.TopNQueryRunnerFactory;
+import org.apache.druid.query.topn.TopNResultValue;
+import org.apache.druid.segment.incremental.IncrementalIndex;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
+public class MapVirtualColumnTopNTest
+{
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ private QueryRunner<Result<TopNResultValue>> runner;
+
+ @Before
+ public void setup() throws IOException
+ {
+ final IncrementalIndex incrementalIndex =
MapVirtualColumnTestBase.generateIndex();
+
+ final TopNQueryRunnerFactory factory = new TopNQueryRunnerFactory(
+ new StupidPool<>("map-virtual-column-test", () ->
ByteBuffer.allocate(1024)),
+ new TopNQueryQueryToolChest(
+ new TopNQueryConfig(),
+ QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator()
+ ),
+ QueryRunnerTestHelper.NOOP_QUERYWATCHER
+ );
+
+ runner = QueryRunnerTestHelper.makeQueryRunner(
+ factory,
+ "index1",
+ new IncrementalIndexSegment(incrementalIndex, "index1"),
+ "incremental"
+ );
+ }
+
+ @Test
+ public void testWithMapColumn()
+ {
+ final TopNQuery query = new TopNQuery(
+ new TableDataSource(QueryRunnerTestHelper.dataSource),
+ VirtualColumns.create(
+ ImmutableList.of(
+ new MapVirtualColumn("keys", "values", "params")
+ )
+ ),
+ new DefaultDimensionSpec("params", "params"), // params is the map type
+ new NumericTopNMetricSpec("count"),
+ 1,
+ new
MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("2011/2012"))),
+ null,
+ Granularities.ALL,
+ ImmutableList.of(new CountAggregatorFactory("count")),
+ null,
+ null
+ );
+
+ expectedException.expect(UnsupportedOperationException.class);
+ expectedException.expectMessage("Map column doesn't support getRow()");
+ runner.run(QueryPlus.wrap(query), new HashMap<>()).toList();
+ }
+
+ @Test
+ public void testWithSubColumn()
+ {
+ final TopNQuery query = new TopNQuery(
+ new TableDataSource(QueryRunnerTestHelper.dataSource),
+ VirtualColumns.create(
+ ImmutableList.of(
+ new MapVirtualColumn("keys", "values", "params")
+ )
+ ),
+ new DefaultDimensionSpec("params.key3", "params.key3"), // params.key3
is string
+ new NumericTopNMetricSpec("count"),
+ 2,
+ new
MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("2011/2012"))),
+ null,
+ Granularities.ALL,
+ ImmutableList.of(new CountAggregatorFactory("count")),
+ null,
+ null
+ );
+
+ final List<Result<TopNResultValue>> result =
runner.run(QueryPlus.wrap(query), new HashMap<>()).toList();
+ final List<Result<TopNResultValue>> expected = Collections.singletonList(
+ new Result<>(
+ DateTimes.of("2011-01-12T00:00:00.000Z"),
+ new TopNResultValue(
+ ImmutableList.of(
+ new
DimensionAndMetricValueExtractor(MapVirtualColumnTestBase.mapOf("count", 2L,
"params.key3", null)),
+ new
DimensionAndMetricValueExtractor(MapVirtualColumnTestBase.mapOf("count", 1L,
"params.key3", "value3"))
+ )
+ )
+ )
+ );
+
+ Assert.assertEquals(expected, result);
+ }
+}
diff --git
a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryMergeBufferTest.java
b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryMergeBufferTest.java
index 537ecb3..abcf40e 100644
---
a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryMergeBufferTest.java
+++
b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryMergeBufferTest.java
@@ -104,7 +104,6 @@ public class GroupByQueryMergeBufferTest
public static final DruidProcessingConfig PROCESSING_CONFIG = new
DruidProcessingConfig()
{
-
@Override
public String getFormatString()
{
diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java
b/processing/src/test/java/org/apache/druid/segment/TestIndex.java
index 201354a..dd865fe 100644
--- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java
+++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java
@@ -19,6 +19,7 @@
package org.apache.druid.segment;
+import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
import com.google.common.io.CharSource;
import com.google.common.io.LineProcessor;
@@ -315,15 +316,16 @@ public class TestIndex
),
"utf8"
);
- return loadIncrementalIndex(retVal, source, parser);
+ return loadIncrementalIndex(() -> retVal, source, parser);
}
public static IncrementalIndex loadIncrementalIndex(
- final IncrementalIndex retVal,
+ final Supplier<IncrementalIndex> indexSupplier,
final CharSource source,
final StringInputRowParser parser
) throws IOException
{
+ final IncrementalIndex retVal = indexSupplier.get();
final AtomicLong startTime = new AtomicLong();
int lineCount = source.readLines(
new LineProcessor<Integer>()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
