fjy closed pull request #6396: Implement MapVirtualColumn.makeDimensionSelector properly URL: https://github.com/apache/incubator-druid/pull/6396
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapTypeMapVirtualColumnDimensionSelector.java b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapTypeMapVirtualColumnDimensionSelector.java new file mode 100644 index 00000000000..45d20cb8d74 --- /dev/null +++ b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapTypeMapVirtualColumnDimensionSelector.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.druid.segment; + +import com.google.common.base.Predicate; +import org.apache.druid.query.filter.ValueMatcher; +import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import org.apache.druid.segment.data.IndexedInts; + +import javax.annotation.Nullable; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * {@link DimensionSelector} for {@link Map} type {@link MapVirtualColumn}. This dimensionSelector only supports + * {@link #getObject()} currently. + */ +final class MapTypeMapVirtualColumnDimensionSelector extends MapVirtualColumnDimensionSelector +{ + MapTypeMapVirtualColumnDimensionSelector( + DimensionSelector keySelector, + DimensionSelector valueSelector + ) + { + super(keySelector, valueSelector); + } + + @Override + public IndexedInts getRow() + { + throw new UnsupportedOperationException("Map column doesn't support getRow()"); + } + + @Override + public ValueMatcher makeValueMatcher(@Nullable String value) + { + return new ValueMatcher() + { + @Override + public boolean matches() + { + // Map column doesn't match with any string + return false; + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + + } + }; + } + + @Override + public ValueMatcher makeValueMatcher(Predicate<String> predicate) + { + return new ValueMatcher() + { + @Override + public boolean matches() + { + return false; + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + + } + }; + } + + @Override + public int getValueCardinality() + { + return CARDINALITY_UNKNOWN; + } + + @Nullable + @Override + public String lookupName(int id) + { + throw new UnsupportedOperationException("Map column doesn't support lookupName()"); + } + + @Override + public boolean nameLookupPossibleInAdvance() + { + return false; + } + + @Nullable + @Override + public IdLookup idLookup() + { + throw new UnsupportedOperationException("Map column doesn't support idLookup()"); + } + + @Override + public Object getObject() + { + final DimensionSelector keySelector = getKeySelector(); + final DimensionSelector valueSelector = getValueSelector(); + + final IndexedInts keyIndices = keySelector.getRow(); + final IndexedInts valueIndices = valueSelector.getRow(); + + final int limit = Math.min(keyIndices.size(), valueIndices.size()); + return IntStream + .range(0, limit) + .boxed() + .collect( + Collectors.toMap( + i -> keySelector.lookupName(keyIndices.get(i)), + i -> valueSelector.lookupName(valueIndices.get(i)) + ) + ); + } + + @Override + public Class classOfObject() + { + return Map.class; + } +} diff --git a/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumn.java b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumn.java index fdded43b053..7451398baa1 100644 --- a/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumn.java +++ b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumn.java @@ -24,12 +24,10 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; -import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.query.filter.DimFilterUtils; -import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ColumnCapabilitiesImpl; import org.apache.druid.segment.column.ValueType; @@ -69,8 +67,16 @@ public MapVirtualColumn( @Override public DimensionSelector makeDimensionSelector(DimensionSpec dimensionSpec, ColumnSelectorFactory factory) { - // Could probably do something useful here if the column name is dot-style. But for now just return nothing. - return dimensionSpec.decorate(DimensionSelectorUtils.constantSelector(null, dimensionSpec.getExtractionFn())); + final DimensionSelector keySelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(keyDimension)); + final DimensionSelector valueSelector = factory.makeDimensionSelector(DefaultDimensionSpec.of(valueDimension)); + final String subColumnName = VirtualColumns.splitColumnName(dimensionSpec.getDimension()).rhs; + if (subColumnName == null) { + return dimensionSpec.decorate(new MapTypeMapVirtualColumnDimensionSelector(keySelector, valueSelector)); + } else { + return dimensionSpec.decorate( + new StringTypeMapVirtualColumnDimensionSelector(keySelector, valueSelector, subColumnName) + ); + } } @Override @@ -164,52 +170,6 @@ public String getObject() } } - private abstract static class MapVirtualColumnValueSelector<T> implements ColumnValueSelector<T> - { - final DimensionSelector keySelector; - final DimensionSelector valueSelector; - - private MapVirtualColumnValueSelector(DimensionSelector keySelector, DimensionSelector valueSelector) - { - this.keySelector = keySelector; - this.valueSelector = valueSelector; - } - - @Override - public double getDouble() - { - assert NullHandling.replaceWithDefault(); - return 0.0; - } - - @Override - public float getFloat() - { - assert NullHandling.replaceWithDefault(); - return 0.0f; - } - - @Override - public long getLong() - { - assert NullHandling.replaceWithDefault(); - return 0L; - } - - @Override - public boolean isNull() - { - return false; - } - - @Override - public void inspectRuntimeShape(RuntimeShapeInspector inspector) - { - inspector.visit("keySelector", keySelector); - inspector.visit("valueSelector", valueSelector); - } - } - @Override public ColumnCapabilities capabilities(String columnName) { diff --git a/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumnDimensionSelector.java b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumnDimensionSelector.java new file mode 100644 index 00000000000..2c56df438eb --- /dev/null +++ b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumnDimensionSelector.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.druid.segment; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; + +abstract class MapVirtualColumnDimensionSelector implements DimensionSelector +{ + private final DimensionSelector keySelector; + private final DimensionSelector valueSelector; + + MapVirtualColumnDimensionSelector( + DimensionSelector keySelector, + DimensionSelector valueSelector + ) + { + this.keySelector = keySelector; + this.valueSelector = valueSelector; + } + + protected DimensionSelector getKeySelector() + { + return keySelector; + } + + protected DimensionSelector getValueSelector() + { + return valueSelector; + } + + @Override + public double getDouble() + { + assert NullHandling.replaceWithDefault(); + return 0.0; + } + + @Override + public float getFloat() + { + assert NullHandling.replaceWithDefault(); + return 0.0f; + } + + @Override + public long getLong() + { + assert NullHandling.replaceWithDefault(); + return 0L; + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("keySelector", keySelector); + inspector.visit("valueSelector", valueSelector); + } +} diff --git a/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumnValueSelector.java b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumnValueSelector.java new file mode 100644 index 00000000000..ba25bb072ea --- /dev/null +++ b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/MapVirtualColumnValueSelector.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.druid.segment; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; + +abstract class MapVirtualColumnValueSelector<T> implements ColumnValueSelector<T> +{ + private final DimensionSelector keySelector; + private final DimensionSelector valueSelector; + + MapVirtualColumnValueSelector(DimensionSelector keySelector, DimensionSelector valueSelector) + { + this.keySelector = keySelector; + this.valueSelector = valueSelector; + } + + @Override + public double getDouble() + { + assert NullHandling.replaceWithDefault(); + return 0.0; + } + + @Override + public float getFloat() + { + assert NullHandling.replaceWithDefault(); + return 0.0f; + } + + @Override + public long getLong() + { + assert NullHandling.replaceWithDefault(); + return 0L; + } + + @Override + public boolean isNull() + { + return false; + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("keySelector", keySelector); + inspector.visit("valueSelector", valueSelector); + } +} diff --git a/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/StringTypeMapVirtualColumnDimensionSelector.java b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/StringTypeMapVirtualColumnDimensionSelector.java new file mode 100644 index 00000000000..9b15a29718e --- /dev/null +++ b/extensions-contrib/virtual-columns/src/main/java/org/apache/druid/segment/StringTypeMapVirtualColumnDimensionSelector.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.druid.segment; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import org.apache.druid.query.filter.ValueMatcher; +import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; +import org.apache.druid.segment.data.IndexedInts; +import org.apache.druid.segment.data.SingleIndexedInt; +import org.apache.druid.segment.data.ZeroIndexedInts; + +import javax.annotation.Nullable; +import java.util.Objects; +import java.util.stream.IntStream; + +/** + * {@link DimensionSelector} for String type {@link MapVirtualColumn}. The performance has not considered yet and so + * it may need to be improved later. + */ +final class StringTypeMapVirtualColumnDimensionSelector extends MapVirtualColumnDimensionSelector +{ + private final String subColumnName; + private final SingleIndexedInt indexedInt = new SingleIndexedInt(); + + StringTypeMapVirtualColumnDimensionSelector( + DimensionSelector keySelector, + DimensionSelector valueSelector, + String subColumnName + ) + { + super(keySelector, valueSelector); + this.subColumnName = Preconditions.checkNotNull(subColumnName, "subColumnName"); + } + + @Override + public IndexedInts getRow() + { + final int valueIndex = findValueIndicesIndexForSubColumn(); + if (valueIndex < 0) { + return ZeroIndexedInts.instance(); + } else { + indexedInt.setValue(valueIndex); + return indexedInt; + } + } + + @Override + public ValueMatcher makeValueMatcher(@Nullable String value) + { + return new ValueMatcher() + { + @Override + public boolean matches() + { + return Objects.equals(value, getObject()); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("keySelector", getKeySelector()); + inspector.visit("valueSelector", getValueSelector()); + inspector.visit("subColumnName", subColumnName); + } + }; + } + + @Override + public ValueMatcher makeValueMatcher(Predicate<String> predicate) + { + return new ValueMatcher() + { + @Override + public boolean matches() + { + return predicate.apply((String) getObject()); + } + + @Override + public void inspectRuntimeShape(RuntimeShapeInspector inspector) + { + inspector.visit("keySelector", getKeySelector()); + inspector.visit("valueSelector", getValueSelector()); + inspector.visit("subColumnName", subColumnName); + } + }; + } + + @Override + public int getValueCardinality() + { + // To get the value cardinarlity, we need to first check all keys and values to find valid pairs, and then find the + // number of distinct values among them. + return CARDINALITY_UNKNOWN; + } + + @Nullable + @Override + public String lookupName(int id) + { + final int valueIndex = findValueIndicesIndexForSubColumn(); + + if (valueIndex == id) { + return getValueSelector().lookupName(id); + } else { + return null; + } + } + + @Override + public boolean nameLookupPossibleInAdvance() + { + return false; + } + + @Nullable + @Override + public IdLookup idLookup() + { + final DimensionSelector valueSelector = getValueSelector(); + final IdLookup valueLookup = valueSelector.idLookup(); + + if (valueLookup != null) { + final int valueIndex = findValueIndicesIndexForSubColumn(); + return name -> { + final int candidate = valueLookup.lookupId(name); + if (candidate == valueIndex) { + return candidate; + } + return -1; + }; + } else { + return null; + } + } + + @Nullable + @Override + public Object getObject() + { + final int valueIndex = findValueIndicesIndexForSubColumn(); + + if (valueIndex < 0) { + return null; + } else { + final DimensionSelector valueSelector = getValueSelector(); + final IndexedInts valueIndices = valueSelector.getRow(); + return valueSelector.lookupName(valueIndices.get(valueIndex)); + } + } + + /** + * Find the index of valueIndices which is {@link IndexedInts} returned from {@link #getValueSelector()#getRow()} + * corresponding to the {@link #subColumnName}. + * + * @return index for valueIndices if found. -1 otherwise. + */ + private int findValueIndicesIndexForSubColumn() + { + final DimensionSelector keySelector = getKeySelector(); + final DimensionSelector valueSelector = getValueSelector(); + + final IndexedInts keyIndices = keySelector.getRow(); + final IndexedInts valueIndices = valueSelector.getRow(); + + final int limit = Math.min(keyIndices.size(), valueIndices.size()); + + return IntStream + .range(0, limit) + .filter(i -> subColumnName.equals(keySelector.lookupName(keyIndices.get(i)))) // subColumnName is never null + .findAny() + .orElse(-1); + } + + @Override + public Class classOfObject() + { + return String.class; + } +} diff --git a/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnGroupByTest.java b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnGroupByTest.java new file mode 100644 index 00000000000..43d6809aa29 --- /dev/null +++ b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnGroupByTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.druid.segment; + +import com.google.common.collect.ImmutableList; +import org.apache.druid.collections.DefaultBlockingPool; +import org.apache.druid.collections.StupidPool; +import org.apache.druid.data.input.MapBasedRow; +import org.apache.druid.data.input.Row; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.DruidProcessingConfig; +import org.apache.druid.query.QueryPlus; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryRunnerTestHelper; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.groupby.GroupByQuery; +import org.apache.druid.query.groupby.GroupByQueryConfig; +import org.apache.druid.query.groupby.GroupByQueryQueryToolChest; +import org.apache.druid.query.groupby.GroupByQueryRunnerFactory; +import org.apache.druid.query.groupby.strategy.GroupByStrategySelector; +import org.apache.druid.query.groupby.strategy.GroupByStrategyV2; +import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; +import org.apache.druid.segment.incremental.IncrementalIndex; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.List; + +public class MapVirtualColumnGroupByTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private QueryRunner<Row> runner; + + @Before + public void setup() throws IOException + { + final IncrementalIndex incrementalIndex = MapVirtualColumnTestBase.generateIndex(); + + final GroupByStrategySelector strategySelector = new GroupByStrategySelector( + GroupByQueryConfig::new, + null, + new GroupByStrategyV2( + new DruidProcessingConfig() + { + @Override + public String getFormatString() + { + return null; + } + + @Override + public int intermediateComputeSizeBytes() + { + return 10 * 1024 * 1024; + } + + @Override + public int getNumMergeBuffers() + { + return 1; + } + + @Override + public int getNumThreads() + { + return 1; + } + }, + GroupByQueryConfig::new, + new StupidPool<>("map-virtual-column-groupby-test", () -> ByteBuffer.allocate(1024)), + new DefaultBlockingPool<>(() -> ByteBuffer.allocate(1024), 1), + new DefaultObjectMapper(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ) + ); + + final GroupByQueryRunnerFactory factory = new GroupByQueryRunnerFactory( + strategySelector, + new GroupByQueryQueryToolChest( + strategySelector, + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ) + ); + + runner = QueryRunnerTestHelper.makeQueryRunner( + factory, + "index", + new IncrementalIndexSegment(incrementalIndex, "index"), + "incremental" + ); + } + + @Test + public void testWithMapColumn() + { + final GroupByQuery query = new GroupByQuery( + new TableDataSource(QueryRunnerTestHelper.dataSource), + new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("2011/2012"))), + VirtualColumns.create(ImmutableList.of(new MapVirtualColumn("keys", "values", "params"))), + null, + Granularities.ALL, + ImmutableList.of(new DefaultDimensionSpec("params", "params")), + ImmutableList.of(new CountAggregatorFactory("count")), + null, + null, + null, + null, + null + ); + + expectedException.expect(UnsupportedOperationException.class); + expectedException.expectMessage("Map column doesn't support getRow()"); + runner.run(QueryPlus.wrap(query), new HashMap<>()).toList(); + } + + @Test + public void testWithSubColumn() + { + final GroupByQuery query = new GroupByQuery( + new TableDataSource(QueryRunnerTestHelper.dataSource), + new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("2011/2012"))), + VirtualColumns.create(ImmutableList.of(new MapVirtualColumn("keys", "values", "params"))), + null, + Granularities.ALL, + ImmutableList.of(new DefaultDimensionSpec("params.key3", "params.key3")), + ImmutableList.of(new CountAggregatorFactory("count")), + null, + null, + null, + null, + null + ); + + final List<Row> result = runner.run(QueryPlus.wrap(query), new HashMap<>()).toList(); + final List<Row> expected = ImmutableList.of( + new MapBasedRow( + DateTimes.of("2011-01-12T00:00:00.000Z"), + MapVirtualColumnTestBase.mapOf("count", 1L, "params.key3", "value3") + ), + new MapBasedRow(DateTimes.of("2011-01-12T00:00:00.000Z"), MapVirtualColumnTestBase.mapOf("count", 2L)) + ); + + Assert.assertEquals(expected, result); + } +} diff --git a/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTest.java b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnSelectTest.java similarity index 91% rename from extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTest.java rename to extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnSelectTest.java index 7f7c4389299..9ff2f94b530 100644 --- a/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTest.java +++ b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnSelectTest.java @@ -23,7 +23,6 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; import com.google.common.io.CharSource; import org.apache.druid.data.input.impl.DelimitedParseSpec; import org.apache.druid.data.input.impl.DimensionsSpec; @@ -60,7 +59,7 @@ /** */ @RunWith(Parameterized.class) -public class MapVirtualColumnTest +public class MapVirtualColumnSelectTest { @Parameterized.Parameters public static Iterable<Object[]> constructorFeeder() throws IOException @@ -104,7 +103,7 @@ "2011-01-12T00:00:00.000Z\tc\tkey1,key5\tvalue1,value5,value9\n" ); - IncrementalIndex index1 = TestIndex.loadIncrementalIndex(index, input, parser); + IncrementalIndex index1 = TestIndex.loadIncrementalIndex(() -> index, input, parser); QueryableIndex index2 = TestIndex.persistRealtimeAndLoadMMapped(index1); return QueryRunnerTestHelper.transformToConstructionFeeder( @@ -127,7 +126,7 @@ private final QueryRunner runner; - public MapVirtualColumnTest(QueryRunner runner) + public MapVirtualColumnSelectTest(QueryRunner runner) { this.runner = runner; } @@ -159,26 +158,26 @@ public void testBasic() Druids.SelectQueryBuilder builder = testBuilder(); List<Map> expectedResults = Arrays.asList( - mapOf( + MapVirtualColumnTestBase.mapOf( "dim", "a", "params.key1", "value1", "params.key3", "value3", "params.key5", null, - "params", mapOf("key1", "value1", "key2", "value2", "key3", "value3") + "params", MapVirtualColumnTestBase.mapOf("key1", "value1", "key2", "value2", "key3", "value3") ), - mapOf( + MapVirtualColumnTestBase.mapOf( "dim", "b", "params.key1", null, "params.key3", null, "params.key5", null, - "params", mapOf("key4", "value4") + "params", MapVirtualColumnTestBase.mapOf("key4", "value4") ), - mapOf( + MapVirtualColumnTestBase.mapOf( "dim", "c", "params.key1", "value1", "params.key3", null, "params.key5", "value5", - "params", mapOf("key1", "value1", "key5", "value5") + "params", MapVirtualColumnTestBase.mapOf("key1", "value1", "key5", "value5") ) ); List<VirtualColumn> virtualColumns = Collections.singletonList(new MapVirtualColumn("keys", "values", "params")); @@ -189,15 +188,6 @@ public void testBasic() checkSelectQuery(selectQuery, expectedResults); } - private Map mapOf(Object... elements) - { - Map map = Maps.newHashMap(); - for (int i = 0; i < elements.length; i += 2) { - map.put(elements[i], elements[i + 1]); - } - return map; - } - private void checkSelectQuery(SelectQuery searchQuery, List<Map> expected) { List<Result<SelectResultValue>> results = runner.run(QueryPlus.wrap(searchQuery), ImmutableMap.of()).toList(); diff --git a/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTestBase.java b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTestBase.java new file mode 100644 index 00000000000..7736ee246b9 --- /dev/null +++ b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTestBase.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.druid.segment; + +import com.google.common.io.CharSource; +import org.apache.druid.data.input.impl.DelimitedParseSpec; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.StringInputRowParser; +import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.segment.incremental.IncrementalIndex; +import org.apache.druid.segment.incremental.IncrementalIndexSchema; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class MapVirtualColumnTestBase +{ + static IncrementalIndex generateIndex() throws IOException + { + final CharSource input = CharSource.wrap( + "2011-01-12T00:00:00.000Z\ta\tkey1,key2,key3\tvalue1,value2,value3\n" + + "2011-01-12T00:00:00.000Z\tb\tkey4,key5,key6\tvalue4\n" + + "2011-01-12T00:00:00.000Z\tc\tkey1,key5\tvalue1,value5,value9\n" + ); + + final StringInputRowParser parser = new StringInputRowParser( + new DelimitedParseSpec( + new TimestampSpec("ts", "auto", null), + new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Arrays.asList("dim", "keys", "values")), null, null), + "\t", + ",", + Arrays.asList("ts", "dim", "keys", "values"), + false, + 0 + ), + "utf8" + ); + + final IncrementalIndexSchema schema = new IncrementalIndexSchema.Builder() + .withMinTimestamp(DateTimes.of("2011-01-12T00:00:00.000Z").getMillis()) + .build(); + + return TestIndex.loadIncrementalIndex( + () -> new IncrementalIndex.Builder() + .setIndexSchema(schema) + .setMaxRowCount(10000) + .buildOnheap(), + input, + parser + ); + } + + static <K, V> Map<K, V> mapOf(Object... elements) + { + final Map<K, V> map = new HashMap<>(); + for (int i = 0; i < elements.length; i += 2) { + //noinspection unchecked + map.put((K) elements[i], (V) elements[i + 1]); + } + return map; + } +} diff --git a/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTopNTest.java b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTopNTest.java new file mode 100644 index 00000000000..e2236815921 --- /dev/null +++ b/extensions-contrib/virtual-columns/src/test/java/org/apache/druid/segment/MapVirtualColumnTopNTest.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.druid.segment; + +import com.google.common.collect.ImmutableList; +import org.apache.druid.collections.StupidPool; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.QueryPlus; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryRunnerTestHelper; +import org.apache.druid.query.Result; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; +import org.apache.druid.query.topn.DimensionAndMetricValueExtractor; +import org.apache.druid.query.topn.NumericTopNMetricSpec; +import org.apache.druid.query.topn.TopNQuery; +import org.apache.druid.query.topn.TopNQueryConfig; +import org.apache.druid.query.topn.TopNQueryQueryToolChest; +import org.apache.druid.query.topn.TopNQueryRunnerFactory; +import org.apache.druid.query.topn.TopNResultValue; +import org.apache.druid.segment.incremental.IncrementalIndex; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +public class MapVirtualColumnTopNTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private QueryRunner<Result<TopNResultValue>> runner; + + @Before + public void setup() throws IOException + { + final IncrementalIndex incrementalIndex = MapVirtualColumnTestBase.generateIndex(); + + final TopNQueryRunnerFactory factory = new TopNQueryRunnerFactory( + new StupidPool<>("map-virtual-column-test", () -> ByteBuffer.allocate(1024)), + new TopNQueryQueryToolChest( + new TopNQueryConfig(), + QueryRunnerTestHelper.NoopIntervalChunkingQueryRunnerDecorator() + ), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ); + + runner = QueryRunnerTestHelper.makeQueryRunner( + factory, + "index1", + new IncrementalIndexSegment(incrementalIndex, "index1"), + "incremental" + ); + } + + @Test + public void testWithMapColumn() + { + final TopNQuery query = new TopNQuery( + new TableDataSource(QueryRunnerTestHelper.dataSource), + VirtualColumns.create( + ImmutableList.of( + new MapVirtualColumn("keys", "values", "params") + ) + ), + new DefaultDimensionSpec("params", "params"), // params is the map type + new NumericTopNMetricSpec("count"), + 1, + new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("2011/2012"))), + null, + Granularities.ALL, + ImmutableList.of(new CountAggregatorFactory("count")), + null, + null + ); + + expectedException.expect(UnsupportedOperationException.class); + expectedException.expectMessage("Map column doesn't support getRow()"); + runner.run(QueryPlus.wrap(query), new HashMap<>()).toList(); + } + + @Test + public void testWithSubColumn() + { + final TopNQuery query = new TopNQuery( + new TableDataSource(QueryRunnerTestHelper.dataSource), + VirtualColumns.create( + ImmutableList.of( + new MapVirtualColumn("keys", "values", "params") + ) + ), + new DefaultDimensionSpec("params.key3", "params.key3"), // params.key3 is string + new NumericTopNMetricSpec("count"), + 2, + new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("2011/2012"))), + null, + Granularities.ALL, + ImmutableList.of(new CountAggregatorFactory("count")), + null, + null + ); + + final List<Result<TopNResultValue>> result = runner.run(QueryPlus.wrap(query), new HashMap<>()).toList(); + final List<Result<TopNResultValue>> expected = Collections.singletonList( + new Result<>( + DateTimes.of("2011-01-12T00:00:00.000Z"), + new TopNResultValue( + ImmutableList.of( + new DimensionAndMetricValueExtractor(MapVirtualColumnTestBase.mapOf("count", 2L, "params.key3", null)), + new DimensionAndMetricValueExtractor(MapVirtualColumnTestBase.mapOf("count", 1L, "params.key3", "value3")) + ) + ) + ) + ); + + Assert.assertEquals(expected, result); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryMergeBufferTest.java b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryMergeBufferTest.java index 537ecb3be59..abcf40e1ed4 100644 --- a/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryMergeBufferTest.java +++ b/processing/src/test/java/org/apache/druid/query/groupby/GroupByQueryMergeBufferTest.java @@ -104,7 +104,6 @@ public int getMinRemainBufferNum() public static final DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig() { - @Override public String getFormatString() { diff --git a/processing/src/test/java/org/apache/druid/segment/TestIndex.java b/processing/src/test/java/org/apache/druid/segment/TestIndex.java index 201354a430b..dd865fe9fa1 100644 --- a/processing/src/test/java/org/apache/druid/segment/TestIndex.java +++ b/processing/src/test/java/org/apache/druid/segment/TestIndex.java @@ -19,6 +19,7 @@ package org.apache.druid.segment; +import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.io.CharSource; import com.google.common.io.LineProcessor; @@ -315,15 +316,16 @@ public static IncrementalIndex loadIncrementalIndex( ), "utf8" ); - return loadIncrementalIndex(retVal, source, parser); + return loadIncrementalIndex(() -> retVal, source, parser); } public static IncrementalIndex loadIncrementalIndex( - final IncrementalIndex retVal, + final Supplier<IncrementalIndex> indexSupplier, final CharSource source, final StringInputRowParser parser ) throws IOException { + final IncrementalIndex retVal = indexSupplier.get(); final AtomicLong startTime = new AtomicLong(); int lineCount = source.readLines( new LineProcessor<Integer>() ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
