This is an automated email from the ASF dual-hosted git repository. jonwei pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push: new b06ac54 add PrefixFilteredDimensionSpec for multi-value dimensions (#6307) b06ac54 is described below commit b06ac54a5ee9176a97baad9d69ce96aa27265eae Author: dongyifeng <dyf6...@163.com> AuthorDate: Sat Oct 13 08:51:09 2018 +0800 add PrefixFilteredDimensionSpec for multi-value dimensions (#6307) * add PrefixFilteredDimensionSpec for multi-value dimensions * add docs for PrefixFilteredDimensionSpec * remove unnecessary null handling * add null check to the result of NullHandling --- docs/content/querying/dimensionspecs.md | 6 ++ .../druid/query/dimension/DimensionSpec.java | 3 +- ...nSpec.java => PrefixFilteredDimensionSpec.java} | 49 +++++------ .../dimension/RegexFilteredDimensionSpec.java | 3 +- .../dimension/PrefixFilteredDimensionSpecTest.java | 98 ++++++++++++++++++++++ 5 files changed, 130 insertions(+), 29 deletions(-) diff --git a/docs/content/querying/dimensionspecs.md b/docs/content/querying/dimensionspecs.md index b70fa68..e9a41e0 100644 --- a/docs/content/querying/dimensionspecs.md +++ b/docs/content/querying/dimensionspecs.md @@ -66,6 +66,12 @@ Following filtered dimension spec retains only the values matching regex. Note t { "type" : "regexFiltered", "delegate" : <dimensionSpec>, "pattern": <java regex pattern> } ``` +Following filtered dimension spec retains only the values starting with the same prefix. + +```json +{ "type" : "prefixFiltered", "delegate" : <dimensionSpec>, "prefix": <prefix string> } +``` + For more details and examples, see [multi-value dimensions](multi-value-dimensions.html). ### Lookup DimensionSpecs diff --git a/processing/src/main/java/org/apache/druid/query/dimension/DimensionSpec.java b/processing/src/main/java/org/apache/druid/query/dimension/DimensionSpec.java index 3adf5d5..bbf970e 100644 --- a/processing/src/main/java/org/apache/druid/query/dimension/DimensionSpec.java +++ b/processing/src/main/java/org/apache/druid/query/dimension/DimensionSpec.java @@ -37,7 +37,8 @@ import javax.annotation.Nullable; @JsonSubTypes.Type(name = "default", value = DefaultDimensionSpec.class), @JsonSubTypes.Type(name = "extraction", value = ExtractionDimensionSpec.class), @JsonSubTypes.Type(name = "regexFiltered", value = RegexFilteredDimensionSpec.class), - @JsonSubTypes.Type(name = "listFiltered", value = ListFilteredDimensionSpec.class) + @JsonSubTypes.Type(name = "listFiltered", value = ListFilteredDimensionSpec.class), + @JsonSubTypes.Type(name = "prefixFiltered", value = PrefixFilteredDimensionSpec.class) }) public interface DimensionSpec extends Cacheable { diff --git a/processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java b/processing/src/main/java/org/apache/druid/query/dimension/PrefixFilteredDimensionSpec.java similarity index 74% copy from processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java copy to processing/src/main/java/org/apache/druid/query/dimension/PrefixFilteredDimensionSpec.java index 0ad457d..d4904b5 100644 --- a/processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java +++ b/processing/src/main/java/org/apache/druid/query/dimension/PrefixFilteredDimensionSpec.java @@ -22,42 +22,37 @@ package org.apache.druid.query.dimension; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; -import it.unimi.dsi.fastutil.ints.Int2IntMap; -import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.filter.DimFilterUtils; import org.apache.druid.segment.DimensionSelector; - -import javax.annotation.Nullable; +import it.unimi.dsi.fastutil.ints.Int2IntMap; +import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap; import java.nio.ByteBuffer; -import java.util.regex.Pattern; +import javax.annotation.Nullable; /** */ -public class RegexFilteredDimensionSpec extends BaseFilteredDimensionSpec +public class PrefixFilteredDimensionSpec extends BaseFilteredDimensionSpec { - private static final byte CACHE_TYPE_ID = 0x2; + private static final byte CACHE_TYPE_ID = 0x4; - private final String pattern; + private final String prefix; - private final Pattern compiledRegex; - - public RegexFilteredDimensionSpec( + public PrefixFilteredDimensionSpec( @JsonProperty("delegate") DimensionSpec delegate, - @JsonProperty("pattern") String pattern //rows not matching the pattern will be discarded + @JsonProperty("prefix") String prefix //rows not starting with the prefix will be discarded ) { super(delegate); - this.pattern = Preconditions.checkNotNull(pattern, "pattern must not be null"); - this.compiledRegex = Pattern.compile(pattern); + this.prefix = Preconditions.checkNotNull(prefix, "prefix must not be null"); } @JsonProperty - public String getPattern() + public String getPrefix() { - return pattern; + return prefix; } @Override @@ -76,7 +71,8 @@ public class RegexFilteredDimensionSpec extends BaseFilteredDimensionSpec @Override public boolean apply(@Nullable String input) { - return compiledRegex.matcher(NullHandling.nullToEmptyIfNeeded(input)).matches(); + String val = NullHandling.nullToEmptyIfNeeded(input); + return val == null ? false : val.startsWith(prefix); } } ); @@ -87,7 +83,7 @@ public class RegexFilteredDimensionSpec extends BaseFilteredDimensionSpec forwardMapping.defaultReturnValue(-1); for (int i = 0; i < selectorCardinality; i++) { String val = NullHandling.nullToEmptyIfNeeded(selector.lookupName(i)); - if (val != null && compiledRegex.matcher(val).matches()) { + if (val != null && val.startsWith(prefix)) { forwardMapping.put(i, count++); } } @@ -103,12 +99,12 @@ public class RegexFilteredDimensionSpec extends BaseFilteredDimensionSpec public byte[] getCacheKey() { byte[] delegateCacheKey = delegate.getCacheKey(); - byte[] regexBytes = StringUtils.toUtf8(pattern); - return ByteBuffer.allocate(2 + delegateCacheKey.length + regexBytes.length) + byte[] prefixBytes = StringUtils.toUtf8(prefix); + return ByteBuffer.allocate(2 + delegateCacheKey.length + prefixBytes.length) .put(CACHE_TYPE_ID) .put(delegateCacheKey) .put(DimFilterUtils.STRING_SEPARATOR) - .put(regexBytes) + .put(prefixBytes) .array(); } @@ -122,28 +118,27 @@ public class RegexFilteredDimensionSpec extends BaseFilteredDimensionSpec return false; } - RegexFilteredDimensionSpec that = (RegexFilteredDimensionSpec) o; + PrefixFilteredDimensionSpec that = (PrefixFilteredDimensionSpec) o; if (!delegate.equals(that.delegate)) { return false; } - return pattern.equals(that.pattern); - + return prefix.equals(that.prefix); } @Override public int hashCode() { int result = delegate.hashCode(); - result = 31 * result + pattern.hashCode(); + result = 31 * result + prefix.hashCode(); return result; } @Override public String toString() { - return "RegexFilteredDimensionSpec{" + - "pattern='" + pattern + '\'' + + return "PrefixFilteredDimensionSpec{" + + "Prefix='" + prefix + '\'' + '}'; } } diff --git a/processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java b/processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java index 0ad457d..84883c6 100644 --- a/processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java +++ b/processing/src/main/java/org/apache/druid/query/dimension/RegexFilteredDimensionSpec.java @@ -76,7 +76,8 @@ public class RegexFilteredDimensionSpec extends BaseFilteredDimensionSpec @Override public boolean apply(@Nullable String input) { - return compiledRegex.matcher(NullHandling.nullToEmptyIfNeeded(input)).matches(); + String val = NullHandling.nullToEmptyIfNeeded(input); + return val == null ? false : compiledRegex.matcher(val).matches(); } } ); diff --git a/processing/src/test/java/org/apache/druid/query/dimension/PrefixFilteredDimensionSpecTest.java b/processing/src/test/java/org/apache/druid/query/dimension/PrefixFilteredDimensionSpecTest.java new file mode 100644 index 0000000..2dc8fec --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/dimension/PrefixFilteredDimensionSpecTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.dimension; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.segment.DimensionSelector; +import org.apache.druid.segment.TestHelper; +import org.apache.druid.segment.data.IndexedInts; +import java.util.Arrays; +import org.junit.Assert; +import org.junit.Test; + +/** + */ +public class PrefixFilteredDimensionSpecTest +{ + + @Test + public void testSerde() throws Exception + { + ObjectMapper mapper = TestHelper.makeJsonMapper(); + + String jsonStr = "{\n" + + " \"type\": \"prefixFiltered\",\n" + + " \"delegate\": {\n" + + " \"type\": \"default\",\n" + + " \"dimension\": \"foo\",\n" + + " \"outputName\": \"bar\"\n" + + " },\n" + + " \"prefix\": \"xxx\"\n" + + "}"; + + PrefixFilteredDimensionSpec actual = (PrefixFilteredDimensionSpec) mapper.readValue( + mapper.writeValueAsString(mapper.readValue(jsonStr, DimensionSpec.class)), + DimensionSpec.class); + + PrefixFilteredDimensionSpec expected = new PrefixFilteredDimensionSpec( + new DefaultDimensionSpec("foo", "bar"), + "xxx" + ); + + Assert.assertEquals(expected, actual); + } + + @Test + public void testGetCacheKey() + { + PrefixFilteredDimensionSpec spec1 = new PrefixFilteredDimensionSpec( + new DefaultDimensionSpec("foo", "bar"), + "xxx" + ); + + PrefixFilteredDimensionSpec spec2 = new PrefixFilteredDimensionSpec( + new DefaultDimensionSpec("foo", "bar"), + "xyz" + ); + + Assert.assertFalse(Arrays.equals(spec1.getCacheKey(), spec2.getCacheKey())); + } + + @Test + public void testDecorator() + { + PrefixFilteredDimensionSpec spec = new PrefixFilteredDimensionSpec( + new DefaultDimensionSpec("foo", "far"), + "c" + ); + + DimensionSelector selector = spec.decorate(TestDimensionSelector.instance); + + Assert.assertEquals(1, selector.getValueCardinality()); + + IndexedInts row = selector.getRow(); + Assert.assertEquals(1, row.size()); + Assert.assertEquals(0, row.get(0)); + + Assert.assertEquals("c", selector.lookupName(0)); + + Assert.assertEquals(0, selector.idLookup().lookupId("c")); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org