Repository: kylin Updated Branches: refs/heads/KYLIN-2428 [created] 96a415927
KYLIN-2428 cleanup unnecessary shaded libraries which are provided by Hadoop runtime Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/07fc6e61 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/07fc6e61 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/07fc6e61 Branch: refs/heads/KYLIN-2428 Commit: 07fc6e611de4555c06f498bfd94dd05303aae7b3 Parents: 7a6dd1c Author: Billy Liu <billy...@apache.org> Authored: Mon Feb 6 22:34:37 2017 +0800 Committer: Billy Liu <billy...@apache.org> Committed: Mon Feb 6 22:34:37 2017 +0800 ---------------------------------------------------------------------- core-common/pom.xml | 22 +- core-cube/pom.xml | 18 +- .../apache/kylin/gridtable/GTInvertedIndex.java | 223 ------------------- .../gridtable/GTInvertedIndexOfColumn.java | 133 ----------- .../gridtable/SimpleInvertedIndexTest.java | 196 ---------------- core-job/pom.xml | 9 + core-metadata/pom.xml | 23 ++ core-storage/pom.xml | 20 ++ engine-spark/pom.xml | 1 + jdbc/pom.xml | 2 +- pom.xml | 209 +++++++++-------- query/pom.xml | 6 + server/pom.xml | 4 + source-kafka/pom.xml | 16 +- storage-hbase/pom.xml | 4 - 15 files changed, 201 insertions(+), 685 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/core-common/pom.xml ---------------------------------------------------------------------- diff --git a/core-common/pom.xml b/core-common/pom.xml index 016d470..95d3c29 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -35,26 +35,10 @@ <dependencies> <!-- Basic Utilities --> <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>commons-cli</groupId> - <artifactId>commons-cli</artifactId> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-email</artifactId> </dependency> @@ -63,9 +47,11 @@ <artifactId>httpclient</artifactId> </dependency> <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> </dependency> + + <!-- Provided --> <dependency> <groupId>com.google.code.findbugs</groupId> <artifactId>jsr305</artifactId> http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/core-cube/pom.xml ---------------------------------------------------------------------- diff --git a/core-cube/pom.xml b/core-cube/pom.xml index 0a30432..fe80783 100644 --- a/core-cube/pom.xml +++ b/core-cube/pom.xml @@ -48,9 +48,23 @@ <groupId>commons-collections</groupId> <artifactId>commons-collections</artifactId> </dependency> + + <!-- Provided --> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> <dependency> - <groupId>com.n3twork.druid</groupId> - <artifactId>extendedset</artifactId> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> </dependency> <dependency> <groupId>com.esotericsoftware</groupId> http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java deleted file mode 100644 index a2c713a..0000000 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.gridtable; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.ImmutableBitSet; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; - -import it.uniroma3.mat.extendedset.intset.ConciseSet; - -/** - * A thread-safe inverted index of row blocks in memory. - * - * Note function not() must return all blocks, because index only know what block contains a value, - * but not sure what block does not contain a value. - * - * @author yangli9 - */ -public class GTInvertedIndex { - - private final GTInfo info; - private final ImmutableBitSet colPreferIndex; - private final ImmutableBitSet colBlocks; - private final GTInvertedIndexOfColumn[] index; // for each column - - private volatile int nIndexedBlocks; - - public GTInvertedIndex(GTInfo info) { - this.info = info; - this.colPreferIndex = info.colPreferIndex; - this.colBlocks = info.selectColumnBlocks(colPreferIndex); - - index = new GTInvertedIndexOfColumn[info.getColumnCount()]; - for (int i = 0; i < colPreferIndex.trueBitCount(); i++) { - int c = colPreferIndex.trueBitAt(i); - index[c] = new GTInvertedIndexOfColumn(info.codeSystem.getComparator()); - } - } - - public void add(GTRowBlock block) { - - @SuppressWarnings("unchecked") - Set<ByteArray>[] distinctValues = new Set[info.getColumnCount()]; - for (int i = 0; i < colPreferIndex.trueBitCount(); i++) { - int c = colPreferIndex.trueBitAt(i); - distinctValues[c] = new HashSet<ByteArray>(); - } - - GTRowBlock.Reader reader = block.getReader(colBlocks); - GTRecord record = new GTRecord(info); - while (reader.hasNext()) { - reader.fetchNext(record); - for (int i = 0; i < colPreferIndex.trueBitCount(); i++) { - int c = colPreferIndex.trueBitAt(i); - distinctValues[c].add(record.get(c)); - } - } - - for (int i = 0; i < colPreferIndex.trueBitCount(); i++) { - int c = colPreferIndex.trueBitAt(i); - index[c].add(distinctValues[c], block.getSequenceId()); - } - - nIndexedBlocks = Math.max(nIndexedBlocks, block.seqId + 1); - } - - public ConciseSet filter(TupleFilter filter) { - return filter(filter, nIndexedBlocks); - } - - public ConciseSet filter(TupleFilter filter, int totalBlocks) { - // number of indexed blocks may increase as we do evaluation - int indexedBlocks = nIndexedBlocks; - - Evaluator evaluator = new Evaluator(indexedBlocks); - ConciseSet r = evaluator.evaluate(filter); - - // add blocks that have not been indexed - for (int i = indexedBlocks; i < totalBlocks; i++) { - r.add(i); - } - - return r; - } - - private class Evaluator { - private int indexedBlocks; - - Evaluator(int indexedBlocks) { - this.indexedBlocks = indexedBlocks; - } - - public ConciseSet evaluate(TupleFilter filter) { - if (filter == null) { - return all(); - } - - if (filter instanceof LogicalTupleFilter) - return evalLogical((LogicalTupleFilter) filter); - - if (filter instanceof CompareTupleFilter) - return evalCompare((CompareTupleFilter) filter); - - // unable to evaluate - return all(); - } - - @SuppressWarnings("unchecked") - private ConciseSet evalCompare(CompareTupleFilter filter) { - int col = col(filter); - if (index[col] == null) - return all(); - - switch (filter.getOperator()) { - case ISNULL: - return index[col].getNull(); - case ISNOTNULL: - return all(); - case EQ: - return index[col].getEquals((ByteArray) filter.getFirstValue()); - case NEQ: - return all(); - case IN: - return index[col].getIn((Iterable<ByteArray>) filter.getValues()); - case NOTIN: - return all(); - case LT: - return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), false); - case LTE: - return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), true); - case GT: - return index[col].getRange((ByteArray) filter.getFirstValue(), false, null, false); - case GTE: - return index[col].getRange((ByteArray) filter.getFirstValue(), true, null, false); - default: - throw new IllegalStateException("Unsupported operator " + filter.getOperator()); - } - } - - private ConciseSet evalLogical(LogicalTupleFilter filter) { - List<? extends TupleFilter> children = filter.getChildren(); - - switch (filter.getOperator()) { - case AND: - return evalLogicalAnd(children); - case OR: - return evalLogicalOr(children); - case NOT: - return evalLogicalNot(children); - default: - throw new IllegalStateException("Unsupported operator " + filter.getOperator()); - } - } - - private ConciseSet evalLogicalAnd(List<? extends TupleFilter> children) { - ConciseSet set = all(); - - for (TupleFilter c : children) { - ConciseSet t = evaluate(c); - if (t == null) - continue; // because it's AND - - set.retainAll(t); - } - return set; - } - - private ConciseSet evalLogicalOr(List<? extends TupleFilter> children) { - ConciseSet set = new ConciseSet(); - - for (TupleFilter c : children) { - ConciseSet t = evaluate(c); - if (t == null) - return null; // because it's OR - - set.addAll(t); - } - return set; - } - - private ConciseSet evalLogicalNot(List<? extends TupleFilter> children) { - return all(); - } - - private ConciseSet all() { - return not(new ConciseSet()); - } - - private ConciseSet not(ConciseSet set) { - set.add(indexedBlocks); - set.complement(); - return set; - } - - private int col(CompareTupleFilter filter) { - return filter.getColumn().getColumnDesc().getZeroBasedIndex(); - } - - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java deleted file mode 100644 index bfacc0f..0000000 --- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.gridtable; - -import java.util.NavigableMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.kylin.common.util.ByteArray; - -import com.google.common.collect.Maps; - -import it.uniroma3.mat.extendedset.intset.ConciseSet; - -public class GTInvertedIndexOfColumn { - - final private IGTComparator comparator; - final private ReentrantReadWriteLock rwLock; - - private int nBlocks; - private NavigableMap<ByteArray, ConciseSet> rangeIndex; - private ConciseSet nullIndex; - - public GTInvertedIndexOfColumn(IGTComparator comparator) { - this.comparator = comparator; - this.rwLock = new ReentrantReadWriteLock(); - this.rangeIndex = Maps.newTreeMap(comparator); - this.nullIndex = new ConciseSet(); - } - - public void add(Iterable<ByteArray> codes, int blockId) { - rwLock.writeLock().lock(); - try { - for (ByteArray code : codes) { - if (comparator.isNull(code)) { - nullIndex.add(blockId); - continue; - } - ConciseSet set = rangeIndex.get(code); - if (set == null) { - set = new ConciseSet(); - rangeIndex.put(code.copy(), set); - } - set.add(blockId); - } - - if (blockId >= nBlocks) { - nBlocks = blockId + 1; - } - - } finally { - rwLock.writeLock().unlock(); - } - } - - public ConciseSet getNull() { - rwLock.readLock().lock(); - try { - return nullIndex.clone(); - } finally { - rwLock.readLock().unlock(); - } - } - - public ConciseSet getEquals(ByteArray code) { - rwLock.readLock().lock(); - try { - ConciseSet set = rangeIndex.get(code); - if (set == null) - return new ConciseSet(); - else - return set.clone(); - } finally { - rwLock.readLock().unlock(); - } - } - - public ConciseSet getIn(Iterable<ByteArray> codes) { - rwLock.readLock().lock(); - try { - ConciseSet r = new ConciseSet(); - for (ByteArray code : codes) { - ConciseSet set = rangeIndex.get(code); - if (set != null) - r.addAll(set); - } - return r; - } finally { - rwLock.readLock().unlock(); - } - } - - public ConciseSet getRange(ByteArray from, boolean fromInclusive, ByteArray to, boolean toInclusive) { - rwLock.readLock().lock(); - try { - ConciseSet r = new ConciseSet(); - if (from == null && to == null) { - r.add(nBlocks); - r.complement(); - return r; - } - NavigableMap<ByteArray, ConciseSet> subMap; - if (from == null) { - subMap = rangeIndex.headMap(to, toInclusive); - } else if (to == null) { - subMap = rangeIndex.tailMap(from, fromInclusive); - } else { - subMap = rangeIndex.subMap(from, fromInclusive, to, toInclusive); - } - for (ConciseSet set : subMap.values()) { - r.addAll(set); - } - return r; - } finally { - rwLock.readLock().unlock(); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java ---------------------------------------------------------------------- diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java deleted file mode 100644 index 5e3e771..0000000 --- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.gridtable; - -import static org.junit.Assert.assertEquals; - -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.util.ArrayList; - -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.datatype.StringSerializer; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.LogicalTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum; -import org.apache.kylin.metadata.model.TblColRef; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import com.google.common.collect.Lists; - -import it.uniroma3.mat.extendedset.intset.ConciseSet; - -public class SimpleInvertedIndexTest extends LocalFileMetadataTestCase { - - GTInfo info; - GTInvertedIndex index; - ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList(); - ArrayList<ConciseSet> basicResults = Lists.newArrayList(); - - @BeforeClass - public static void setUp() throws Exception { - staticCreateTestMetadata(); - } - - @AfterClass - public static void after() throws Exception { - cleanAfterClass(); - } - - public SimpleInvertedIndexTest() { - - info = UnitTestSupport.advancedInfo(); - TblColRef colA = info.colRef(0); - - // block i contains value "i", the last is NULL - index = new GTInvertedIndex(info); - GTRowBlock mockBlock = GTRowBlock.allocate(info); - GTRowBlock.Writer writer = mockBlock.getWriter(); - GTRecord record = new GTRecord(info); - for (int i = 0; i < 10; i++) { - record.setValues(i < 9 ? "" + i : null, "", "", new Long(0), new BigDecimal(0)); - for (int j = 0; j < info.getRowBlockSize(); j++) { - writer.append(record); - } - writer.readyForFlush(); - index.add(mockBlock); - - writer.clearForNext(); - } - - basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL)); - basicResults.add(set(9)); - - basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL)); - basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); - - basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0)); - basicResults.add(set(0)); - - basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0)); - basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); - - basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5)); - basicResults.add(set(0, 5)); - - basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5)); - basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); - - basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3)); - basicResults.add(set(0, 1, 2)); - - basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3)); - basicResults.add(set(0, 1, 2, 3)); - - basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3)); - basicResults.add(set(4, 5, 6, 7, 8)); - - basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3)); - basicResults.add(set(3, 4, 5, 6, 7, 8)); - } - - @Test - public void testBasics() { - for (int i = 0; i < basicFilters.size(); i++) { - assertEquals(basicResults.get(i), index.filter(basicFilters.get(i))); - } - } - - @Test - public void testLogicalAnd() { - for (int i = 0; i < basicFilters.size(); i++) { - for (int j = 0; j < basicFilters.size(); j++) { - LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j)); - ConciseSet r = basicResults.get(i).clone(); - r.retainAll(basicResults.get(j)); - assertEquals(r, index.filter(f)); - } - } - } - - @Test - public void testLogicalOr() { - for (int i = 0; i < basicFilters.size(); i++) { - for (int j = 0; j < basicFilters.size(); j++) { - LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j)); - ConciseSet r = basicResults.get(i).clone(); - r.addAll(basicResults.get(j)); - assertEquals(r, index.filter(f)); - } - } - } - - @Test - public void testNotEvaluable() { - ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); - - CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0); - assertEquals(all, index.filter(notEvaluable)); - - LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable); - assertEquals(all, index.filter(or)); - - LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable); - assertEquals(basicResults.get(0), index.filter(and)); - } - - public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) { - CompareTupleFilter filter = new CompareTupleFilter(op); - filter.addChild(columnFilter(col)); - for (int i : ids) { - filter.addChild(constFilter(i)); - } - return filter; - } - - public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) { - LogicalTupleFilter filter = new LogicalTupleFilter(op); - for (TupleFilter f : filters) - filter.addChild(f); - return filter; - } - - public static ColumnTupleFilter columnFilter(TblColRef col) { - return new ColumnTupleFilter(col); - } - - public static ConstantTupleFilter constFilter(int id) { - byte[] space = new byte[10]; - ByteBuffer buf = ByteBuffer.wrap(space); - StringSerializer stringSerializer = new StringSerializer(DataType.getType("string")); - stringSerializer.serialize("" + id, buf); - ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position()); - return new ConstantTupleFilter(data); - } - - public static ConciseSet set(int... ints) { - ConciseSet set = new ConciseSet(); - for (int i : ints) - set.add(i); - return set; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/core-job/pom.xml ---------------------------------------------------------------------- diff --git a/core-job/pom.xml b/core-job/pom.xml index 36d34c8..9478555 100644 --- a/core-job/pom.xml +++ b/core-job/pom.xml @@ -39,6 +39,15 @@ <artifactId>kylin-core-cube</artifactId> </dependency> + <!-- Provided --> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-recipes</artifactId> http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/core-metadata/pom.xml ---------------------------------------------------------------------- diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml index 87c4438..0832e80 100644 --- a/core-metadata/pom.xml +++ b/core-metadata/pom.xml @@ -39,6 +39,29 @@ <artifactId>kylin-core-common</artifactId> </dependency> + <!-- Provided --> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + </dependency> + + <!-- Compiled --> <dependency> <groupId>net.sf.ehcache</groupId> <artifactId>ehcache</artifactId> http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/core-storage/pom.xml ---------------------------------------------------------------------- diff --git a/core-storage/pom.xml b/core-storage/pom.xml index 9bd3f04..8c3da5f 100644 --- a/core-storage/pom.xml +++ b/core-storage/pom.xml @@ -39,8 +39,28 @@ <artifactId>kylin-core-cube</artifactId> </dependency> + <!-- Provided --> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + </dependency> + <!-- Env & Test --> <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-common</artifactId> <type>test-jar</type> http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/engine-spark/pom.xml ---------------------------------------------------------------------- diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml index be2cdf6..fe6d998 100644 --- a/engine-spark/pom.xml +++ b/engine-spark/pom.xml @@ -70,6 +70,7 @@ <dependency> <groupId>org.reflections</groupId> <artifactId>reflections</artifactId> + <scope>provided</scope> </dependency> <!-- Hadoop dependency --> http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 874ead6..34bdc8a 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -71,7 +71,7 @@ <excludes> <exclude>com.google.protobuf:*</exclude> <exclude>commons-logging:*</exclude> - <exclude>commons-codec:*</exclude> + <exclude>org.slf4j:jul-to-slf4j</exclude> </excludes> </artifactSet> http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b82eee2..3d6c98e 100644 --- a/pom.xml +++ b/pom.xml @@ -55,15 +55,32 @@ <!-- HBase versions --> <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version> + + <!-- Kafka versions --> <kafka.version>0.10.1.0</kafka.version> - <!-- Hadoop deps, keep compatible with hadoop2.version --> + <!-- Spark versions --> + <spark.version>1.6.3</spark.version> + <kryo.version>4.0.0</kryo.version> + <reflections.version>0.9.10</reflections.version> + + <!-- Calcite Version --> + <calcite.version>1.11.0</calcite.version> + <avatica.version>1.9.0</avatica.version> + + <!-- Hadoop Common deps, keep compatible with hadoop2.version --> <zookeeper.version>3.4.6</zookeeper.version> <curator.version>2.7.1</curator.version> - <jackson.version>2.2.4</jackson.version> <jsr305.version>3.0.1</jsr305.version> <guava.version>14.0</guava.version> + <jsch.version>0.1.53</jsch.version> <commons-cli.version>1.2</commons-cli.version> + <commons-lang.version>2.6</commons-lang.version> + <commons-io.version>2.5</commons-io.version> + <commons-math3.version>3.6.1</commons-math3.version> + + <!-- Calcite deps, keep compatible with calcite.version --> + <jackson.version>2.6.3</jackson.version> <!-- Test Dependency versions --> <antlr.version>3.4</antlr.version> @@ -75,26 +92,16 @@ <jamm.version>0.3.1</jamm.version> <!-- Commons --> - <commons-lang.version>2.6</commons-lang.version> <commons-lang3.version>3.4</commons-lang3.version> <commons-collections.version>3.2.2</commons-collections.version> - <commons-io.version>2.5</commons-io.version> <commons-daemon.version>1.0.15</commons-daemon.version> <commons-email.version>1.4</commons-email.version> - <commons-math3.version>3.6.1</commons-math3.version> - - <!-- Spark --> - <spark.version>1.6.3</spark.version> <!-- Utility --> <log4j.version>1.2.17</log4j.version> <slf4j.version>1.7.21</slf4j.version> - <reflections.version>0.9.10</reflections.version> - <jsch.version>0.1.53</jsch.version> <xerces.version>2.11.0</xerces.version> <xalan.version>2.7.2</xalan.version> - <extendedset.version>1.3.4</extendedset.version> - <kryo.version>4.0.0</kryo.version> <ehcache.version>2.10.2.2.21</ehcache.version> <apache-httpclient.version>4.2.5</apache-httpclient.version> <roaring.version>0.6.18</roaring.version> @@ -111,10 +118,6 @@ <opensaml.version>2.6.4</opensaml.version> <aspectj.version>1.8.9</aspectj.version> - <!-- Calcite Version --> - <calcite.version>1.11.0</calcite.version> - <avatica.version>1.9.0</avatica.version> - <!-- Sonar --> <sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin> <sonar.dynamicAnalysis>reuseReports</sonar.dynamicAnalysis> @@ -390,6 +393,66 @@ <version>${hadoop2.version}</version> <optional>true</optional> </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>${zookeeper.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-framework</artifactId> + <version>${curator.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-recipes</artifactId> + <version>${curator.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + <version>${jsr305.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + <version>${guava.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>com.jcraft</groupId> + <artifactId>jsch</artifactId> + <version>${jsch.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>commons-cli</groupId> + <artifactId>commons-cli</artifactId> + <version>${commons-cli.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>${commons-lang.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-math3</artifactId> + <version>${commons-math3.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>${commons-io.version}</version> + <scope>provided</scope> + </dependency> <!-- HBase2 dependencies --> <dependency> @@ -476,23 +539,45 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>${jackson.version}</version> + </dependency> + <!-- Spark dependency --> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> <version>${spark.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_2.10</artifactId> <version>${spark.version}</version> + <scope>provided</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-hive_2.10</artifactId> <version>${spark.version}</version> + <scope>provided</scope> </dependency> <dependency> + <groupId>com.esotericsoftware</groupId> + <artifactId>kryo-shaded</artifactId> + <version>${kryo.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.reflections</groupId> + <artifactId>reflections</artifactId> + <version>${reflections.version}</version> + <scope>provided</scope> + </dependency> + <!-- Kafka dependency --> + <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.10</artifactId> <version>${kafka.version}</version> @@ -501,26 +586,6 @@ <!-- Other dependencies --> <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>${junit.version}</version> - </dependency> - <dependency> - <groupId>org.apache.zookeeper</groupId> - <artifactId>zookeeper</artifactId> - <version>${zookeeper.version}</version> - </dependency> - <dependency> - <groupId>commons-cli</groupId> - <artifactId>commons-cli</artifactId> - <version>${commons-cli.version}</version> - </dependency> - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <version>${commons-lang.version}</version> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> <version>${commons-lang3.version}</version> @@ -531,21 +596,15 @@ <version>${commons-collections.version}</version> </dependency> <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-math3</artifactId> - <version>${commons-math3.version}</version> - </dependency> - - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - <version>${commons-io.version}</version> - </dependency> - <dependency> <groupId>commons-daemon</groupId> <artifactId>commons-daemon</artifactId> <version>${commons-daemon.version}</version> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-email</artifactId> + <version>${commons-email.version}</version> + </dependency> <!-- Logging --> <dependency> @@ -571,40 +630,18 @@ <version>${slf4j.version}</version> </dependency> + <!-- Test --> <dependency> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-databind</artifactId> - <version>${jackson.version}</version> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-email</artifactId> - <version>${commons-email.version}</version> - </dependency> - <dependency> - <groupId>com.google.code.findbugs</groupId> - <artifactId>jsr305</artifactId> - <version>${jsr305.version}</version> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>${guava.version}</version> - </dependency> - <dependency> - <groupId>org.reflections</groupId> - <artifactId>reflections</artifactId> - <version>${reflections.version}</version> - </dependency> - <dependency> - <groupId>com.jcraft</groupId> - <artifactId>jsch</artifactId> - <version>${jsch.version}</version> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>${junit.version}</version> + <scope>test</scope> </dependency> <dependency> <groupId>org.dbunit</groupId> <artifactId>dbunit</artifactId> <version>${dbunit.version}</version> + <scope>test</scope> </dependency> <dependency> <groupId>org.apache.maven</groupId> @@ -615,6 +652,7 @@ <groupId>com.h2database</groupId> <artifactId>h2</artifactId> <version>${h2.version}</version> + <scope>test</scope> </dependency> <dependency> <groupId>xerces</groupId> @@ -627,31 +665,10 @@ <version>${xalan.version}</version> </dependency> <dependency> - <groupId>com.n3twork.druid</groupId> - <artifactId>extendedset</artifactId> - <version>${extendedset.version}</version> - </dependency> - <dependency> - <groupId>com.esotericsoftware</groupId> - <artifactId>kryo-shaded</artifactId> - <version>${kryo.version}</version> - </dependency> - <dependency> <groupId>com.github.jbellis</groupId> <artifactId>jamm</artifactId> <version>${jamm.version}</version> </dependency> - - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-framework</artifactId> - <version>${curator.version}</version> - </dependency> - <dependency> - <groupId>org.apache.curator</groupId> - <artifactId>curator-recipes</artifactId> - <version>${curator.version}</version> - </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/query/pom.xml ---------------------------------------------------------------------- diff --git a/query/pom.xml b/query/pom.xml index 6ab74a7..d35c74c 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -47,10 +47,16 @@ <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-storage</artifactId> </dependency> + + <!-- Provided --> <dependency> <groupId>commons-cli</groupId> <artifactId>commons-cli</artifactId> </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> <!-- Env & Test --> <dependency> http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/server/pom.xml ---------------------------------------------------------------------- diff --git a/server/pom.xml b/server/pom.xml index 454ed6b..3c96e60 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -233,6 +233,10 @@ <artifactId>jetty-all</artifactId> <groupId>org.eclipse.jetty.aggregate</groupId> </exclusion> + <exclusion> + <artifactId>calcite-core</artifactId> + <groupId>org.apache.calcite</groupId> + </exclusion> </exclusions> </dependency> http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/source-kafka/pom.xml ---------------------------------------------------------------------- diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml index e2fe448..da5e033 100644 --- a/source-kafka/pom.xml +++ b/source-kafka/pom.xml @@ -35,23 +35,19 @@ <dependencies> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-engine-mr</artifactId> + <artifactId>kylin-core-common</artifactId> </dependency> - <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-core-common</artifactId> + <artifactId>kylin-engine-mr</artifactId> </dependency> + <!-- Provided --> <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.10</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> </dependency> + <!-- Env & Test --> <dependency> <groupId>org.apache.hadoop</groupId> @@ -76,9 +72,5 @@ <type>test-jar</type> <scope>test</scope> </dependency> - - - </dependencies> - </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/07fc6e61/storage-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml index 3aea531..054b2e7 100644 --- a/storage-hbase/pom.xml +++ b/storage-hbase/pom.xml @@ -119,12 +119,8 @@ <include>org.apache.kylin:kylin-core-metadata</include> <include>org.apache.kylin:kylin-core-dictionary</include> <include>org.apache.kylin:kylin-core-cube</include> - <include>com.ning:compress-lzf</include> <include>org.roaringbitmap:RoaringBitmap</include> <include>com.tdunning:t-digest</include> - <!-- below for inverted index only --> - <include>com.n3twork.druid:extendedset</include> - <include>org.apache.commons:commons-lang3</include> </includes> </artifactSet> <filters>