KYLIN-1858 remove all ii related code
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2cc0b9c4 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2cc0b9c4 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2cc0b9c4 Branch: refs/heads/1.5.x-CDH5.7 Commit: 2cc0b9c43f77624dfaa09b6410a818ac020fb475 Parents: cd5e603 Author: Hongbin Ma <mahong...@apache.org> Authored: Fri Jul 8 15:14:37 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Fri Jul 8 15:14:37 2016 +0800 ---------------------------------------------------------------------- assembly/pom.xml | 6 - .../kylin/job/hadoop/invertedindex/IICLI.java | 106 - .../kylin/job/hadoop/invertedindex/IITest.java | 266 -- .../apache/kylin/common/KylinConfigBase.java | 2 +- .../kylin/common/persistence/ResourceStore.java | 2 - .../org/apache/kylin/common/util/ClassUtil.java | 1 - .../common/util/AbstractKylinTestCase.java | 2 - engine-mr/pom.xml | 5 - .../org/apache/kylin/engine/mr/IMROutput.java | 26 +- .../kylin/engine/mr/JobBuilderSupport.java | 9 - .../java/org/apache/kylin/engine/mr/MRUtil.java | 5 - .../engine/mr/common/AbstractHadoopJob.java | 18 - .../mr/invertedindex/BatchIIJobBuilder.java | 84 - .../kylin/engine/mr/invertedindex/IIJob.java | 71 - .../mr/invertedindex/InvertedIndexJob.java | 135 - .../mr/invertedindex/InvertedIndexMapper.java | 77 - .../invertedindex/InvertedIndexPartitioner.java | 73 - .../mr/invertedindex/InvertedIndexReducer.java | 128 - .../UpdateIIInfoAfterBuildStep.java | 83 - engine-streaming/pom.xml | 5 - .../invertedindex/test_kylin_ii_inner_join.json | 16 - .../invertedindex/test_kylin_ii_left_join.json | 15 - .../invertedindex/test_streaming_table_ii.json | 15 - .../test_kylin_ii_inner_join_desc.json | 69 - .../test_kylin_ii_left_join_desc.json | 69 - .../test_streaming_table_ii_desc.json | 26 - .../localmeta/project/default.json | 10 - .../.settings/org.eclipse.core.resources.prefs | 6 - .../.settings/org.eclipse.jdt.core.prefs | 386 -- .../.settings/org.eclipse.jdt.ui.prefs | 7 - invertedindex/pom.xml | 103 - .../invertedindex/IICapabilityChecker.java | 182 - .../kylin/invertedindex/IIDescManager.java | 251 -- .../apache/kylin/invertedindex/IIInstance.java | 369 -- .../apache/kylin/invertedindex/IIManager.java | 272 -- .../apache/kylin/invertedindex/IISegment.java | 279 -- .../invertedindex/index/BatchSliceMaker.java | 95 - .../invertedindex/index/BitMapContainer.java | 227 -- .../index/ColumnValueContainer.java | 43 - .../index/CompressedValueContainer.java | 184 - .../index/IncrementalSliceMaker.java | 123 - .../invertedindex/index/RawTableRecord.java | 133 - .../index/ShardingSliceBuilder.java | 53 - .../apache/kylin/invertedindex/index/Slice.java | 215 - .../kylin/invertedindex/index/SliceBuilder.java | 78 - .../kylin/invertedindex/index/TableRecord.java | 193 - .../invertedindex/index/TableRecordInfo.java | 147 - .../index/TableRecordInfoDigest.java | 175 - .../invertedindex/measure/FixedHLLCodec.java | 78 - .../measure/FixedLenMeasureCodec.java | 50 - .../measure/FixedPointLongCodec.java | 117 - .../kylin/invertedindex/model/IIDesc.java | 418 -- .../model/IIJoinedFlatTableDesc.java | 68 - .../invertedindex/model/IIKeyValueCodec.java | 235 -- .../model/IIKeyValueCodecWithState.java | 100 - .../apache/kylin/invertedindex/model/IIRow.java | 90 - .../invertedindex/model/KeyValueCodec.java | 48 - .../invertedindex/util/IIDictionaryBuilder.java | 86 - .../kylin/invertedindex/IIDescManagerTest.java | 103 - .../apache/kylin/invertedindex/IIDescTest.java | 66 - .../kylin/invertedindex/IIInstanceTest.java | 70 - .../invertedindex/InvertedIndexLocalTest.java | 264 -- .../org/apache/kylin/invertedindex/LZFTest.java | 49 - .../measure/FixedPointLongCodecTest.java | 62 - kylin-it/pom.xml | 6 - .../apache/kylin/query/ITCombinationTest.java | 1 - .../org/apache/kylin/query/ITIIQueryTest.java | 92 - .../apache/kylin/query/ITKylinQueryTest.java | 1 - .../hbase/ii/ITInvertedIndexHBaseTest.java | 115 - pom.xml | 1 - query/pom.xml | 6 - .../apache/kylin/query/routing/Candidate.java | 1 - .../apache/kylin/rest/service/BasicService.java | 10 - .../apache/kylin/rest/service/CacheService.java | 22 - .../apache/kylin/rest/service/ModelService.java | 9 - .../kylin/rest/service/ServiceTestBase.java | 4 - storage-hbase/pom.xml | 6 - .../kylin/storage/hbase/HBaseStorage.java | 17 +- .../coprocessor/CoprocessorProjector.java | 16 +- .../common/coprocessor/CoprocessorRowType.java | 12 - .../hbase/cube/v1/HBaseClientKVIterator.java | 94 - .../kylin/storage/hbase/ii/IIBulkLoadJob.java | 65 - .../storage/hbase/ii/IICreateHFileJob.java | 88 - .../storage/hbase/ii/IICreateHFileMapper.java | 71 - .../storage/hbase/ii/IICreateHTableJob.java | 158 - .../hbase/ii/InvertedIndexStorageQuery.java | 67 - .../endpoint/BitMapFilterEvaluator.java | 249 -- .../endpoint/ClearTextDictionary.java | 66 - .../endpoint/EndpointAggregationCache.java | 46 - .../endpoint/EndpointAggregators.java | 287 -- .../coprocessor/endpoint/EndpointEnabler.java | 53 - .../endpoint/EndpointTupleConverter.java | 94 - .../endpoint/EndpointTupleIterator.java | 364 -- .../endpoint/HbaseServerKVIterator.java | 104 - .../ii/coprocessor/endpoint/IIEndpoint.java | 336 -- .../coprocessor/endpoint/IIResponseAdapter.java | 24 - .../coprocessor/endpoint/LocalDictionary.java | 74 - .../endpoint/SliceBitMapProvider.java | 55 - .../endpoint/generated/IIProtos.java | 3773 ------------------ .../ii/coprocessor/endpoint/protobuf/II.proto | 65 - .../storage/hbase/steps/HBaseMROutput.java | 18 - .../kylin/storage/hbase/steps/HBaseMRSteps.java | 78 - .../hbase/util/DeployCoprocessorCLI.java | 35 +- .../hbase/util/IIDeployCoprocessorCLI.java | 157 - .../storage/hbase/util/StorageCleanupJob.java | 16 - .../storage/hbase/util/UpdateHTableHostCLI.java | 15 - .../hbase/common/TsConditionEraserTest.java | 160 - .../endpoint/BitMapFilterEvaluatorTest.java | 228 -- .../endpoint/EndpointAggregationTest.java | 177 - .../endpoint/TableRecordInfoTest.java | 63 - .../endpoint/TsConditionExtractorTest.java | 214 - 111 files changed, 11 insertions(+), 14351 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/assembly/pom.xml ---------------------------------------------------------------------- diff --git a/assembly/pom.xml b/assembly/pom.xml index 3904225..fc1ef63 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -56,12 +56,6 @@ <version>${project.parent.version}</version> </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-invertedindex</artifactId> - <version>${project.parent.version}</version> - </dependency> - <!-- Env & Test --> <dependency> <groupId>org.apache.kylin</groupId> http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IICLI.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IICLI.java b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IICLI.java deleted file mode 100644 index 7e7be34..0000000 --- a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IICLI.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.job.hadoop.invertedindex; - -import java.io.IOException; -import java.util.Iterator; - -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.SequenceFile.Reader; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.engine.mr.HadoopUtil; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.index.RawTableRecord; -import org.apache.kylin.invertedindex.index.Slice; -import org.apache.kylin.invertedindex.index.TableRecord; -import org.apache.kylin.invertedindex.index.TableRecordInfo; -import org.apache.kylin.invertedindex.model.IIKeyValueCodec; -import org.apache.kylin.invertedindex.model.IIRow; - -/** - * @author yangli9 - */ -public class IICLI { - - public static void main(String[] args) throws IOException { - Configuration hconf = HadoopUtil.getCurrentConfiguration(); - IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); - - String iiName = args[0]; - IIInstance ii = mgr.getII(iiName); - - String path = args[1]; - System.out.println("Reading from " + path + " ..."); - - TableRecordInfo info = new TableRecordInfo(ii.getFirstSegment()); - IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest()); - int count = 0; - for (Slice slice : codec.decodeKeyValue(readSequenceKVs(hconf, path))) { - for (RawTableRecord rec : slice) { - System.out.printf(new TableRecord(rec, info).toString()); - count++; - } - } - System.out.println("Total " + count + " records"); - } - - public static Iterable<IIRow> readSequenceKVs(Configuration hconf, String path) throws IOException { - final Reader reader = new Reader(hconf, SequenceFile.Reader.file(new Path(path))); - return new Iterable<IIRow>() { - @Override - public Iterator<IIRow> iterator() { - return new Iterator<IIRow>() { - ImmutableBytesWritable k = new ImmutableBytesWritable(); - ImmutableBytesWritable v = new ImmutableBytesWritable(); - IIRow pair = new IIRow(k, v, null); - - @Override - public boolean hasNext() { - boolean hasNext = false; - try { - hasNext = reader.next(k, v); - } catch (IOException e) { - throw new RuntimeException(e); - } finally { - if (hasNext == false) { - IOUtils.closeQuietly(reader); - } - } - return hasNext; - } - - @Override - public IIRow next() { - return pair; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - }; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java ---------------------------------------------------------------------- diff --git a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java b/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java deleted file mode 100644 index 3390586..0000000 --- a/assembly/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.job.hadoop.invertedindex; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Queue; -import java.util.Set; - -import javax.annotation.Nullable; - -import org.apache.commons.lang.NotImplementedException; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.regionserver.RegionScanner; -import org.apache.kylin.common.util.FIFOIterable; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.common.util.Pair; -import org.apache.kylin.common.util.StreamingBatch; -import org.apache.kylin.common.util.StreamingMessage; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.invertedindex.index.Slice; -import org.apache.kylin.invertedindex.index.SliceBuilder; -import org.apache.kylin.invertedindex.index.TableRecordInfo; -import org.apache.kylin.invertedindex.index.TableRecordInfoDigest; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.invertedindex.model.IIKeyValueCodec; -import org.apache.kylin.invertedindex.model.IIKeyValueCodecWithState; -import org.apache.kylin.invertedindex.model.IIRow; -import org.apache.kylin.invertedindex.model.KeyValueCodec; -import org.apache.kylin.metadata.filter.ColumnTupleFilter; -import org.apache.kylin.metadata.filter.CompareTupleFilter; -import org.apache.kylin.metadata.filter.ConstantTupleFilter; -import org.apache.kylin.metadata.filter.TupleFilter; -import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.model.ParameterDesc; -import org.apache.kylin.metadata.model.TblColRef; -import org.apache.kylin.source.kafka.StreamingParser; -import org.apache.kylin.source.kafka.StringStreamingParser; -import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter; -import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector; -import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType; -import org.apache.kylin.storage.hbase.common.coprocessor.FilterDecorator; -import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.ClearTextDictionary; -import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.EndpointAggregators; -import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.IIEndpoint; -import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.generated.IIProtos; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import com.google.common.base.Function; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -import kafka.message.Message; -import kafka.message.MessageAndOffset; - -/** - */ -public class IITest extends LocalFileMetadataTestCase { - - String iiName = "test_kylin_ii_inner_join"; - IIInstance ii; - IIDesc iiDesc; - - List<IIRow> iiRows; - - final String[] inputData = new String[] { // - "FP-non GTC,0,15,145970,0,28,Toys,2008-10-08 07:18:40,USER_Y,Toys & Hobbies,Models & Kits,Automotive,0,Ebay,USER_S,15,Professional-Other,2012-08-16,2012-08-11,0,2012-08-16,145970,10000329,26.8551,0", // - "ABIN,0,-99,43479,0,21,Photo,2012-09-11 20:26:04,USER_Y,Cameras & Photo,Film Photography,Other,0,Ebay,USER_S,-99,Not Applicable,2012-08-16,2012-08-11,0,2012-08-16,43479,10000807,26.2474,0", // - "ABIN,0,16,80053,0,12,Computers,2012-06-19 21:15:09,USER_Y,Computers/Tablets & Networking,MonitorProjectors & Accs,Monitors,0,Ebay,USER_S,16,Consumer-Other,2012-08-16,2012-08-11,0,2012-08-16,80053,10000261,94.2273,0" }; - - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - this.ii = IIManager.getInstance(getTestConfig()).getII(iiName); - this.iiDesc = ii.getDescriptor(); - - List<MessageAndOffset> messages = Lists.transform(Arrays.asList(inputData), new Function<String, MessageAndOffset>() { - @Nullable - @Override - public MessageAndOffset apply(String input) { - return new MessageAndOffset(new Message(input.getBytes()), System.currentTimeMillis()); - } - }); - - final StreamingParser parser = StringStreamingParser.instance; - final List<StreamingMessage> streamingMessages = Lists.transform(messages, new Function<MessageAndOffset, StreamingMessage>() { - @Nullable - @Override - public StreamingMessage apply(@Nullable MessageAndOffset input) { - return parser.parse(input); - } - }); - StreamingBatch batch = new StreamingBatch(streamingMessages, Pair.newPair(0L, System.currentTimeMillis())); - - iiRows = Lists.newArrayList(); - final Slice slice = new SliceBuilder(iiDesc, (short) 0).buildSlice((batch)); - IIKeyValueCodec codec = new IIKeyValueCodec(slice.getInfo()); - for (IIRow iiRow : codec.encodeKeyValue(slice)) { - iiRows.add(iiRow); - } - } - - @After - public void after() throws Exception { - cleanupTestMetadata(); - } - - /** - * simulate stream building into slices, and encode the slice into IIRows. - * Then reconstruct the IIRows to slice. - */ - @Test - public void basicTest() { - Queue<IIRow> buffer = Lists.newLinkedList(); - FIFOIterable bufferIterable = new FIFOIterable(buffer); - TableRecordInfo info = new TableRecordInfo(iiDesc); - TableRecordInfoDigest digest = info.getDigest(); - KeyValueCodec codec = new IIKeyValueCodecWithState(digest); - Iterator<Slice> slices = codec.decodeKeyValue(bufferIterable).iterator(); - - Assert.assertTrue(!slices.hasNext()); - Assert.assertEquals(iiRows.size(), digest.getColumnCount()); - - for (int i = 0; i < digest.getColumnCount(); ++i) { - buffer.add(iiRows.get(i)); - - if (i != digest.getColumnCount() - 1) { - Assert.assertTrue(!slices.hasNext()); - } else { - Assert.assertTrue(slices.hasNext()); - } - } - - Slice newSlice = slices.next(); - Assert.assertEquals(newSlice.getLocalDictionaries()[0].getSize(), 2); - } - - @Test - public void IIEndpointTest() { - TableRecordInfo info = new TableRecordInfo(ii.getDescriptor()); - if (ii.getFirstSegment() == null) { - IISegment segment = IIManager.getInstance(getTestConfig()).buildSegment(ii, 0, System.currentTimeMillis()); - ii.getSegments().add(segment); - } - CoprocessorRowType type = CoprocessorRowType.fromTableRecordInfo(info, ii.getFirstSegment().getColumns()); - CoprocessorProjector projector = CoprocessorProjector.makeForEndpoint(info, Collections.singletonList(ii.getDescriptor().findColumnRef("default.test_kylin_fact", "lstg_format_name"))); - - FunctionDesc f1 = new FunctionDesc(); - f1.setExpression("SUM"); - ParameterDesc p1 = new ParameterDesc(); - p1.setType("column"); - p1.setValue("PRICE"); - f1.setParameter(p1); - f1.setReturnType("decimal(19,4)"); - - TblColRef column = ii.getDescriptor().findColumnRef("default.test_kylin_fact", "cal_dt"); - CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GTE); - ColumnTupleFilter columnFilter = new ColumnTupleFilter(column); - compareFilter.addChild(columnFilter); - ConstantTupleFilter constantFilter = null; - constantFilter = new ConstantTupleFilter(("2012-08-16")); - compareFilter.addChild(constantFilter); - - EndpointAggregators aggregators = EndpointAggregators.fromFunctions(info, Collections.singletonList(f1)); - CoprocessorFilter filter = CoprocessorFilter.fromFilter(new ClearTextDictionary(info), compareFilter, FilterDecorator.FilterConstantsTreatment.AS_IT_IS); - - final Iterator<IIRow> iiRowIterator = iiRows.iterator(); - - IIEndpoint endpoint = new IIEndpoint(); - IIProtos.IIResponseInternal response = endpoint.getResponse(new RegionScanner() { - @Override - public HRegionInfo getRegionInfo() { - throw new NotImplementedException(); - } - - @Override - public boolean isFilterDone() throws IOException { - throw new NotImplementedException(); - } - - @Override - public boolean reseek(byte[] row) throws IOException { - throw new NotImplementedException(); - } - - @Override - public long getMaxResultSize() { - throw new NotImplementedException(); - - } - - @Override - public long getMvccReadPoint() { - throw new NotImplementedException(); - } - - @Override - public boolean nextRaw(List<Cell> result) throws IOException { - if (iiRowIterator.hasNext()) { - IIRow iiRow = iiRowIterator.next(); - result.addAll(iiRow.makeCells()); - return true; - } else { - return false; - } - } - - @Override - public boolean nextRaw(List<Cell> result, int limit) throws IOException { - throw new NotImplementedException(); - } - - @Override - public boolean next(List<Cell> results) throws IOException { - throw new NotImplementedException(); - } - - @Override - public boolean next(List<Cell> result, int limit) throws IOException { - throw new NotImplementedException(); - } - - @Override - public void close() throws IOException { - throw new NotImplementedException(); - } - }, type, projector, aggregators, filter); - - Assert.assertEquals(2, response.getRowsList().size()); - System.out.println(response.getRowsList().size()); - Set<String> answers = Sets.newHashSet("120.4747", "26.8551"); - for (IIProtos.IIResponseInternal.IIRow responseRow : response.getRowsList()) { - ByteBuffer bf = responseRow.getMeasures().asReadOnlyByteBuffer(); - List<Object> metrics = aggregators.deserializeMetricValues(bf); - Assert.assertTrue(answers.contains(metrics.get(0))); - answers.remove(metrics.get(0)); - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 84c4db9..bfbaaa4 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -203,7 +203,7 @@ abstract public class KylinConfigBase implements Serializable { public String[] getRealizationProviders() { return getOptionalStringArray("kylin.realization.providers", // - new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager", "org.apache.kylin.invertedindex.IIManager" }); + new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" }); } public CliCommandExecutor getCliCommandExecutor() throws IOException { http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java index cf00c05..99a9695 100644 --- a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java +++ b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceStore.java @@ -42,9 +42,7 @@ abstract public class ResourceStore { private static final Logger logger = LoggerFactory.getLogger(ResourceStore.class); public static final String CUBE_RESOURCE_ROOT = "/cube"; - public static final String II_RESOURCE_ROOT = "/invertedindex"; public static final String CUBE_DESC_RESOURCE_ROOT = "/cube_desc"; - public static final String II_DESC_RESOURCE_ROOT = "/invertedindex_desc"; public static final String DATA_MODEL_DESC_RESOURCE_ROOT = "/model_desc"; public static final String DICT_RESOURCE_ROOT = "/dict"; public static final String PROJECT_RESOURCE_ROOT = "/project"; http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java index ef89754..3c63c41 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java @@ -54,7 +54,6 @@ public class ClassUtil { classRenameMap.put("org.apache.kylin.job.common.HadoopShellExecutable", "org.apache.kylin.engine.mr.common.HadoopShellExecutable"); classRenameMap.put("org.apache.kylin.job.common.MapReduceExecutable", "org.apache.kylin.engine.mr.common.MapReduceExecutable"); classRenameMap.put("org.apache.kylin.job.cube.CubingJob", "org.apache.kylin.engine.mr.CubingJob"); - classRenameMap.put("org.apache.kylin.job.invertedindex.IIJob", "org.apache.kylin.engine.mr.invertedindex.IIJob"); classRenameMap.put("org.apache.kylin.job.cube.GarbageCollectionStep", "org.apache.kylin.storage.hbase.steps.DeprecatedGCStep"); classRenameMap.put("org.apache.kylin.job.cube.MergeDictionaryStep", "org.apache.kylin.engine.mr.steps.MergeDictionaryStep"); classRenameMap.put("org.apache.kylin.job.cube.UpdateCubeInfoAfterBuildStep", "org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterBuildStep"); http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java ---------------------------------------------------------------------- diff --git a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java index 684d828..556019f 100644 --- a/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java +++ b/core-common/src/test/java/org/apache/kylin/common/util/AbstractKylinTestCase.java @@ -31,8 +31,6 @@ public abstract class AbstractKylinTestCase { public static final String[] SERVICES_WITH_CACHE = { // "org.apache.kylin.cube.CubeManager", // "org.apache.kylin.cube.CubeDescManager", // - "org.apache.kylin.invertedindex.IIDescManager", // - "org.apache.kylin.invertedindex.IIManager", // "org.apache.kylin.storage.hybrid.HybridManager", // "org.apache.kylin.metadata.realization.RealizationRegistry", // "org.apache.kylin.metadata.project.ProjectManager", // http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/pom.xml ---------------------------------------------------------------------- diff --git a/engine-mr/pom.xml b/engine-mr/pom.xml index 363e459..12a6b73 100644 --- a/engine-mr/pom.xml +++ b/engine-mr/pom.xml @@ -50,11 +50,6 @@ </dependency> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-invertedindex</artifactId> - <version>${project.parent.version}</version> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-job</artifactId> <version>${project.parent.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java index 276af65..beebb4e 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/IMROutput.java @@ -19,7 +19,6 @@ package org.apache.kylin.engine.mr; import org.apache.kylin.cube.CubeSegment; -import org.apache.kylin.invertedindex.IISegment; import org.apache.kylin.job.execution.DefaultChainedExecutable; public interface IMROutput { @@ -27,11 +26,8 @@ public interface IMROutput { /** Return a helper to participate in batch cubing job flow. */ public IMRBatchCubingOutputSide getBatchCubingOutputSide(CubeSegment seg); - /** Return a helper to participate in batch cubing job flow. */ - public IMRBatchInvertedIndexingOutputSide getBatchInvertedIndexingOutputSide(IISegment seg); - /** - * Participate the batch cubing flow as the output side. Responsible for saving + * Participate the batch cubing flow as the output side. Responsible for savg * the cuboid output to storage (Phase 3). * * - Phase 1: Create Flat Table @@ -80,24 +76,4 @@ public interface IMROutput { public void addStepPhase3_Cleanup(DefaultChainedExecutable jobFlow); } - /** - * Participate the batch inverted indexing flow as the output side. Responsible for saving - * the output to storage (Phase 3). - * - * - Phase 1: Create Flat Table - * - Phase 2: Build Dictionary - * - Phase 3: Build II - * - Phase 4: Update Metadata & Cleanup - */ - public interface IMRBatchInvertedIndexingOutputSide { - - /** - * Add step that saves II output from HDFS to storage. - * - */ - public void addStepPhase3_BuildII(DefaultChainedExecutable jobFlow, String rootPath); - - /** Add step that does any necessary clean up. */ - public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow); - } } http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java index 5228088..5a82fe5 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/JobBuilderSupport.java @@ -25,7 +25,6 @@ import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.engine.mr.common.BatchConstants; import org.apache.kylin.engine.mr.common.HadoopShellExecutable; import org.apache.kylin.engine.mr.common.MapReduceExecutable; -import org.apache.kylin.engine.mr.invertedindex.UpdateIIInfoAfterBuildStep; import org.apache.kylin.engine.mr.steps.CreateDictionaryJob; import org.apache.kylin.engine.mr.steps.CubingExecutableUtil; import org.apache.kylin.engine.mr.steps.FactDistinctColumnsJob; @@ -130,14 +129,6 @@ public class JobBuilderSupport { return result; } - public UpdateIIInfoAfterBuildStep createUpdateIIInfoAfterBuildStep(String jobId) { - final UpdateIIInfoAfterBuildStep updateIIInfoStep = new UpdateIIInfoAfterBuildStep(); - updateIIInfoStep.setName(ExecutableConstants.STEP_NAME_UPDATE_II_INFO); - updateIIInfoStep.setInvertedIndexName(seg.getRealization().getName()); - updateIIInfoStep.setJobId(jobId); - return updateIIInfoStep; - } - // ============================================================================ public String getJobWorkingDir(String jobId) { http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java index b67dc38..abe05d7 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java @@ -29,7 +29,6 @@ import org.apache.kylin.engine.mr.IMROutput.IMRBatchCubingOutputSide; import org.apache.kylin.engine.mr.IMROutput.IMRBatchMergeOutputSide; import org.apache.kylin.engine.mr.IMROutput2.IMRBatchCubingOutputSide2; import org.apache.kylin.engine.mr.IMROutput2.IMRBatchMergeOutputSide2; -import org.apache.kylin.invertedindex.IISegment; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.realization.IRealizationSegment; @@ -70,10 +69,6 @@ public class MRUtil { return StorageFactory.createEngineAdapter(seg, IMROutput2.class).getBatchMergeOutputSide(seg); } - public static IMROutput.IMRBatchInvertedIndexingOutputSide getBatchInvertedIndexingOutputSide(IISegment seg) { - return StorageFactory.createEngineAdapter(seg, IMROutput.class).getBatchInvertedIndexingOutputSide(seg); - } - // use this method instead of ToolRunner.run() because ToolRunner.run() is not thread-sale // Refer to: http://stackoverflow.com/questions/22462665/is-hadoops-toorunner-thread-safe public static int runMRJob(Tool tool, String[] args) throws Exception { http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java index 9b5ed67..bafe65b 100644 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java +++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java @@ -63,7 +63,6 @@ import org.apache.kylin.common.util.StringSplitter; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.engine.mr.HadoopUtil; -import org.apache.kylin.invertedindex.IIInstance; import org.apache.kylin.job.JobInstance; import org.apache.kylin.job.exception.JobException; import org.apache.kylin.metadata.MetadataManager; @@ -441,23 +440,6 @@ public abstract class AbstractHadoopJob extends Configured implements Tool { attachKylinPropsAndMetadata(dumpList, cube.getConfig(), conf); } - protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException { - MetadataManager metaMgr = MetadataManager.getInstance(ii.getConfig()); - - // write II / model_desc / II_desc / dict / table - ArrayList<String> dumpList = new ArrayList<String>(); - dumpList.add(ii.getResourcePath()); - dumpList.add(ii.getDescriptor().getModel().getResourcePath()); - dumpList.add(ii.getDescriptor().getResourcePath()); - - for (String tableName : ii.getDescriptor().getModel().getAllTables()) { - TableDesc table = metaMgr.getTableDesc(tableName); - dumpList.add(table.getResourcePath()); - } - - attachKylinPropsAndMetadata(dumpList, ii.getConfig(), conf); - } - protected void attachKylinPropsAndMetadata(ArrayList<String> dumpList, KylinConfig kylinConfig, Configuration conf) throws IOException { File tmp = File.createTempFile("kylin_job_meta", ""); FileUtils.forceDelete(tmp); // we need a directory, so delete the file first http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java deleted file mode 100644 index e582fa0..0000000 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/BatchIIJobBuilder.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.engine.mr.invertedindex; - -import org.apache.kylin.engine.mr.IMRInput.IMRBatchCubingInputSide; -import org.apache.kylin.engine.mr.IMROutput; -import org.apache.kylin.engine.mr.JobBuilderSupport; -import org.apache.kylin.engine.mr.MRUtil; -import org.apache.kylin.engine.mr.common.BatchConstants; -import org.apache.kylin.engine.mr.common.MapReduceExecutable; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.job.constant.ExecutableConstants; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BatchIIJobBuilder extends JobBuilderSupport { - - private static final Logger logger = LoggerFactory.getLogger(BatchIIJobBuilder.class); - - private final IMRBatchCubingInputSide inputSide; - private final IMROutput.IMRBatchInvertedIndexingOutputSide outputSide; - - public BatchIIJobBuilder(IISegment newSegment, String submitter) { - super(newSegment, submitter); - this.inputSide = MRUtil.getBatchCubingInputSide(newSegment); - this.outputSide = MRUtil.getBatchInvertedIndexingOutputSide(newSegment); - } - - public IIJob build() { - logger.info("MR new job to BUILD segment " + seg); - - final IIJob result = IIJob.createBuildJob((IISegment) seg, submitter, config); - final String jobId = result.getId(); - - final String iiRootPath = getRealizationRootPath(jobId) + "/"; - // Phase 1: Create Flat Table - inputSide.addStepPhase1_CreateFlatTable(result); - - // Phase 2: Build Inverted Index - result.addTask(createInvertedIndexStep((IISegment) seg, iiRootPath)); - outputSide.addStepPhase3_BuildII(result, iiRootPath); - - // Phase 3: Update Metadata & Cleanup - result.addTask(createUpdateIIInfoAfterBuildStep(jobId)); - inputSide.addStepPhase4_Cleanup(result); - outputSide.addStepPhase4_Cleanup(result); - - return result; - } - - private MapReduceExecutable createInvertedIndexStep(IISegment seg, String iiOutputTempPath) { - MapReduceExecutable buildIIStep = new MapReduceExecutable(); - - StringBuilder cmd = new StringBuilder(); - appendMapReduceParameters(cmd); - - buildIIStep.setName(ExecutableConstants.STEP_NAME_BUILD_II); - - appendExecCmdParameters(cmd, BatchConstants.ARG_II_NAME, seg.getRealization().getName()); - appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, iiOutputTempPath); - appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, ExecutableConstants.STEP_NAME_BUILD_II); - - buildIIStep.setMapReduceParams(cmd.toString()); - buildIIStep.setMapReduceJobClass(InvertedIndexJob.class); - return buildIIStep; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIJob.java deleted file mode 100644 index 7c0748a..0000000 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/IIJob.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.engine.mr.invertedindex; - -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.TimeZone; - -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.job.engine.JobEngineConfig; -import org.apache.kylin.job.execution.DefaultChainedExecutable; - -/** - */ -public class IIJob extends DefaultChainedExecutable { - - public IIJob() { - super(); - } - - private static final String II_INSTANCE_NAME = "iiName"; - private static final String SEGMENT_ID = "segmentId"; - - void setIIName(String name) { - setParam(II_INSTANCE_NAME, name); - } - - public String getIIName() { - return getParam(II_INSTANCE_NAME); - } - - void setSegmentId(String segmentId) { - setParam(SEGMENT_ID, segmentId); - } - - public String getSegmentId() { - return getParam(SEGMENT_ID); - } - - public static IIJob createBuildJob(IISegment seg, String submitter, JobEngineConfig config) { - return initialJob(seg, "BUILD", submitter, config); - } - - private static IIJob initialJob(IISegment seg, String type, String submitter, JobEngineConfig config) { - IIJob result = new IIJob(); - SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss"); - format.setTimeZone(TimeZone.getTimeZone(config.getTimeZone())); - result.setIIName(seg.getIIInstance().getName()); - result.setSegmentId(seg.getUuid()); - result.setName(seg.getIIInstance().getName() + " - " + seg.getName() + " - " + type + " - " + format.format(new Date(System.currentTimeMillis()))); - result.setSubmitter(submitter); - return result; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java deleted file mode 100644 index f7adf6b..0000000 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexJob.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.engine.mr.invertedindex; - -import java.io.IOException; - -import org.apache.commons.cli.Options; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.engine.mr.IMRInput; -import org.apache.kylin.engine.mr.MRUtil; -import org.apache.kylin.engine.mr.common.AbstractHadoopJob; -import org.apache.kylin.engine.mr.common.BatchConstants; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.IISegment; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - */ -public class InvertedIndexJob extends AbstractHadoopJob { - protected static final Logger logger = LoggerFactory.getLogger(InvertedIndexJob.class); - - @Override - public int run(String[] args) throws Exception { - Options options = new Options(); - - try { - options.addOption(OPTION_JOB_NAME); - options.addOption(OPTION_II_NAME); - options.addOption(OPTION_OUTPUT_PATH); - parseOptions(options, args); - - job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME)); - String iiname = getOptionValue(OPTION_II_NAME); - Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH)); - - // ---------------------------------------------------------------------------- - - logger.info("Starting: " + job.getJobName()); - - IIInstance ii = getII(iiname); - short sharding = ii.getDescriptor().getSharding(); - - setJobClasspath(job, ii.getConfig()); - - setupMapper(ii.getFirstSegment()); - setupReducer(output, sharding); - attachMetadata(ii); - - return waitForCompletion(job); - - } catch (Exception e) { - printUsage(options); - throw e; - } finally { - if (job != null) - cleanupTempConfFile(job.getConfiguration()); - } - - } - - private IIInstance getII(String iiName) { - IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); - IIInstance ii = mgr.getII(iiName); - if (ii == null) - throw new IllegalArgumentException("No Inverted Index found by name " + iiName); - return ii; - } - - private void attachMetadata(IIInstance ii) throws IOException { - - Configuration conf = job.getConfiguration(); - attachKylinPropsAndMetadata(ii, conf); - - IISegment seg = ii.getFirstSegment(); - conf.set(BatchConstants.CFG_II_NAME, ii.getName()); - conf.set(BatchConstants.CFG_II_SEGMENT_NAME, seg.getName()); - } - - private void setupMapper(IISegment segment) throws IOException { - - IMRInput.IMRTableInputFormat flatTableInputFormat = MRUtil.getBatchCubingInputSide(segment).getFlatTableInputFormat(); - flatTableInputFormat.configureJob(job); - - job.setMapperClass(InvertedIndexMapper.class); - job.setMapOutputKeyClass(LongWritable.class); - job.setPartitionerClass(InvertedIndexPartitioner.class); - } - - private void setupReducer(Path output, short sharding) throws IOException { - job.setReducerClass(InvertedIndexReducer.class); - job.setOutputFormatClass(SequenceFileOutputFormat.class); - job.setOutputKeyClass(ImmutableBytesWritable.class); - job.setOutputValueClass(ImmutableBytesWritable.class); - - job.setNumReduceTasks(sharding); - - FileOutputFormat.setOutputPath(job, output); - - job.getConfiguration().set(BatchConstants.CFG_OUTPUT_PATH, output.toString()); - - deletePath(job.getConfiguration(), output); - } - - public static void main(String[] args) throws Exception { - InvertedIndexJob job = new InvertedIndexJob(); - int exitCode = ToolRunner.run(job, args); - System.exit(exitCode); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexMapper.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexMapper.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexMapper.java deleted file mode 100644 index 27e2470..0000000 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexMapper.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.engine.mr.invertedindex; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Writable; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.DateFormat; -import org.apache.kylin.engine.mr.IMRInput; -import org.apache.kylin.engine.mr.KylinMapper; -import org.apache.kylin.engine.mr.MRUtil; -import org.apache.kylin.engine.mr.common.AbstractHadoopJob; -import org.apache.kylin.engine.mr.common.BatchConstants; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.invertedindex.index.TableRecordInfo; -import org.apache.kylin.metadata.model.SegmentStatusEnum; - -/** - * @author yangli9 - */ -public class InvertedIndexMapper<KEYIN> extends KylinMapper<KEYIN, Object, LongWritable, Writable> { - - private TableRecordInfo info; - - private LongWritable outputKey; - private IMRInput.IMRTableInputFormat flatTableInputFormat; - - @Override - protected void setup(Context context) throws IOException { - super.bindCurrentConfiguration(context.getConfiguration()); - - Configuration conf = context.getConfiguration(); - - KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); - IIManager mgr = IIManager.getInstance(config); - IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME)); - IISegment seg = ii.getSegment(conf.get(BatchConstants.CFG_II_SEGMENT_NAME), SegmentStatusEnum.NEW); - this.info = new TableRecordInfo(seg); - - outputKey = new LongWritable(); - - flatTableInputFormat = MRUtil.getBatchCubingInputSide(ii.getFirstSegment()).getFlatTableInputFormat(); - } - - @Override - public void map(KEYIN key, Object record, Context context) throws IOException, InterruptedException { - - Writable writableRecord = (Writable) record; - String[] row = flatTableInputFormat.parseMapperInput(writableRecord); - String timestampString = row[info.getTimestampColumn()]; - - outputKey.set(DateFormat.stringToMillis(timestampString)); - // - context.write(outputKey, writableRecord); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexPartitioner.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexPartitioner.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexPartitioner.java deleted file mode 100644 index dcf707f..0000000 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexPartitioner.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.engine.mr.invertedindex; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.mapreduce.Partitioner; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.engine.mr.common.AbstractHadoopJob; -import org.apache.kylin.engine.mr.common.BatchConstants; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.invertedindex.index.TableRecord; -import org.apache.kylin.invertedindex.index.TableRecordInfo; -import org.apache.kylin.metadata.model.SegmentStatusEnum; - -/** - * @author yangli9 - */ -public class InvertedIndexPartitioner extends Partitioner<LongWritable, ImmutableBytesWritable> implements Configurable { - - private Configuration conf; - private TableRecordInfo info; - private TableRecord rec; - - @Override - public int getPartition(LongWritable key, ImmutableBytesWritable value, int numPartitions) { - rec.setBytes(value.get(), value.getOffset(), value.getLength()); - return rec.getShard(); - } - - @Override - public void setConf(Configuration conf) { - this.conf = conf; - try { - KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); - IIManager mgr = IIManager.getInstance(config); - IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME)); - IISegment seg = ii.getSegment(conf.get(BatchConstants.CFG_II_SEGMENT_NAME), SegmentStatusEnum.NEW); - this.info = new TableRecordInfo(seg); - this.rec = this.info.createTableRecord(); - } catch (IOException e) { - throw new RuntimeException("", e); - } - } - - @Override - public Configuration getConf() { - return conf; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java deleted file mode 100644 index e7e760a..0000000 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/InvertedIndexReducer.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.engine.mr.invertedindex; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.io.LongWritable; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.util.Pair; -import org.apache.kylin.common.util.StreamingBatch; -import org.apache.kylin.common.util.StreamingMessage; -import org.apache.kylin.engine.mr.IMRInput; -import org.apache.kylin.engine.mr.KylinReducer; -import org.apache.kylin.engine.mr.MRUtil; -import org.apache.kylin.engine.mr.common.AbstractHadoopJob; -import org.apache.kylin.engine.mr.common.BatchConstants; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.invertedindex.index.Slice; -import org.apache.kylin.invertedindex.index.SliceBuilder; -import org.apache.kylin.invertedindex.index.TableRecordInfo; -import org.apache.kylin.invertedindex.model.IIKeyValueCodec; -import org.apache.kylin.invertedindex.model.IIRow; - -import com.google.common.collect.Lists; - -/** - */ -public class InvertedIndexReducer extends KylinReducer<LongWritable, Object, ImmutableBytesWritable, ImmutableBytesWritable> { - - private TableRecordInfo info; - private IIKeyValueCodec kv; - private IMRInput.IMRTableInputFormat flatTableInputFormat; - private SliceBuilder sliceBuilder; - private ArrayList<StreamingMessage> messages; - private int sliceSize; - private ImmutableBytesWritable immutableBytesWritable; - private ByteBuffer valueBuf; - - @Override - protected void setup(Context context) throws IOException { - super.bindCurrentConfiguration(context.getConfiguration()); - - Configuration conf = context.getConfiguration(); - KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); - IIManager mgr = IIManager.getInstance(config); - IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME)); - IISegment seg = ii.getFirstSegment(); - info = new TableRecordInfo(seg); - kv = new IIKeyValueCodec(info.getDigest()); - flatTableInputFormat = MRUtil.getBatchCubingInputSide(ii.getFirstSegment()).getFlatTableInputFormat(); - sliceSize = ii.getDescriptor().getSliceSize(); - short shard = (short) context.getTaskAttemptID().getTaskID().getId(); - System.out.println("Generating to shard - " + shard); - sliceBuilder = new SliceBuilder(seg.getIIDesc(), shard); - messages = Lists.newArrayListWithCapacity(sliceSize); - immutableBytesWritable = new ImmutableBytesWritable(); - valueBuf = ByteBuffer.allocate(1024 * 1024); // 1MB - } - - @Override - public void reduce(LongWritable key, Iterable<Object> values, Context context) // - throws IOException, InterruptedException { - for (Object v : values) { - String[] row = flatTableInputFormat.parseMapperInput(v); - messages.add((parse(row))); - if (messages.size() >= sliceSize) { - buildAndOutput(new StreamingBatch(messages, Pair.newPair(System.currentTimeMillis(), System.currentTimeMillis())), context); - messages = Lists.newArrayList(); - } - } - } - - private StreamingMessage parse(String[] row) { - return new StreamingMessage(Lists.newArrayList(row), System.currentTimeMillis(), System.currentTimeMillis(), Collections.<String, Object> emptyMap()); - } - - @Override - protected void cleanup(Context context) throws IOException, InterruptedException { - if (!messages.isEmpty()) { - buildAndOutput(new StreamingBatch(messages, Pair.newPair(System.currentTimeMillis(), System.currentTimeMillis())), context); - messages.clear(); - } - - } - - private void buildAndOutput(StreamingBatch streamingBatch, Context context) throws IOException, InterruptedException { - final Slice slice = sliceBuilder.buildSlice(streamingBatch); - ImmutableBytesWritable value, dictionary; - for (IIRow pair : kv.encodeKeyValue(slice)) { - value = pair.getValue(); - dictionary = pair.getDictionary(); - int newLength = 4 + value.getLength() + dictionary.getLength(); - if (newLength > valueBuf.limit()) { - valueBuf = ByteBuffer.allocate(newLength); - } - valueBuf.clear(); - valueBuf.putInt(value.getLength()); - valueBuf.put(value.get(), value.getOffset(), value.getLength()); - valueBuf.put(dictionary.get(), dictionary.getOffset(), dictionary.getLength()); - immutableBytesWritable.set(valueBuf.array(), 0, newLength); - context.write(pair.getKey(), immutableBytesWritable); - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/UpdateIIInfoAfterBuildStep.java ---------------------------------------------------------------------- diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/UpdateIIInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/UpdateIIInfoAfterBuildStep.java deleted file mode 100644 index a509c1f..0000000 --- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/invertedindex/UpdateIIInfoAfterBuildStep.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.engine.mr.invertedindex; - -import java.io.IOException; - -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.job.exception.ExecuteException; -import org.apache.kylin.job.execution.AbstractExecutable; -import org.apache.kylin.job.execution.ExecutableContext; -import org.apache.kylin.job.execution.ExecuteResult; -import org.apache.kylin.metadata.model.SegmentStatusEnum; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - */ -public class UpdateIIInfoAfterBuildStep extends AbstractExecutable { - private static final Logger logger = LoggerFactory.getLogger(UpdateIIInfoAfterBuildStep.class); - - private static final String II_NAME = "iiName"; - private static final String JOB_ID = "jobId"; - - public UpdateIIInfoAfterBuildStep() { - super(); - } - - public void setInvertedIndexName(String cubeName) { - this.setParam(II_NAME, cubeName); - } - - private String getInvertedIndexName() { - return getParam(II_NAME); - } - - public void setJobId(String id) { - setParam(JOB_ID, id); - } - - private String getJobId() { - return getParam(JOB_ID); - } - - @Override - protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { - - IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); - IIInstance ii = mgr.getII(getInvertedIndexName()); - IISegment segment = ii.getFirstSegment(); - segment.setStatus(SegmentStatusEnum.READY); - - segment.setLastBuildJobID(getJobId()); - segment.setLastBuildTime(System.currentTimeMillis()); - - try { - mgr.updateII(ii); - return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed"); - } catch (IOException e) { - logger.error("fail to update inverted index after build", e); - return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage()); - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/engine-streaming/pom.xml ---------------------------------------------------------------------- diff --git a/engine-streaming/pom.xml b/engine-streaming/pom.xml index 54955a6..7110377 100644 --- a/engine-streaming/pom.xml +++ b/engine-streaming/pom.xml @@ -43,11 +43,6 @@ </dependency> <dependency> <groupId>org.apache.kylin</groupId> - <artifactId>kylin-invertedindex</artifactId> - <version>${project.parent.version}</version> - </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> <artifactId>kylin-core-storage</artifactId> <version>${project.parent.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json b/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json deleted file mode 100644 index 13fc282..0000000 --- a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_inner_join.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - - "uuid": "dk853e80-41be-49a5-89ca-9fb7294db186", - "name": "test_kylin_ii_inner_join", - "owner": null, - "version": null, - "cost": 10, - "status": "DISABLED", - "segments": [], - "last_modified": 1420016227424, - "descriptor": "test_kylin_ii_inner_join_desc", - "create_time": null, - "input_records_size": 0, - "size_kb": 0, - "source_records_count": 0 -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json b/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json deleted file mode 100644 index 28b23ef..0000000 --- a/examples/test_case_data/localmeta/invertedindex/test_kylin_ii_left_join.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - - "uuid": "daa53e80-41be-49a5-89ca-9fb7294db186", - "name": "test_kylin_ii_left_join", - "owner": null, - "cost": 10, - "status": "DISABLED", - "segments": [], - "last_modified": 1420016227424, - "descriptor": "test_kylin_ii_left_join_desc", - "create_time": null, - "input_records_size": 0, - "size_kb": 0, - "source_records_count": 0 -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json b/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json deleted file mode 100644 index 4ec2711..0000000 --- a/examples/test_case_data/localmeta/invertedindex/test_streaming_table_ii.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - - "uuid": "daa53e80-41be-49a5-89ca-9fb729999812", - "name": "test_streaming_table_ii", - "owner": null, - "cost": 10, - "status": "DISABLED", - "segments": [], - "last_modified": 0, - "descriptor": "test_streaming_table_ii_desc", - "create_time": null, - "input_records_size": 0, - "size_kb": 0, - "source_records_count": 0 -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json deleted file mode 100644 index eb383fb..0000000 --- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_inner_join_desc.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - - "uuid": "88bf87b5-c7b5-4420-a12a-07f6b37b3187", - "last_modified": 0, - "name": "test_kylin_ii_inner_join_desc", - "model_name": "test_kylin_inner_join_model_desc", - "timestamp_dimension": "cal_dt", - "value_dimensions": [ - { - "table": "default.test_kylin_fact", - "columns": [ - "lstg_format_name", - "LSTG_SITE_ID", - "SLR_SEGMENT_CD" - ] - }, - { - "table": "default.test_category_groupings", - "columns": [ - "leaf_categ_id", - "site_id", - "USER_DEFINED_FIELD1", - "USER_DEFINED_FIELD3", - "UPD_DATE", - "UPD_USER", - "meta_categ_name", - "categ_lvl2_name", - "categ_lvl3_name" - ] - }, - { - "table": "edw.test_sites", - "columns": [ - "site_id", - "site_name", - "cre_user" - ] - }, - { - "table": "edw.test_seller_type_dim", - "columns": [ - "seller_type_cd", - "seller_type_desc" - ] - }, - { - "table": "edw.test_cal_dt", - "columns": [ - "cal_dt", - "week_beg_dt" - ] - }, - { - "table": "default.test_kylin_fact", - "columns": [ - "TRANS_ID", - "CAL_DT", - "LEAF_CATEG_ID", - "SELLER_ID" - ] - } - ], - "metrics": [ - "PRICE", - "ITEM_COUNT" - ], - "sharding": 1, - "slice_size": 1000 -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json deleted file mode 100644 index 2f152ad..0000000 --- a/examples/test_case_data/localmeta/invertedindex_desc/test_kylin_ii_left_join_desc.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - - "uuid": "74bf87b5-c7b5-4420-a12a-07f6b37b3187", - "last_modified": 0, - "name": "test_kylin_ii_left_join_desc", - "model_name": "test_kylin_left_join_model_desc", - "timestamp_dimension": "cal_dt", - "value_dimensions": [ - { - "table": "default.test_kylin_fact", - "columns": [ - "lstg_format_name", - "LSTG_SITE_ID", - "SLR_SEGMENT_CD" - ] - }, - { - "table": "default.test_category_groupings", - "columns": [ - "leaf_categ_id", - "site_id", - "USER_DEFINED_FIELD1", - "USER_DEFINED_FIELD3", - "UPD_DATE", - "UPD_USER", - "meta_categ_name", - "categ_lvl2_name", - "categ_lvl3_name" - ] - }, - { - "table": "edw.test_sites", - "columns": [ - "site_id", - "site_name", - "cre_user" - ] - }, - { - "table": "edw.test_seller_type_dim", - "columns": [ - "seller_type_cd", - "seller_type_desc" - ] - }, - { - "table": "edw.test_cal_dt", - "columns": [ - "cal_dt", - "week_beg_dt" - ] - }, - { - "table": "default.test_kylin_fact", - "columns": [ - "TRANS_ID", - "CAL_DT", - "LEAF_CATEG_ID", - "SELLER_ID" - ] - } - ], - "metrics": [ - "PRICE", - "ITEM_COUNT" - ], - "sharding": 1, - "slice_size": 1000 -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json b/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json deleted file mode 100644 index c39ae90..0000000 --- a/examples/test_case_data/localmeta/invertedindex_desc/test_streaming_table_ii_desc.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - - "uuid": "74bf87b5-c7b5-4420-a12a-07f6b3864789", - "last_modified": 0, - "name": "test_streaming_table_ii_desc", - "model_name": "test_streaming_table_model_desc", - "timestamp_dimension": "minute_start", - "value_dimensions": [ - { - "table": "default.streaming_table", - "columns": [ - "minute_start", - "hour_start", - "day_start", - "itm", - "site" - ] - } - ], - "metrics": [ - "gmv", - "item_count" - ], - "sharding": 1, - "slice_size": 1000 -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/examples/test_case_data/localmeta/project/default.json ---------------------------------------------------------------------- diff --git a/examples/test_case_data/localmeta/project/default.json b/examples/test_case_data/localmeta/project/default.json index 88538c9..3026ff8 100644 --- a/examples/test_case_data/localmeta/project/default.json +++ b/examples/test_case_data/localmeta/project/default.json @@ -24,16 +24,6 @@ "realization": "test_kylin_cube_without_slr_left_join_empty" }, { - "name": "test_kylin_ii_left_join", - "type": "INVERTED_INDEX", - "realization": "test_kylin_ii_left_join" - }, - { - "name": "test_kylin_ii_inner_join", - "type": "INVERTED_INDEX", - "realization": "test_kylin_ii_inner_join" - }, - { "name": "test_streaming_table_cube", "type": "CUBE", "realization": "test_streaming_table_cube" http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/.settings/org.eclipse.core.resources.prefs ---------------------------------------------------------------------- diff --git a/invertedindex/.settings/org.eclipse.core.resources.prefs b/invertedindex/.settings/org.eclipse.core.resources.prefs deleted file mode 100644 index 29abf99..0000000 --- a/invertedindex/.settings/org.eclipse.core.resources.prefs +++ /dev/null @@ -1,6 +0,0 @@ -eclipse.preferences.version=1 -encoding//src/main/java=UTF-8 -encoding//src/main/resources=UTF-8 -encoding//src/test/java=UTF-8 -encoding//src/test/resources=UTF-8 -encoding/<project>=UTF-8
