http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java deleted file mode 100644 index dfaf106..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodec.java +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.model; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.common.util.Array; -import org.apache.kylin.common.util.ByteArray; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.dict.DictionarySerializer; -import org.apache.kylin.invertedindex.index.ColumnValueContainer; -import org.apache.kylin.invertedindex.index.CompressedValueContainer; -import org.apache.kylin.invertedindex.index.Slice; -import org.apache.kylin.invertedindex.index.TableRecordInfoDigest; -import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec; -import org.apache.kylin.metadata.datatype.DataType; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -/** - * @author yangli9 - */ -public class IIKeyValueCodec implements KeyValueCodec { - - public static final int SHARD_LEN = 2; - public static final int TIMEPART_LEN = 8; - public static final int COLNO_LEN = 2; - protected final TableRecordInfoDigest digest; - - public IIKeyValueCodec(TableRecordInfoDigest digest) { - this.digest = digest; - } - - @Override - public Collection<IIRow> encodeKeyValue(Slice slice) { - ArrayList<IIRow> result = Lists.newArrayList(); - ColumnValueContainer[] containers = slice.getColumnValueContainers(); - for (int col = 0; col < containers.length; col++) { - if (containers[col] instanceof CompressedValueContainer) { - final IIRow row = collectKeyValues(slice, col, (CompressedValueContainer) containers[col]); - result.add(row); - } else { - throw new IllegalArgumentException("Unknown container class " + containers[col].getClass()); - } - } - return result; - } - - private IIRow collectKeyValues(Slice slice, int col, CompressedValueContainer container) { - ImmutableBytesWritable key = encodeKey(slice.getShard(), slice.getTimestamp(), col); - ImmutableBytesWritable value = container.toBytes(); - final Dictionary<?> dictionary = slice.getLocalDictionaries() != null ? slice.getLocalDictionaries()[col] : null; - if (dictionary == null) { - return new IIRow(key, value, new ImmutableBytesWritable(BytesUtil.EMPTY_BYTE_ARRAY)); - } else { - ByteArray bytes = DictionarySerializer.serialize(dictionary); - return new IIRow(key, value, new ImmutableBytesWritable(bytes.array(), bytes.offset(), bytes.length())); - } - } - - ImmutableBytesWritable encodeKey(short shard, long timestamp, int col) { - byte[] bytes = new byte[20]; - int len = encodeKey(shard, timestamp, col, bytes, 0); - return new ImmutableBytesWritable(bytes, 0, len); - } - - int encodeKey(short shard, long timestamp, int col, byte[] buf, int offset) { - int i = offset; - - BytesUtil.writeUnsigned(shard, buf, i, SHARD_LEN); - i += SHARD_LEN; - BytesUtil.writeLong(timestamp, buf, i, TIMEPART_LEN); - i += TIMEPART_LEN; - - BytesUtil.writeUnsigned(col, buf, i, COLNO_LEN); - i += COLNO_LEN; - - return i - offset; - } - - @Override - public Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs) { - return new IIRowDecoder(digest, kvs.iterator()); - //return new Decoder(kvs, incompleteDigest); - } - - private static TableRecordInfoDigest createDigest(int nColumns, boolean[] isMetric, String[] dataTypes, Dictionary<?>[] dictionaries) { - int[] dictMaxIds = new int[nColumns]; - int[] lengths = new int[nColumns]; - final boolean emptyDictionary = Array.isEmpty(dictionaries); - for (int i = 0; i < nColumns; ++i) { - if (isMetric[i]) { - final FixedLenMeasureCodec<?> fixedLenMeasureCodec = FixedLenMeasureCodec.get(DataType.getType(dataTypes[i])); - lengths[i] = fixedLenMeasureCodec.getLength(); - } else { - if (emptyDictionary) { - final DataType dataType = DataType.getType(dataTypes[i]); - if (dataType.isNumberFamily()) { - lengths[i] = 16; - } else if (dataType.isStringFamily()) { - lengths[i] = 256; - } else if (dataType.isDateTimeFamily()) { - lengths[i] = 19; - } else { - throw new RuntimeException("invalid data type:" + dataType); - } - dictMaxIds[i] = Integer.MAX_VALUE; - } else { - final Dictionary<?> dictionary = dictionaries[i]; - lengths[i] = dictionary.getSizeOfId(); - dictMaxIds[i] = dictionary.getMaxId(); - } - } - } - // offsets - int pos = 0; - int[] offsets = new int[nColumns]; - for (int i = 0; i < nColumns; i++) { - offsets[i] = pos; - pos += lengths[i]; - } - - int byteFormLen = pos; - - return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, dataTypes); - } - - protected static class IIRowDecoder implements Iterable<Slice> { - - protected final TableRecordInfoDigest incompleteDigest; - protected final Iterator<IIRow> iiRowIterator; - protected Iterator<IIRow> feedingIterator;//this is for extending - - protected IIRowDecoder(TableRecordInfoDigest digest, Iterator<IIRow> iiRowIterator) { - this.incompleteDigest = digest; - this.iiRowIterator = iiRowIterator; - this.feedingIterator = this.iiRowIterator; - } - - @Override - public Iterator<Slice> iterator() { - return new Iterator<Slice>() { - @Override - public boolean hasNext() { - return iiRowIterator.hasNext(); - } - - @Override - public Slice next() { - int columns = 0; - ColumnValueContainer[] valueContainers = new ColumnValueContainer[incompleteDigest.getColumnCount()]; - Dictionary<?>[] localDictionaries = new Dictionary<?>[incompleteDigest.getColumnCount()]; - boolean firstTime = true; - short curShard = 0; - long curTimestamp = 0; - short lastShard = 0; - long lastTimestamp = 0; - - while (feedingIterator.hasNext() && columns < incompleteDigest.getColumnCount()) { - final IIRow row = feedingIterator.next(); - final ImmutableBytesWritable key = row.getKey(); - int i = key.getOffset(); - curShard = (short) BytesUtil.readUnsigned(key.get(), i, SHARD_LEN); - i += SHARD_LEN; - curTimestamp = BytesUtil.readLong(key.get(), i, TIMEPART_LEN); - i += TIMEPART_LEN; - - if (!firstTime) { - Preconditions.checkArgument(curShard == lastShard, "shard should be equals in one slice, curShard is" + curShard + " lastShard is " + lastShard); - Preconditions.checkArgument(curTimestamp == lastTimestamp, "timestamp should be equals in one slice, curTimestamp is" + curTimestamp + " lastTimestamp is " + lastTimestamp); - } - - int curCol = BytesUtil.readUnsigned(key.get(), i, COLNO_LEN); - if (incompleteDigest.isMetrics(curCol)) { - CompressedValueContainer c = new CompressedValueContainer(incompleteDigest, curCol, 0); - c.fromBytes(row.getValue()); - valueContainers[curCol] = c; - } else { - final ImmutableBytesWritable dictBytes = row.getDictionary(); - if (dictBytes.getLength() != 0) { - final Dictionary<?> dictionary = DictionarySerializer.deserialize(new ByteArray(dictBytes.get(), dictBytes.getOffset(), dictBytes.getLength())); - CompressedValueContainer c = new CompressedValueContainer(dictionary.getSizeOfId(), dictionary.getMaxId() - dictionary.getMinId() + 1, 0); - c.fromBytes(row.getValue()); - valueContainers[curCol] = c; - localDictionaries[curCol] = dictionary; - } else { - CompressedValueContainer c = new CompressedValueContainer(incompleteDigest.length(curCol), incompleteDigest.getMaxID(curCol) - 0 + 1, 0); - c.fromBytes(row.getValue()); - valueContainers[curCol] = c; - } - } - columns++; - lastShard = curShard; - lastTimestamp = curTimestamp; - firstTime = false; - } - Preconditions.checkArgument(columns == incompleteDigest.getColumnCount(), "column count is " + columns + " should be equals to incompleteDigest.getColumnCount() " + incompleteDigest.getColumnCount()); - - TableRecordInfoDigest digest = createDigest(columns, incompleteDigest.getIsMetric(), incompleteDigest.getMetricDataTypes(), localDictionaries); - Slice slice = new Slice(digest, curShard, curTimestamp, valueContainers); - slice.setLocalDictionaries(localDictionaries); - return slice; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - - } - -}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java deleted file mode 100644 index 8747916..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIKeyValueCodecWithState.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.model; - -import java.util.Iterator; -import java.util.LinkedList; - -import org.apache.kylin.common.util.FIFOIterable; -import org.apache.kylin.common.util.FIFOIterator; -import org.apache.kylin.invertedindex.index.Slice; -import org.apache.kylin.invertedindex.index.TableRecordInfoDigest; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -/** - */ -public class IIKeyValueCodecWithState extends IIKeyValueCodec { - - public IIKeyValueCodecWithState(TableRecordInfoDigest digest) { - super(digest); - } - - /** - * - * @param kvs kvs must be a {@link org.apache.kylin.common.util.FIFOIterable } to avoid {@link java.util.ConcurrentModificationException}. - * @return - */ - @Override - public Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs) { - if (!(kvs instanceof FIFOIterable)) { - throw new IllegalArgumentException("kvs must be a {@link org.apache.kylin.common.util.FIFOIterable } to avoid {@link java.util.ConcurrentModificationException}."); - } - return new IIRowDecoderWithState(digest, kvs.iterator()); - } - - //TODO refactor this class, does not have to extend IIKeyValueCodec, composite might be enough, mhb - protected static class IIRowDecoderWithState extends IIRowDecoder { - - final LinkedList<IIRow> buffer = Lists.newLinkedList(); - private Iterator<Slice> superIterator = null; - - private IIRowDecoderWithState(TableRecordInfoDigest digest, Iterator<IIRow> iiRowIterator) { - super(digest, iiRowIterator); - this.feedingIterator = new FIFOIterator<>(buffer); - } - - private Iterator<Slice> getSuperIterator() { - if (superIterator == null) { - superIterator = super.iterator(); - } - return superIterator; - } - - @Override - public Iterator<Slice> iterator() { - return new Iterator<Slice>() { - @Override - public boolean hasNext() { - while (buffer.size() < incompleteDigest.getColumnCount() && iiRowIterator.hasNext()) { - buffer.add(iiRowIterator.next()); - } - return buffer.size() == incompleteDigest.getColumnCount(); - } - - @Override - public Slice next() { - while (buffer.size() < incompleteDigest.getColumnCount() && iiRowIterator.hasNext()) { - buffer.add(iiRowIterator.next()); - } - Preconditions.checkArgument(buffer.size() == incompleteDigest.getColumnCount(), "not enough IIRows!"); - Slice ret = IIRowDecoderWithState.this.getSuperIterator().next(); - buffer.clear(); - return ret; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java deleted file mode 100644 index 9d13487..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIRow.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * - * contributor license agreements. See the NOTICE file distributed with - * - * this work for additional information regarding copyright ownership. - * - * The ASF licenses this file to You under the Apache License, Version 2.0 - * - * (the "License"); you may not use this file except in compliance with - * - * the License. You may obtain a copy of the License at - * - * - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * - * - * Unless required by applicable law or agreed to in writing, software - * - * distributed under the License is distributed on an "AS IS" BASIS, - * - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and - * - * limitations under the License. - * - * / - */ - -package org.apache.kylin.invertedindex.model; - -import java.util.List; - -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.common.util.BytesUtil; - -import com.google.common.collect.Lists; - -/** - */ -public final class IIRow { - - private final ImmutableBytesWritable key; - private final ImmutableBytesWritable value; - private final ImmutableBytesWritable dictionary; - - public IIRow(ImmutableBytesWritable key, ImmutableBytesWritable value, ImmutableBytesWritable dictionary) { - this.key = key; - this.value = value; - this.dictionary = dictionary; - } - - public IIRow() { - this(new ImmutableBytesWritable(), new ImmutableBytesWritable(), new ImmutableBytesWritable()); - } - - public ImmutableBytesWritable getKey() { - return key; - } - - public ImmutableBytesWritable getValue() { - return value; - } - - public ImmutableBytesWritable getDictionary() { - return dictionary; - } - - public void updateWith(Cell c) { - if (BytesUtil.compareBytes(IIDesc.HBASE_QUALIFIER_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_QUALIFIER_BYTES.length) == 0) { - this.getKey().set(c.getRowArray(), c.getRowOffset(), c.getRowLength()); - this.getValue().set(c.getValueArray(), c.getValueOffset(), c.getValueLength()); - } else if (BytesUtil.compareBytes(IIDesc.HBASE_DICTIONARY_BYTES, 0, c.getQualifierArray(), c.getQualifierOffset(), IIDesc.HBASE_DICTIONARY_BYTES.length) == 0) { - this.getDictionary().set(c.getValueArray(), c.getValueOffset(), c.getValueLength()); - } - } - - public List<Cell> makeCells() { - Cell a = new KeyValue(this.getKey().copyBytes(), IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES, this.getValue().copyBytes()); - Cell b = new KeyValue(this.getKey().copyBytes(), IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES, this.getDictionary().copyBytes()); - return Lists.newArrayList(a, b); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java deleted file mode 100644 index 7843ee7..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/KeyValueCodec.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * - * contributor license agreements. See the NOTICE file distributed with - * - * this work for additional information regarding copyright ownership. - * - * The ASF licenses this file to You under the Apache License, Version 2.0 - * - * (the "License"); you may not use this file except in compliance with - * - * the License. You may obtain a copy of the License at - * - * - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * - * - * Unless required by applicable law or agreed to in writing, software - * - * distributed under the License is distributed on an "AS IS" BASIS, - * - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and - * - * limitations under the License. - * - * / - */ - -package org.apache.kylin.invertedindex.model; - -import java.util.Collection; - -import org.apache.kylin.invertedindex.index.Slice; - -/** - */ -public interface KeyValueCodec { - - Collection<IIRow> encodeKeyValue(Slice slice); - - Iterable<Slice> decodeKeyValue(Iterable<IIRow> kvs); -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java deleted file mode 100644 index 36a8781..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/util/IIDictionaryBuilder.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * - * contributor license agreements. See the NOTICE file distributed with - * - * this work for additional information regarding copyright ownership. - * - * The ASF licenses this file to You under the Apache License, Version 2.0 - * - * (the "License"); you may not use this file except in compliance with - * - * the License. You may obtain a copy of the License at - * - * - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * - * - * Unless required by applicable law or agreed to in writing, software - * - * distributed under the License is distributed on an "AS IS" BASIS, - * - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and - * - * limitations under the License. - * - * / - */ - -package org.apache.kylin.invertedindex.util; - -import java.io.IOException; -import java.util.Collection; -import java.util.List; - -import javax.annotation.Nullable; - -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.dict.DictionaryGenerator; -import org.apache.kylin.dict.IterableDictionaryValueEnumerator; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.metadata.model.TblColRef; - -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import com.google.common.collect.HashMultimap; - -/** - */ -public final class IIDictionaryBuilder { - - private IIDictionaryBuilder() { - } - - public static Dictionary<?>[] buildDictionary(List<List<String>> table, IIDesc desc) throws IOException { - HashMultimap<TblColRef, String> valueMap = HashMultimap.create(); - final List<TblColRef> allColumns = desc.listAllColumns(); - for (List<String> row : table) { - for (int i = 0; i < row.size(); i++) { - String cell = row.get(i); - if (!desc.isMetricsCol(i)) { - valueMap.put(allColumns.get(i), cell); - } - } - } - - Dictionary<?>[] result = new Dictionary<?>[allColumns.size()]; - for (TblColRef tblColRef : valueMap.keySet()) { - final Collection<byte[]> bytes = Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() { - @Nullable - @Override - public byte[] apply(String input) { - return input == null ? null : input.getBytes(); - } - }); - final Dictionary<?> dict = DictionaryGenerator.buildDictionary(tblColRef.getType(), new IterableDictionaryValueEnumerator(bytes)); - result[desc.findColumn(tblColRef)] = dict; - } - return result; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescManagerTest.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescManagerTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescManagerTest.java deleted file mode 100644 index 96dcbfc..0000000 --- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescManagerTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex; - -import java.io.IOException; -import java.util.UUID; - -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** - */ -public class IIDescManagerTest extends LocalFileMetadataTestCase { - - public static final String TEST_II_DESC_NAME = "test_kylin_ii_left_join_desc"; - - @Before - public void setup() { - createTestMetadata(); - } - - @After - public void clean() { - this.cleanupTestMetadata(); - } - - @Test - public void testCRUD() throws IOException { - IIDescManager mgr = IIDescManager.getInstance(getTestConfig()); - - String newDescName = "Copy of " + TEST_II_DESC_NAME; - - try { - IIDesc testRecord = mgr.getIIDesc(newDescName); - if (testRecord != null) - mgr.removeIIDesc(testRecord); - } catch (IOException e) { - // just ensure the old one is removed - } - - Assert.assertNull(mgr.getIIDesc(newDescName)); - IIDesc desc = mgr.getIIDesc(TEST_II_DESC_NAME); - - desc.setName(newDescName); - desc.setLastModified(0); - - mgr.createIIDesc(desc); - - desc = mgr.getIIDesc(newDescName); - - Assert.assertNotNull(desc); - - mgr.updateIIDesc(desc); // this will trigger cache wipe; please ignore the HTTP error in logs. - - mgr.removeIIDesc(desc); - - Assert.assertNull(mgr.getIIDesc(newDescName)); - - } - - @Test - public void testReload() throws IOException { - IIDescManager mgr = IIDescManager.getInstance(getTestConfig()); - - IIDesc desc = mgr.getIIDesc(TEST_II_DESC_NAME); - - // do some modification - desc.setUuid(UUID.randomUUID().toString()); - - IIDesc newDesc = mgr.getIIDesc(TEST_II_DESC_NAME); - - Assert.assertEquals(desc, newDesc); - - // reload the cache - mgr.reloadIIDescLocal(TEST_II_DESC_NAME); - - newDesc = mgr.getIIDesc(TEST_II_DESC_NAME); - - Assert.assertNotEquals(desc, newDesc); - - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescTest.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescTest.java deleted file mode 100644 index 4b5517f..0000000 --- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIDescTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex; - -import java.io.IOException; - -import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.metadata.model.DataModelDesc; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** - */ -public class IIDescTest extends LocalFileMetadataTestCase { - - @Before - public void setup() { - this.createTestMetadata(); - - } - - @After - public void clear() { - this.cleanupTestMetadata(); - } - - @Test - public void testGetIIDesc() { - - IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_left_join_desc"); - DataModelDesc model = iiDesc.getModel(); - Assert.assertNotNull(iiDesc); - Assert.assertNotNull(model); - - } - - @Test - public void testSerialization() throws IOException { - IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_left_join_desc"); - String str = JsonUtil.writeValueAsIndentString(iiDesc); - System.out.println(str); - IIDesc desc2 = JsonUtil.readValue(str, IIDesc.class); - - Assert.assertEquals(iiDesc, desc2); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java deleted file mode 100644 index 16bbdf0..0000000 --- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/IIInstanceTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex; - -import java.io.IOException; -import java.util.List; - -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -/** - */ -public class IIInstanceTest extends LocalFileMetadataTestCase { - @Before - public void setup() { - createTestMetadata(); - } - - @After - public void clean() { - cleanupTestMetadata(); - } - - @Test - public void testGetIIsByDesc() throws IOException { - IIManager mgr = IIManager.getInstance(getTestConfig()); - - List<IIInstance> iiInstances = mgr.getIIsByDesc("test_kylin_ii_left_join_desc"); - - Assert.assertTrue(iiInstances.size() > 0); - - } - - @Test - public void testCreateIIInstance() throws IOException { - - IIDesc iiDesc = IIDescManager.getInstance(getTestConfig()).getIIDesc("test_kylin_ii_left_join_desc"); - - IIInstance ii = IIInstance.create("new ii", "default", iiDesc); - - IIManager iiMgr = IIManager.getInstance(getTestConfig()); - - List<IIInstance> allIIList = iiMgr.listAllIIs(); - - iiMgr.createII(ii); - - Assert.assertNotNull(iiMgr.getII("new ii")); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java deleted file mode 100644 index 7b6a688..0000000 --- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/InvertedIndexLocalTest.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex; - -import static org.junit.Assert.assertEquals; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import javax.annotation.Nullable; - -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.common.util.LocalFileMetadataTestCase; -import org.apache.kylin.dict.DictionaryGenerator; -import org.apache.kylin.dict.IterableDictionaryValueEnumerator; -import org.apache.kylin.invertedindex.index.CompressedValueContainer; -import org.apache.kylin.invertedindex.index.RawTableRecord; -import org.apache.kylin.invertedindex.index.ShardingSliceBuilder; -import org.apache.kylin.invertedindex.index.Slice; -import org.apache.kylin.invertedindex.index.TableRecord; -import org.apache.kylin.invertedindex.index.TableRecordInfo; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.invertedindex.model.IIKeyValueCodec; -import org.apache.kylin.invertedindex.model.IIRow; -import org.apache.kylin.metadata.model.TblColRef; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.google.common.base.Function; -import com.google.common.collect.Collections2; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.SetMultimap; -import com.google.common.collect.Sets; - -public class InvertedIndexLocalTest extends LocalFileMetadataTestCase { - - IIInstance ii; - TableRecordInfo info; - List<String> lines; - private Dictionary<?>[] dictionaryMap; - - @Before - public void setUp() throws Exception { - this.createTestMetadata(); - this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join"); - - File file = new File(LOCALMETA_TEST_DATA, "data/flatten_data_for_ii.csv"); - FileInputStream in = new FileInputStream(file); - this.lines = IOUtils.readLines(in, "UTF-8"); - in.close(); - - dictionaryMap = buildDictionary(Lists.transform(lines, new Function<String, List<String>>() { - @Nullable - @Override - public List<String> apply(@Nullable String input) { - return Lists.newArrayList(input.split(",")); - } - }), ii.getDescriptor()); - this.info = new TableRecordInfo(ii.getDescriptor(), dictionaryMap); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void testCompressedValueContainer() { - // create container - CompressedValueContainer container = new CompressedValueContainer(info.getDigest(), 0, 500); - Dictionary<String> dict = info.dict(0); - - byte[] buf = new byte[dict.getSizeOfId()]; - ImmutableBytesWritable bytes = new ImmutableBytesWritable(buf); - - for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) { - BytesUtil.writeUnsigned(v, buf, 0, dict.getSizeOfId()); - container.append(bytes); - } - BytesUtil.writeUnsigned(Dictionary.NULL_ID[dict.getSizeOfId()], buf, 0, dict.getSizeOfId()); - container.append(bytes); - container.closeForChange(); - - // copy by serialization - ImmutableBytesWritable copy = container.toBytes(); - CompressedValueContainer container2 = new CompressedValueContainer(info.getDigest(), 0, 500); - container2.fromBytes(copy); - - // check the copy - int i = 0; - for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) { - container2.getValueAt(i++, bytes); - int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength()); - assertEquals(v, value); - } - container2.getValueAt(i++, bytes); - int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(), bytes.getLength()); - assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()], value); - assertEquals(container, container2); - } - - @Test - public void testCodec() throws IOException { - List<TableRecord> records = loadRecordsSorted(); - System.out.println(records.size() + " records"); - List<Slice> slices = buildTimeSlices(records); - System.out.println(slices.size() + " slices"); - - IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest()); - List<IIRow> kvs = encodeKVs(codec, slices); - System.out.println(kvs.size() + " KV pairs"); - - List<Slice> slicesCopy = decodeKVs(codec, kvs); - assertEquals(slices.size(), slicesCopy.size()); - for (int i = 0; i < slices.size(); i++) { - assertEquals(slices.get(i), slicesCopy.get(i)); - } - - List<TableRecord> recordsCopy = iterateRecords(slicesCopy); - assertEquals(new HashSet<TableRecord>(records), new HashSet<TableRecord>(recordsCopy)); - dump(recordsCopy); - } - - private Dictionary<?>[] buildDictionary(List<List<String>> table, IIDesc desc) throws IOException { - SetMultimap<TblColRef, String> valueMap = HashMultimap.create(); - Set<TblColRef> dimensionColumns = Sets.newHashSet(); - for (int i = 0; i < desc.listAllColumns().size(); i++) { - if (!desc.isMetricsCol(i)) { - dimensionColumns.add(desc.listAllColumns().get(i)); - } - } - for (List<String> row : table) { - for (int i = 0; i < row.size(); i++) { - String cell = row.get(i); - valueMap.put(desc.listAllColumns().get(i), cell); - } - } - Dictionary<?>[] result = new Dictionary<?>[desc.listAllColumns().size()]; - for (TblColRef tblColRef : valueMap.keys()) { - result[desc.findColumn(tblColRef)] = DictionaryGenerator.buildDictionary(tblColRef.getType(), new IterableDictionaryValueEnumerator(Collections2.transform(valueMap.get(tblColRef), new Function<String, byte[]>() { - @Nullable - @Override - public byte[] apply(String input) { - try { - return input.getBytes("UTF-8"); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } - } - }))); - } - return result; - } - - private List<TableRecord> loadRecordsSorted() throws IOException { - List<TableRecord> records = Lists.newArrayList(); - for (String line : lines) { - String[] fields = line.split(","); - TableRecord rec = info.createTableRecord(); - for (int col = 0; col < fields.length; col++) { - rec.setValueString(col, fields[col]); - } - records.add(rec); - } - - Collections.sort(records, new Comparator<TableRecord>() { - @Override - public int compare(TableRecord a, TableRecord b) { - long x = a.getTimestamp() - b.getTimestamp(); - if (x > 0) - return 1; - else if (x == 0) - return 0; - else - return -1; - } - }); - - return records; - } - - private List<Slice> buildTimeSlices(List<TableRecord> records) throws IOException { - ShardingSliceBuilder builder = new ShardingSliceBuilder(info); - List<Slice> slices = Lists.newArrayList(); - for (TableRecord rec : records) { - //here assume there less records than slice size for each shard - Slice slice = builder.append(rec); - if (slice != null) { - slice.setLocalDictionaries(dictionaryMap); - slices.add(slice); - } - } - List<Slice> finals = builder.close(); - for (Slice slice : finals) { - slice.setLocalDictionaries(dictionaryMap); - } - slices.addAll(finals); - - Collections.sort(slices); - return slices; - } - - private List<IIRow> encodeKVs(IIKeyValueCodec codec, List<Slice> slices) { - - List<IIRow> kvs = Lists.newArrayList(); - for (Slice slice : slices) { - kvs.addAll(codec.encodeKeyValue(slice)); - } - return kvs; - } - - private List<Slice> decodeKVs(IIKeyValueCodec codec, List<IIRow> kvs) { - List<Slice> slices = Lists.newArrayList(); - for (Slice slice : codec.decodeKeyValue(kvs)) { - slices.add(slice); - } - return slices; - } - - private List<TableRecord> iterateRecords(List<Slice> slices) { - List<TableRecord> records = Lists.newArrayList(); - for (Slice slice : slices) { - for (RawTableRecord rec : slice) { - records.add(new TableRecord((RawTableRecord) rec.clone(), info)); - } - } - return records; - } - - private void dump(Iterable<TableRecord> records) { - for (TableRecord rec : records) { - System.out.println(rec.toString()); - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/test/java/org/apache/kylin/invertedindex/LZFTest.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/LZFTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/LZFTest.java deleted file mode 100644 index 943e76c..0000000 --- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/LZFTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex; - -import java.io.IOException; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.junit.Test; - -import com.ning.compress.lzf.LZFDecoder; -import com.ning.compress.lzf.LZFEncoder; - -/** - */ -public class LZFTest { - @Test - public void test() throws IOException { - - byte[] raw = new byte[] { 1, 2, 3, 3, 2, 23 }; - byte[] data = LZFEncoder.encode(raw); - - byte[] data2 = new byte[data.length * 2]; - java.lang.System.arraycopy(data, 0, data2, 0, data.length); - ImmutableBytesWritable bytes = new ImmutableBytesWritable(); - bytes.set(data2, 0, data.length); - - try { - byte[] uncompressed = LZFDecoder.decode(bytes.get(), bytes.getOffset(), bytes.getLength()); - } catch (IOException e) { - throw new RuntimeException("LZF decode failure", e); - } - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java b/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java deleted file mode 100644 index 6b21282..0000000 --- a/invertedindex/src/test/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodecTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.measure; - -import org.apache.kylin.metadata.datatype.DataType; -import org.junit.Test; - -/** - */ -public class FixedPointLongCodecTest { - - @Test - public void testEncode1() { - FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)")); - long x = codec.getValueIgnoringDecimalPoint("12.12345"); - org.junit.Assert.assertEquals(1212345, x); - } - - @Test - public void testEncode2() { - FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)")); - long x = codec.getValueIgnoringDecimalPoint("12.1234"); - org.junit.Assert.assertEquals(1212340, x); - } - - @Test - public void testEncode3() { - FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)")); - long x = codec.getValueIgnoringDecimalPoint("12.123456"); - org.junit.Assert.assertEquals(1212345, x); - } - - @Test - public void testEncode4() { - FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)")); - long x = codec.getValueIgnoringDecimalPoint("12"); - org.junit.Assert.assertEquals(1200000, x); - } - - @Test - public void testDecode1() { - FixedPointLongCodec codec = new FixedPointLongCodec(DataType.getType("decimal(18,5)")); - String x = codec.restoreDecimalPoint(1212345); - org.junit.Assert.assertEquals("12.12345", x); - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/kylin-it/pom.xml ---------------------------------------------------------------------- diff --git a/kylin-it/pom.xml b/kylin-it/pom.xml index 271bae8..a96dcd9 100644 --- a/kylin-it/pom.xml +++ b/kylin-it/pom.xml @@ -74,12 +74,6 @@ <version>${project.parent.version}</version> </dependency> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-invertedindex</artifactId> - <version>${project.parent.version}</version> - </dependency> - - <dependency> <groupId>org.apache.calcite</groupId> <artifactId>calcite-linq4j</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java index f0a3178..cf18b20 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITCombinationTest.java @@ -42,7 +42,6 @@ public class ITCombinationTest extends ITKylinQueryTest { @BeforeClass public static void setUp() throws SQLException { Map<RealizationType, Integer> priorities = Maps.newHashMap(); - priorities.put(RealizationType.INVERTED_INDEX, 2); priorities.put(RealizationType.HYBRID, 0); priorities.put(RealizationType.CUBE, 0); Candidate.setPriorities(priorities); http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java deleted file mode 100644 index fd74dc1..0000000 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITIIQueryTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.query; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Map; - -import org.apache.kylin.metadata.realization.RealizationType; -import org.apache.kylin.query.routing.Candidate; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runners.Parameterized; - -import com.google.common.collect.Maps; - -/** - */ -@Ignore("II query is no longer maintained") -//@RunWith(Parameterized.class) -public class ITIIQueryTest extends ITKylinQueryTest { - @BeforeClass - public static void setUp() throws Exception { - - // give II higher priority than other realizations - Map<RealizationType, Integer> priorities = Maps.newHashMap(); - priorities.put(RealizationType.INVERTED_INDEX, 0); - priorities.put(RealizationType.HYBRID, 1); - priorities.put(RealizationType.CUBE, 2); - Candidate.setPriorities(priorities); - } - - @AfterClass - public static void tearDown() throws Exception { - ITKylinQueryTest.tearDown();//invoke super class - Candidate.restorePriorities(); - } - - @Parameterized.Parameters - public static Collection<Object[]> configs() { - return Arrays.asList(new Object[][] { { "inner" }, { "left" } }); - } - - public ITIIQueryTest(String joinType) throws Exception { - - ITKylinQueryTest.clean(); - - ITKylinQueryTest.joinType = joinType; - ITKylinQueryTest.setupAll(); - - } - - @Test - public void testSingleRunQuery() throws Exception { - super.testSingleRunQuery(); - } - - @Test - public void testDetailedQuery() throws Exception { - execAndCompQuery("src/test/resources/query/sql_ii", null, true); - } - - @Override - @Test - @Ignore("Skip Precisely Distinct Count Queries for II") - public void testPreciselyDistinctCountQuery() { - } - - @Override - @Test - @Ignore("Skip Dimension Distinct Count Queries for II") - public void testDimDistinctCountQuery() { - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java index d909a99..a53c624 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/ITKylinQueryTest.java @@ -51,7 +51,6 @@ public class ITKylinQueryTest extends KylinTestBase { @BeforeClass public static void setUp() throws Exception { Map<RealizationType, Integer> priorities = Maps.newHashMap(); - priorities.put(RealizationType.INVERTED_INDEX, 2); priorities.put(RealizationType.HYBRID, 0); priorities.put(RealizationType.CUBE, 0); Candidate.setPriorities(priorities); http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java b/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java deleted file mode 100644 index a04c5ae..0000000 --- a/kylin-it/src/test/java/org/apache/kylin/storage/hbase/ii/ITInvertedIndexHBaseTest.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.hbase.ii; - -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HConnectionManager; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.HBaseMetadataTestCase; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.invertedindex.index.RawTableRecord; -import org.apache.kylin.invertedindex.index.Slice; -import org.apache.kylin.invertedindex.index.TableRecord; -import org.apache.kylin.invertedindex.index.TableRecordInfo; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.invertedindex.model.IIKeyValueCodec; -import org.apache.kylin.storage.hbase.HBaseConnection; -import org.apache.kylin.storage.hbase.cube.v1.HBaseClientKVIterator; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.google.common.collect.Lists; - -/** - * @author yangli9 - */ -public class ITInvertedIndexHBaseTest extends HBaseMetadataTestCase { - - IIInstance ii; - IISegment seg; - HConnection hconn; - - TableRecordInfo info; - - @Before - public void setup() throws Exception { - this.createTestMetadata(); - - this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii_left_join"); - this.seg = ii.getFirstSegment(); - - Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration(); - hconn = HConnectionManager.createConnection(hconf); - - this.info = new TableRecordInfo(seg); - } - - @After - public void after() throws Exception { - this.cleanupTestMetadata(); - } - - @Test - public void testLoad() throws Exception { - - String tableName = seg.getStorageLocationIdentifier(); - IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest()); - - List<Slice> slices = Lists.newArrayList(); - HBaseClientKVIterator kvIterator = new HBaseClientKVIterator(hconn, tableName, IIDesc.HBASE_FAMILY_BYTES); - try { - for (Slice slice : codec.decodeKeyValue(kvIterator)) { - slices.add(slice); - } - } finally { - kvIterator.close(); - } - - List<TableRecord> records = iterateRecords(slices); - //dump(records); - System.out.println("table name:" + tableName + " has " + records.size() + " records"); - } - - private List<TableRecord> iterateRecords(List<Slice> slices) { - List<TableRecord> records = Lists.newArrayList(); - for (Slice slice : slices) { - for (RawTableRecord rec : slice) { - records.add(new TableRecord((RawTableRecord) rec.clone(), info)); - } - } - return records; - } - - @SuppressWarnings("unused") - private void dump(Iterable<TableRecord> records) { - for (TableRecord rec : records) { - byte[] x = rec.getBytes(); - String y = BytesUtil.toReadableText(x); - System.out.println(y); - System.out.println(); - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 2def6d2..7a211fc 100644 --- a/pom.xml +++ b/pom.xml @@ -677,7 +677,6 @@ <module>server-base</module> <module>server</module> <module>jdbc</module> - <module>invertedindex</module> <module>assembly</module> <module>tool</module> <module>kylin-it</module> http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/query/pom.xml ---------------------------------------------------------------------- diff --git a/query/pom.xml b/query/pom.xml index 87a4eba..a0b8d94 100644 --- a/query/pom.xml +++ b/query/pom.xml @@ -48,12 +48,6 @@ <version>${project.parent.version}</version> </dependency> <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-invertedindex</artifactId> - <version>${project.parent.version}</version> - </dependency> - - <dependency> <groupId>org.apache.calcite</groupId> <artifactId>calcite-linq4j</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/query/src/main/java/org/apache/kylin/query/routing/Candidate.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java index 28c60a8..c7c99ab 100644 --- a/query/src/main/java/org/apache/kylin/query/routing/Candidate.java +++ b/query/src/main/java/org/apache/kylin/query/routing/Candidate.java @@ -35,7 +35,6 @@ public class Candidate implements Comparable<Candidate> { static { DEFAULT_PRIORITIES.put(RealizationType.HYBRID, 0); DEFAULT_PRIORITIES.put(RealizationType.CUBE, 1); - DEFAULT_PRIORITIES.put(RealizationType.INVERTED_INDEX, 2); } /** for test only */ http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java index 7197f03..5f07adf 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java @@ -30,8 +30,6 @@ import org.apache.kylin.cube.CubeManager; import org.apache.kylin.engine.mr.CubingJob; import org.apache.kylin.engine.mr.steps.CubingExecutableUtil; import org.apache.kylin.engine.streaming.StreamingManager; -import org.apache.kylin.invertedindex.IIDescManager; -import org.apache.kylin.invertedindex.IIManager; import org.apache.kylin.job.execution.AbstractExecutable; import org.apache.kylin.job.execution.ExecutableState; import org.apache.kylin.job.execution.Output; @@ -94,14 +92,6 @@ public abstract class BasicService { return ExecutableManager.getInstance(getConfig()); } - public IIDescManager getIIDescManager() { - return IIDescManager.getInstance(getConfig()); - } - - public IIManager getIIManager() { - return IIManager.getInstance(getConfig()); - } - public BadQueryHistoryManager getBadQueryHistoryManager() { return BadQueryHistoryManager.getInstance(getConfig()); } http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java index 9185544..02efe70 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -38,8 +38,6 @@ import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.cube.CubeManager; import org.apache.kylin.dict.DictionaryManager; import org.apache.kylin.engine.streaming.StreamingManager; -import org.apache.kylin.invertedindex.IIDescManager; -import org.apache.kylin.invertedindex.IIManager; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; @@ -186,28 +184,16 @@ public class CacheService extends BasicService { case PROJECT: reloadProjectCache(cacheKey); break; - case INVERTED_INDEX: - //II update does not need to update storage cache because it is dynamic already - getIIManager().reloadIILocal(cacheKey); - getHybridManager().reloadHybridInstanceByChild(RealizationType.INVERTED_INDEX, cacheKey); - getProjectManager().clearL2Cache(); - break; - case INVERTED_INDEX_DESC: - getIIDescManager().reloadIIDescLocal(cacheKey); - break; case TABLE: getMetadataManager().reloadTableCache(cacheKey); - IIDescManager.clearCache(); CubeDescManager.clearCache(); break; case EXTERNAL_FILTER: getMetadataManager().reloadExtFilter(cacheKey); - IIDescManager.clearCache(); CubeDescManager.clearCache(); break; case DATA_MODEL: getMetadataManager().reloadDataModelDesc(cacheKey); - IIDescManager.clearCache(); CubeDescManager.clearCache(); break; case ALL: @@ -215,8 +201,6 @@ public class CacheService extends BasicService { MetadataManager.clearCache(); CubeDescManager.clearCache(); CubeManager.clearCache(); - IIDescManager.clearCache(); - IIManager.clearCache(); HybridManager.clearCache(); RealizationRegistry.clearCache(); ProjectManager.clearCache(); @@ -259,12 +243,6 @@ public class CacheService extends BasicService { case PROJECT: ProjectManager.clearCache(); break; - case INVERTED_INDEX: - getIIManager().removeIILocal(cacheKey); - break; - case INVERTED_INDEX_DESC: - getIIDescManager().removeIIDescLocal(cacheKey); - break; case TABLE: throw new UnsupportedOperationException(log); case EXTERNAL_FILTER: http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java index 9d8ccfb..4cfa209 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/ModelService.java @@ -24,7 +24,6 @@ import java.util.List; import org.apache.kylin.cube.model.CubeDesc; import org.apache.kylin.engine.mr.HadoopUtil; -import org.apache.kylin.invertedindex.model.IIDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.rest.constant.Constant; @@ -117,14 +116,6 @@ public class ModelService extends BasicService { } } - //check II desc exist - List<IIDesc> iiDescs = getIIDescManager().listAllDesc(); - for (IIDesc iidesc : iiDescs) { - if (iidesc.getModelName().equals(desc.getName())) { - throw new InternalErrorException("Model referenced by IIDesc."); - } - } - getMetadataManager().dropModel(desc); accessService.clean(desc, true); http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java ---------------------------------------------------------------------- diff --git a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java index f8dc945..ae4c089 100644 --- a/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java +++ b/server/src/test/java/org/apache/kylin/rest/service/ServiceTestBase.java @@ -21,8 +21,6 @@ package org.apache.kylin.rest.service; import org.apache.kylin.common.util.LocalFileMetadataTestCase; import org.apache.kylin.cube.CubeDescManager; import org.apache.kylin.cube.CubeManager; -import org.apache.kylin.invertedindex.IIDescManager; -import org.apache.kylin.invertedindex.IIManager; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.metadata.realization.RealizationRegistry; @@ -65,8 +63,6 @@ public class ServiceTestBase extends LocalFileMetadataTestCase { MetadataManager.clearCache(); CubeDescManager.clearCache(); CubeManager.clearCache(); - IIDescManager.clearCache(); - IIManager.clearCache(); RealizationRegistry.clearCache(); ProjectManager.clearCache(); CacheService.removeAllOLAPDataSources(); http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/pom.xml ---------------------------------------------------------------------- diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml index 943b2d4..0cb42db 100644 --- a/storage-hbase/pom.xml +++ b/storage-hbase/pom.xml @@ -46,11 +46,6 @@ <artifactId>kylin-engine-streaming</artifactId> <version>${project.parent.version}</version> </dependency> - <dependency> - <groupId>org.apache.kylin</groupId> - <artifactId>kylin-invertedindex</artifactId> - <version>${project.parent.version}</version> - </dependency> <!-- Env & Test --> <dependency> @@ -135,7 +130,6 @@ <include>org.apache.kylin:kylin-core-metadata</include> <include>org.apache.kylin:kylin-core-dictionary</include> <include>org.apache.kylin:kylin-core-cube</include> - <include>org.apache.kylin:kylin-invertedindex</include> <include>com.ning:compress-lzf</include> <include>org.roaringbitmap:RoaringBitmap</include> <!-- below for inverted index only --> http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java index 20bc229..e7a322c 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStorage.java @@ -18,12 +18,12 @@ package org.apache.kylin.storage.hbase; +import com.google.common.base.Preconditions; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.debug.BackdoorToggles; import org.apache.kylin.cube.CubeInstance; import org.apache.kylin.engine.mr.IMROutput; import org.apache.kylin.engine.mr.IMROutput2; -import org.apache.kylin.invertedindex.IIInstance; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.IStorageAware; @@ -36,8 +36,6 @@ import org.apache.kylin.storage.IStorageQuery; import org.apache.kylin.storage.hbase.steps.HBaseMROutput; import org.apache.kylin.storage.hbase.steps.HBaseMROutput2Transition; -import com.google.common.base.Preconditions; - @SuppressWarnings("unused") //used by reflection public class HBaseStorage implements IStorage { @@ -46,21 +44,10 @@ public class HBaseStorage implements IStorage { public final static String v1CubeStorageQuery = "org.apache.kylin.storage.hbase.cube.v1.CubeStorageQuery"; public static String overwriteStorageQuery = null;//for test case - private final static String defaultIIStorageQuery = "org.apache.kylin.storage.hbase.ii.InvertedIndexStorageQuery"; - @Override public IStorageQuery createQuery(IRealization realization) { - if (realization.getType() == RealizationType.INVERTED_INDEX) { - IStorageQuery ret; - try { - ret = (IStorageQuery) Class.forName(defaultIIStorageQuery).getConstructor(IIInstance.class).newInstance((IIInstance) realization); - } catch (Exception e) { - throw new RuntimeException("Failed to initialize storage query for " + defaultIIStorageQuery, e); - } - return ret; - - } else if (realization.getType() == RealizationType.CUBE) { + if (realization.getType() == RealizationType.CUBE) { CubeInstance cubeInstance = (CubeInstance) realization; String cubeStorageQuery; http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorProjector.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorProjector.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorProjector.java index e142536..65c5f92 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorProjector.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorProjector.java @@ -29,7 +29,6 @@ import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.kv.RowKeyEncoder; -import org.apache.kylin.invertedindex.index.TableRecordInfo; import org.apache.kylin.metadata.model.TblColRef; /** @@ -55,20 +54,7 @@ public class CoprocessorProjector { byte[] mask = rowKeyMaskEncoder.encode(new byte[cuboid.getColumns().size()][]); return new CoprocessorProjector(mask, dimensionColumns.size() != 0); } - - public static CoprocessorProjector makeForEndpoint(final TableRecordInfo tableInfo, final Collection<TblColRef> groupby) { - byte[] mask = new byte[tableInfo.getDigest().getByteFormLen()]; - int maskIdx = 0; - for (int i = 0; i < tableInfo.getDigest().getColumnCount(); ++i) { - TblColRef tblColRef = tableInfo.getColumns().get(i); - int length = tableInfo.getDigest().length(i); - byte bits = groupby.contains(tblColRef) ? (byte) 0xff : 0x00; - for (int j = 0; j < length; ++j) { - mask[maskIdx++] = bits; - } - } - return new CoprocessorProjector(mask, groupby.size() != 0); - } + public static byte[] serialize(CoprocessorProjector o) { ByteBuffer buf = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE); http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java index a6e3073..d47bf61 100644 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java +++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/common/coprocessor/CoprocessorRowType.java @@ -27,7 +27,6 @@ import org.apache.kylin.common.util.BytesUtil; import org.apache.kylin.cube.CubeSegment; import org.apache.kylin.cube.cuboid.Cuboid; import org.apache.kylin.cube.kv.RowKeyColumnIO; -import org.apache.kylin.invertedindex.index.TableRecordInfo; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; @@ -39,17 +38,6 @@ import com.google.common.collect.Maps; */ public class CoprocessorRowType { - //for endpoint - public static CoprocessorRowType fromTableRecordInfo(TableRecordInfo tableRecordInfo, List<TblColRef> cols) { - - int[] colSizes = new int[cols.size()]; - for (int i = 0; i < cols.size(); i++) { - colSizes[i] = tableRecordInfo.getDigest().length(i); - } - - //TODO:check0 - return new CoprocessorRowType(cols.toArray(new TblColRef[cols.size()]), colSizes, 0); - } //for observer public static CoprocessorRowType fromCuboid(CubeSegment seg, Cuboid cuboid) { http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java deleted file mode 100644 index 8aace22..0000000 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/HBaseClientKVIterator.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.hbase.cube.v1; - -import java.io.Closeable; -import java.io.IOException; -import java.util.Iterator; - -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HTableInterface; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.invertedindex.model.IIRow; - -/** - * @author yangli9 - * - */ -public class HBaseClientKVIterator implements Iterable<IIRow>, Closeable { - - byte[] family; - - HTableInterface table; - ResultScanner scanner; - Iterator<Result> iterator; - - public HBaseClientKVIterator(HConnection hconn, String tableName, byte[] family) throws IOException { - this.family = family; - - this.table = hconn.getTable(tableName); - this.scanner = table.getScanner(family); - this.iterator = scanner.iterator(); - } - - @Override - public void close() { - IOUtils.closeQuietly(scanner); - IOUtils.closeQuietly(table); - } - - @Override - public Iterator<IIRow> iterator() { - return new MyIterator(); - } - - private class MyIterator implements Iterator<IIRow> { - - ImmutableBytesWritable key = new ImmutableBytesWritable(); - ImmutableBytesWritable value = new ImmutableBytesWritable(); - ImmutableBytesWritable dict = new ImmutableBytesWritable(); - IIRow pair = new IIRow(key, value, dict); - - @Override - public boolean hasNext() { - return iterator.hasNext(); - } - - @Override - public IIRow next() { - Result r = iterator.next(); - Cell c = r.getColumnLatestCell(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_QUALIFIER_BYTES); - key.set(c.getRowArray(), c.getRowOffset(), c.getRowLength()); - value.set(c.getValueArray(), c.getValueOffset(), c.getValueLength()); - c = r.getColumnLatestCell(IIDesc.HBASE_FAMILY_BYTES, IIDesc.HBASE_DICTIONARY_BYTES); - dict.set(c.getValueArray(), c.getValueOffset(), c.getValueLength()); - return pair; - } - - public void remove() { - throw new UnsupportedOperationException(); - } - - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IIBulkLoadJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IIBulkLoadJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IIBulkLoadJob.java deleted file mode 100644 index 22c8ec6..0000000 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IIBulkLoadJob.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.hbase.ii; - -import java.io.IOException; - -import org.apache.commons.cli.Options; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FsShell; -import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles; -import org.apache.hadoop.util.ToolRunner; -import org.apache.kylin.engine.mr.common.AbstractHadoopJob; -import org.apache.kylin.storage.hbase.HBaseConnection; - -/** - */ -public class IIBulkLoadJob extends AbstractHadoopJob { - - @Override - public int run(String[] args) throws Exception { - Options options = new Options(); - - try { - options.addOption(OPTION_INPUT_PATH); - options.addOption(OPTION_HTABLE_NAME); - options.addOption(OPTION_II_NAME); - parseOptions(options, args); - - String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase(); - String input = getOptionValue(OPTION_INPUT_PATH); - - Configuration conf = HBaseConnection.getCurrentHBaseConfiguration(); - FsShell shell = new FsShell(conf); - try { - shell.run(new String[] { "-chmod", "-R", "777", input }); - } catch (Exception e) { - logger.error("Couldn't change the file permissions ", e); - throw new IOException(e); - } - - return ToolRunner.run(new LoadIncrementalHFiles(conf), new String[] { input, tableName }); - - } catch (Exception e) { - printUsage(options); - throw e; - } - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileJob.java ---------------------------------------------------------------------- diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileJob.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileJob.java deleted file mode 100644 index 30dca8e..0000000 --- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/IICreateHFileJob.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.storage.hbase.ii; - -import org.apache.commons.cli.Options; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.engine.mr.common.AbstractHadoopJob; -import org.apache.kylin.invertedindex.IIInstance; -import org.apache.kylin.invertedindex.IIManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author yangli9 - * - */ -public class IICreateHFileJob extends AbstractHadoopJob { - - protected static final Logger logger = LoggerFactory.getLogger(IICreateHFileJob.class); - - public int run(String[] args) throws Exception { - Options options = new Options(); - - try { - options.addOption(OPTION_JOB_NAME); - options.addOption(OPTION_II_NAME); - options.addOption(OPTION_INPUT_PATH); - options.addOption(OPTION_OUTPUT_PATH); - options.addOption(OPTION_HTABLE_NAME); - parseOptions(options, args); - - Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH)); - - job = Job.getInstance(getConf(), getOptionValue(OPTION_JOB_NAME)); - - String iiName = getOptionValue(OPTION_II_NAME); - IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); - IIInstance ii = mgr.getII(iiName); - - setJobClasspath(job, ii.getConfig()); - - addInputDirs(getOptionValue(OPTION_INPUT_PATH), job); - FileOutputFormat.setOutputPath(job, output); - - job.setInputFormatClass(SequenceFileInputFormat.class); - job.setMapperClass(IICreateHFileMapper.class); - job.setMapOutputKeyClass(ImmutableBytesWritable.class); - job.setMapOutputValueClass(KeyValue.class); - - String tableName = getOptionValue(OPTION_HTABLE_NAME); - HTable htable = new HTable(HBaseConfiguration.create(getConf()), tableName); - HFileOutputFormat.configureIncrementalLoad(job, htable); - - this.deletePath(job.getConfiguration(), output); - - return waitForCompletion(job); - } catch (Exception e) { - printUsage(options); - throw e; - } - } - -}