http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java deleted file mode 100644 index 0ee16b8..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/BitMapContainer.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.index; - -//import it.uniroma3.mat.extendedset.intset.ConciseSet; -// -//import java.nio.ByteBuffer; -//import java.nio.IntBuffer; -//import java.util.ArrayList; -//import java.util.Arrays; -//import java.util.List; -// -//import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -// -//import org.apache.kylin.common.util.BytesUtil; -//import org.apache.kylin.dict.Dictionary; - -/** - * @author yangli9 - */ -//public class BitMapContainer implements ColumnValueContainer { -// -// int valueLen; -// int nValues; -// int size; -// ConciseSet[] sets; -// boolean closedForChange; -// -// transient byte[] temp; -// -// public BitMapContainer(TableRecordInfoDigest digest, int col) { -// this.valueLen = digest.length(col); -// this.size = 0; -// this.nValues = digest.getMaxID(col) + 1; -// this.sets = null; -// this.closedForChange = false; -// -// this.temp = new byte[valueLen]; -// } -// -// @Override -// public void append(ImmutableBytesWritable valueBytes) { -// int value = BytesUtil.readUnsigned(valueBytes.get(), valueBytes.getOffset(), valueLen); -// append(value); -// } -// -// public void append(int value) { -// checkUpdateMode(); -// if (value == Dictionary.NULL_ID[valueLen]) { -// value = nValues; // set[nValues] holds NULL -// } -// sets[value].add(size); -// size++; -// } -// -// @Override -// public void getValueAt(int i, ImmutableBytesWritable valueBytes) { -// int value = getValueIntAt(i); -// BytesUtil.writeUnsigned(value, temp, 0, valueLen); -// valueBytes.set(temp, 0, valueLen); -// } -// -// @Override -// public ConciseSet getBitMap(Integer startId, Integer endId) { -// if (startId == null && endId == null) { -// return sets[this.nValues]; -// } -// -// int start = 0; -// int end = this.nValues - 1; -// if (startId != null) { -// start = startId; -// } -// if (endId != null) { -// end = endId; -// } -// -// ConciseSet ret = new ConciseSet(); -// for (int i = start; i <= end; ++i) { -// ConciseSet temp = getBitMap(i); -// ret.addAll(temp); -// } -// return ret; -// } -// -// private ConciseSet getBitMap(int valueId) { -// if (valueId >= 0 && valueId <= getMaxValueId()) -// return sets[valueId]; -// else -// return sets[this.nValues]; -// } -// -// @Override -// public int getMaxValueId() { -// return this.nValues - 1; -// } -// -// public int getValueIntAt(int i) { -// for (int v = 0; v < nValues; v++) { -// if (sets[v].contains(i)) { -// return v; -// } -// } -// // if v is not in [0..nValues-1], then it must be nValue (NULL) -// return Dictionary.NULL_ID[valueLen]; -// } -// -// private void checkUpdateMode() { -// if (isClosedForChange()) { -// throw new IllegalStateException(); -// } -// if (sets == null) { -// sets = new ConciseSet[nValues + 1]; -// for (int i = 0; i <= nValues; i++) { -// sets[i] = new ConciseSet(); -// } -// } -// } -// -// private boolean isClosedForChange() { -// return closedForChange; -// } -// -// @Override -// public void closeForChange() { -// closedForChange = true; -// } -// -// @Override -// public int getSize() { -// return size; -// } -// -// public List<ImmutableBytesWritable> toBytes() { -// if (isClosedForChange() == false) -// closeForChange(); -// -// List<ImmutableBytesWritable> r = new ArrayList<ImmutableBytesWritable>(nValues + 1); -// for (int i = 0; i <= nValues; i++) { -// r.add(setToBytes(sets[i])); -// } -// return r; -// } -// -// public void fromBytes(List<ImmutableBytesWritable> bytes) { -// assert nValues + 1 == bytes.size(); -// sets = new ConciseSet[nValues + 1]; -// size = 0; -// for (int i = 0; i <= nValues; i++) { -// sets[i] = bytesToSet(bytes.get(i)); -// size += sets[i].size(); -// } -// closedForChange = true; -// } -// -// private ImmutableBytesWritable setToBytes(ConciseSet set) { -// byte[] array; -// if (set.isEmpty()) // ConciseSet.toByteBuffer() throws exception when -// // set is empty -// array = BytesUtil.EMPTY_BYTE_ARRAY; -// else -// array = set.toByteBuffer().array(); -// return new ImmutableBytesWritable(array); -// } -// -// private ConciseSet bytesToSet(ImmutableBytesWritable bytes) { -// if (bytes.get() == null || bytes.getLength() == 0) { -// return new ConciseSet(); -// } else { -// IntBuffer intBuffer = ByteBuffer.wrap(bytes.get(), bytes.getOffset(), bytes.getLength()).asIntBuffer(); -// int[] words = new int[intBuffer.capacity()]; -// intBuffer.get(words); -// return new ConciseSet(words, false); -// } -// } -// -// @Override -// public int hashCode() { -// final int prime = 31; -// int result = 1; -// result = prime * result + (closedForChange ? 1231 : 1237); -// result = prime * result + nValues; -// result = prime * result + Arrays.hashCode(sets); -// result = prime * result + size; -// result = prime * result + valueLen; -// return result; -// } -// -// @Override -// public boolean equals(Object obj) { -// if (this == obj) -// return true; -// if (obj == null) -// return false; -// if (getClass() != obj.getClass()) -// return false; -// BitMapContainer other = (BitMapContainer) obj; -// if (closedForChange != other.closedForChange) -// return false; -// if (nValues != other.nValues) -// return false; -// if (!Arrays.equals(sets, other.sets)) -// return false; -// if (size != other.size) -// return false; -// if (valueLen != other.valueLen) -// return false; -// return true; -// } -// -//}
http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java deleted file mode 100644 index ea35bb4..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ColumnValueContainer.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.index; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; - -import it.uniroma3.mat.extendedset.intset.ConciseSet; - -/** - * @author yangli9 - */ -public interface ColumnValueContainer { - - void append(ImmutableBytesWritable valueBytes); - - void closeForChange(); - - int getSize(); - - // works only after closeForChange() - void getValueAt(int i, ImmutableBytesWritable valueBytes); - - ConciseSet getBitMap(Integer startId, Integer endId); - - int getMaxValueId(); - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java deleted file mode 100644 index e395544..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/CompressedValueContainer.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.index; - -import java.io.IOException; -import java.util.Arrays; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.common.util.Dictionary; - -import com.ning.compress.lzf.LZFDecoder; -import com.ning.compress.lzf.LZFEncoder; - -import it.uniroma3.mat.extendedset.intset.ConciseSet; - -/** - * @author yangli9 - */ -public class CompressedValueContainer implements ColumnValueContainer { - int valueLen; - int nValues; - int cap; - int size; - byte[] uncompressed; - byte[] compressed; - - public CompressedValueContainer(TableRecordInfoDigest digest, int col, int cap) { - this(digest.length(col), digest.getMaxID(col) + 1, cap); - } - - public CompressedValueContainer(int valueLen, int nValues, int cap) { - this.valueLen = valueLen; - this.nValues = nValues; - this.cap = cap; - this.size = 0; - this.uncompressed = null; - this.compressed = null; - } - - @Override - public void append(ImmutableBytesWritable valueBytes) { - checkUpdateMode(); - System.arraycopy(valueBytes.get(), valueBytes.getOffset(), uncompressed, valueLen * size, valueLen); - size++; - } - - @Override - public void getValueAt(int i, ImmutableBytesWritable valueBytes) { - valueBytes.set(uncompressed, valueLen * i, valueLen); - } - - @Override - public ConciseSet getBitMap(Integer startId, Integer endId) { - ConciseSet ret = new ConciseSet(); - int nullId = Dictionary.NULL_ID[valueLen]; - - if (startId == null && endId == null) { - //entry for getting null values - for (int i = 0; i < size; ++i) { - int valueID = BytesUtil.readUnsigned(uncompressed, i * valueLen, valueLen); - if (nullId == valueID) { - ret.add(i); - } - } - return ret; - } - - //normal values - for (int i = 0; i < size; ++i) { - int valueID = BytesUtil.readUnsigned(uncompressed, i * valueLen, valueLen); - if (valueID == nullId) { - continue; - } - - if (startId != null && valueID < startId) { - continue; - } - - if (endId != null && valueID > endId) { - continue; - } - - ret.add(i); - } - return ret; - - } - - @Override - public int getMaxValueId() { - return nValues - 1; - } - - private void checkUpdateMode() { - if (isClosedForChange()) { - throw new IllegalArgumentException(); - } - if (uncompressed == null) { - uncompressed = new byte[valueLen * cap]; - } - } - - private boolean isClosedForChange() { - return compressed != null; - } - - @Override - public void closeForChange() { - checkUpdateMode(); - try { - compressed = LZFEncoder.encode(uncompressed, 0, valueLen * size); - } catch (Exception e) { - throw new RuntimeException("LZF encode failure", e); - } - } - - @Override - public int getSize() { - return size; - } - - public ImmutableBytesWritable toBytes() { - if (isClosedForChange() == false) - closeForChange(); - return new ImmutableBytesWritable(compressed); - } - - public void fromBytes(ImmutableBytesWritable bytes) { - try { - uncompressed = LZFDecoder.decode(bytes.get(), bytes.getOffset(), bytes.getLength()); - } catch (IOException e) { - throw new RuntimeException("LZF decode failure", e); - } - size = cap = uncompressed.length / valueLen; - compressed = BytesUtil.EMPTY_BYTE_ARRAY; // mark closed - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + size; - result = prime * result + valueLen; - result = prime * result + Arrays.hashCode(uncompressed); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - CompressedValueContainer other = (CompressedValueContainer) obj; - if (size != other.size) - return false; - if (valueLen != other.valueLen) - return false; - if (!Bytes.equals(uncompressed, 0, size * valueLen, uncompressed, 0, size * valueLen)) - return false; - return true; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/IncrementalSliceMaker.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/IncrementalSliceMaker.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/IncrementalSliceMaker.java deleted file mode 100644 index 69e016d..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/IncrementalSliceMaker.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.index; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; - -/** - * @author yangli9 - */ -public class IncrementalSliceMaker { - - TableRecordInfo info; - private int nColumns; - int nRecordsCap; - - short shard; - long sliceTimestamp; - int nRecords; - private ColumnValueContainer[] containers; - - transient ImmutableBytesWritable temp = new ImmutableBytesWritable(); - - public IncrementalSliceMaker(TableRecordInfo info, short shard) { - this.info = info; - this.nColumns = info.getDigest().getColumnCount(); - this.nRecordsCap = Math.max(1, info.getDescriptor().getSliceSize()); - - this.shard = shard; - this.sliceTimestamp = Long.MIN_VALUE; - this.nRecords = 0; - this.containers = null; - - doneSlice(); // init containers - } - - private Slice doneSlice() { - Slice r = null; - if (nRecords > 0) { - for (int i = 0; i < nColumns; i++) { - containers[i].closeForChange(); - } - r = new Slice(info.getDigest(), shard, sliceTimestamp, containers); - } - - // reset for next slice - nRecords = 0; - containers = new ColumnValueContainer[nColumns]; - // for (int i : info.getDescriptor().getBitmapColumns()) { - // containers[i] = new CompressedValueContainer(info.getDigest(), i, - // nRecordsCap); - // } - for (int i : info.getDescriptor().getValueColumns()) { - containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap); - } - for (int i : info.getDescriptor().getMetricsColumns()) { - containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap); - } - - return r; - - } - - // NOTE: record must be appended in time order - public Slice append(TableRecord rec) { - if (rec.getShard() != shard) - throw new IllegalStateException(); - - Slice doneSlice = null; - - if (isFull()) { - doneSlice = doneSlice(); - } - - if (nRecords == 0) { - sliceTimestamp = increaseSliceTimestamp(rec.getTimestamp()); - } - - nRecords++; - for (int i = 0; i < nColumns; i++) { - rec.getValueBytes(i, temp); - containers[i].append(temp); - } - - return doneSlice; - } - - private long increaseSliceTimestamp(long timestamp) { - if (timestamp < sliceTimestamp) - throw new IllegalStateException(); - - if (timestamp == sliceTimestamp) - return ++timestamp; // ensure slice timestamp increases - else - return timestamp; - } - - public Slice close() { - Slice doneSlice = doneSlice(); - this.sliceTimestamp = Long.MIN_VALUE; - this.nRecords = 0; - return doneSlice; - } - - private boolean isFull() { - return nRecords >= nRecordsCap; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java deleted file mode 100644 index d42cab0..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/RawTableRecord.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.index; - -import java.util.Arrays; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.dimension.DimensionEncoding; -import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec; -import org.apache.kylin.metadata.datatype.LongMutable; - -/** - */ -public class RawTableRecord implements Cloneable { - TableRecordInfoDigest digest; - private byte[] buf; // consecutive column value IDs (encoded by dictionary) - - public RawTableRecord(TableRecordInfoDigest info) { - this.digest = info; - this.buf = new byte[info.getByteFormLen()]; - reset(); - } - - public RawTableRecord(RawTableRecord another) { - this.digest = another.digest; - this.buf = Bytes.copy(another.buf); - } - - public void reset() { - Arrays.fill(buf, DimensionEncoding.NULL); - } - - public boolean isMetric(int col) { - return digest.isMetrics(col); - } - - public FixedLenMeasureCodec<LongMutable> codec(int col) { - return digest.codec(col); - } - - public final int length(int col) { - return digest.length(col); - } - - public final int offset(int col) { - return digest.offset(col); - } - - public int getColumnCount() { - return digest.getColumnCount(); - } - - public void setValueID(int col, int id) { - BytesUtil.writeUnsigned(id, buf, digest.offset(col), digest.length(col)); - } - - public int getValueID(int col) { - return BytesUtil.readUnsigned(buf, digest.offset(col), digest.length(col)); - } - - public void setValueMetrics(int col, LongMutable value) { - digest.codec(col).write(value, buf, digest.offset(col)); - } - - public String getValueMetric(int col) { - digest.codec(col).read(buf, digest.offset(col)); - return (String) digest.codec(col).getValue(); - } - - public byte[] getBytes() { - return buf; - } - - //TODO is it possible to avoid copying? - public void setBytes(byte[] bytes, int offset, int length) { - assert buf.length == length; - System.arraycopy(bytes, offset, buf, 0, length); - } - - public void setValueBytes(int col, ImmutableBytesWritable bytes) { - System.arraycopy(bytes.get(), bytes.getOffset(), buf, digest.offset(col), digest.length(col)); - } - - public void getValueBytes(int col, ImmutableBytesWritable bytes) { - bytes.set(buf, offset(col), length(col)); - } - - @Override - public Object clone() { - return new RawTableRecord(this); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + Arrays.hashCode(buf); - // result = prime * result + ((digest == null) ? 0 : digest.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - RawTableRecord other = (RawTableRecord) obj; - if (!Arrays.equals(buf, other.buf)) - return false; - return true; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java deleted file mode 100644 index 45c9c0a..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/ShardingSliceBuilder.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.index; - -import java.util.List; - -import com.google.common.collect.Lists; - -public class ShardingSliceBuilder { - - IncrementalSliceMaker[] builders; - - public ShardingSliceBuilder(TableRecordInfo info) { - int sharding = info.getDescriptor().getSharding(); - builders = new IncrementalSliceMaker[sharding]; - for (short i = 0; i < sharding; i++) { - builders[i] = new IncrementalSliceMaker(info, i); - } - } - - // NOTE: record must be appended in time order - public Slice append(TableRecord rec) { - short shard = rec.getShard(); - return builders[shard].append(rec); - } - - public List<Slice> close() { - List<Slice> result = Lists.newArrayList(); - for (IncrementalSliceMaker builder : builders) { - Slice slice = builder.close(); - if (slice != null) - result.add(slice); - } - return result; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java deleted file mode 100644 index dc2c5c4..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/Slice.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.index; - -import java.util.Iterator; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.common.util.Dictionary; - -import com.google.common.base.Objects; - -import it.uniroma3.mat.extendedset.intset.ConciseSet; - -/** - * Within a partition (per timestampGranularity), records are further sliced - * (per sliceLength) to fit into HBASE cell. - * - * @author yangli9 - */ -public class Slice implements Iterable<RawTableRecord>, Comparable<Slice> { - - final TableRecordInfoDigest info; - final int nColumns; - - final short shard; - final long timestamp; - final int nRecords; - final ColumnValueContainer[] containers; - private Dictionary<?>[] localDictionaries; - - public Slice(TableRecordInfoDigest digest, short shard, long timestamp, ColumnValueContainer[] containers) { - this.info = digest; - this.nColumns = digest.getColumnCount(); - - this.shard = shard; - this.timestamp = timestamp; - this.nRecords = containers[0].getSize(); - this.containers = containers; - - assert nColumns == containers.length; - for (int i = 0; i < nColumns; i++) { - assert nRecords == containers[i].getSize(); - } - } - - public Dictionary<?>[] getLocalDictionaries() { - return localDictionaries; - } - - public void setLocalDictionaries(Dictionary<?>[] localDictionaries) { - this.localDictionaries = localDictionaries; - } - - public int getRecordCount() { - return this.nRecords; - } - - public short getShard() { - return shard; - } - - public long getTimestamp() { - return timestamp; - } - - public ColumnValueContainer[] getColumnValueContainers() { - return containers; - } - - public ColumnValueContainer getColumnValueContainer(int col) { - return containers[col]; - } - - public Iterator<RawTableRecord> iterateWithBitmap(final ConciseSet resultBitMap) { - if (resultBitMap == null) { - return this.iterator(); - } else { - final RawTableRecord rec = info.createTableRecordBytes(); - final ImmutableBytesWritable temp = new ImmutableBytesWritable(); - - return new Iterator<RawTableRecord>() { - int i = 0; - int iteratedCount = 0; - int resultSize = resultBitMap.size(); - - @Override - public boolean hasNext() { - return iteratedCount < resultSize; - } - - @Override - public RawTableRecord next() { - while (!resultBitMap.contains(i)) { - i++; - } - for (int col = 0; col < nColumns; col++) { - containers[col].getValueAt(i, temp); - rec.setValueBytes(col, temp); - } - iteratedCount++; - i++; - - return rec; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - }; - } - } - - @Override - public Iterator<RawTableRecord> iterator() { - return new Iterator<RawTableRecord>() { - int i = 0; - RawTableRecord rec = info.createTableRecordBytes(); - ImmutableBytesWritable temp = new ImmutableBytesWritable(); - - @Override - public boolean hasNext() { - return i < nRecords; - } - - @Override - public RawTableRecord next() { - for (int col = 0; col < nColumns; col++) { - containers[col].getValueAt(i, temp); - rec.setValueBytes(col, temp); - } - i++; - return rec; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - }; - } - - /* - * (non-Javadoc) - * - * @see java.lang.Object#hashCode() - */ - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((info == null) ? 0 : info.hashCode()); - result = prime * result + shard; - result = prime * result + (int) (timestamp ^ (timestamp >>> 32)); - return result; - } - - /* - * (non-Javadoc) - * - * @see java.lang.Object#equals(java.lang.Object) - */ - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - Slice other = (Slice) obj; - if (shard != other.shard) { - return false; - } - if (timestamp != other.timestamp) { - return false; - } - return Objects.equal(info, other.info); - } - - @Override - public int compareTo(Slice o) { - int comp = this.shard - o.shard; - if (comp != 0) - return comp; - - comp = (int) (this.timestamp - o.timestamp); - return comp; - } - - public TableRecordInfoDigest getInfo() { - return info; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java deleted file mode 100644 index 0076919..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/SliceBuilder.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kylin.invertedindex.index; - -import java.io.IOException; -import java.util.List; - -import javax.annotation.Nullable; - -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.common.util.StreamingBatch; -import org.apache.kylin.common.util.StreamingMessage; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.invertedindex.util.IIDictionaryBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Function; -import com.google.common.collect.Lists; - -/** - */ -public final class SliceBuilder { - - private static Logger logger = LoggerFactory.getLogger(SliceBuilder.class); - - private final BatchSliceMaker sliceMaker; - private final IIDesc iiDesc; - - public SliceBuilder(IIDesc desc, short shard) { - this.iiDesc = desc; - this.sliceMaker = new BatchSliceMaker(desc, shard); - } - - public Slice buildSlice(StreamingBatch microStreamBatch) throws IOException { - final List<List<String>> messages = Lists.transform(microStreamBatch.getMessages(), new Function<StreamingMessage, List<String>>() { - @Nullable - @Override - public List<String> apply(@Nullable StreamingMessage input) { - return input.getData(); - } - }); - final Dictionary<?>[] dictionaries = IIDictionaryBuilder.buildDictionary(messages, iiDesc); - TableRecordInfo tableRecordInfo = new TableRecordInfo(iiDesc, dictionaries); - return build(messages, tableRecordInfo, dictionaries); - } - - private Slice build(List<List<String>> table, final TableRecordInfo tableRecordInfo, Dictionary<?>[] localDictionary) { - final Slice slice = sliceMaker.makeSlice(tableRecordInfo.getDigest(), Lists.transform(table, new Function<List<String>, TableRecord>() { - @Nullable - @Override - public TableRecord apply(@Nullable List<String> input) { - TableRecord result = tableRecordInfo.createTableRecord(); - for (int i = 0; i < input.size(); i++) { - result.setValueString(i, input.get(i)); - } - return result; - } - })); - slice.setLocalDictionaries(localDictionary); - return slice; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java deleted file mode 100644 index 3ee34be..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecord.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.index; - -import java.util.Arrays; - -import org.apache.commons.lang.ObjectUtils; -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.kylin.common.util.DateFormat; -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.common.util.ShardingHash; -import org.apache.kylin.metadata.datatype.LongMutable; - -/** - * TableRecord extends RawTableRecord by decorating it with a TableRecordInfo. - * - * @author yangli9, honma - */ -public class TableRecord implements Cloneable { - - private TableRecordInfo info; - private RawTableRecord rawRecord; - - public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9; - - public TableRecord(RawTableRecord rawRecord, TableRecordInfo info) { - this.info = info; - this.rawRecord = rawRecord; - } - - public TableRecord(TableRecord another) { - this.info = another.info; - this.rawRecord = (RawTableRecord) another.rawRecord.clone(); - } - - public TableRecordInfo getInfo() { - return info; - } - - @Override - public Object clone() { - return new TableRecord(this); - } - - public void reset() { - rawRecord.reset(); - } - - public byte[] getBytes() { - return rawRecord.getBytes(); - } - - public void setBytes(byte[] bytes, int offset, int length) { - rawRecord.setBytes(bytes, offset, length); - } - - public long getTimestamp() { - String str = getValueString(info.getTimestampColumn()); - return DateFormat.stringToMillis(str); - } - - public int length(int col) { - return rawRecord.length(col); - } - - public void setValueStringWithoutDictionary(int col, String value) { - int offset = info.digest.offset(col); - int length = info.digest.length(col); - byte[] src = value.getBytes(); - if (length >= src.length) { - byte[] dst = rawRecord.getBytes(); - System.arraycopy(src, 0, dst, offset, src.length); - Arrays.fill(dst, offset + src.length, offset + length, ROWKEY_PLACE_HOLDER_BYTE); - } else { - byte[] dst = rawRecord.getBytes(); - System.arraycopy(src, 0, dst, offset, length); - } - } - - public String getValueStringWithoutDictionary(int col) { - int offset = info.digest.offset(col); - int length = info.digest.length(col); - byte[] bytes = rawRecord.getBytes(); - int i; - for (i = 0; i < length; ++i) { - if (bytes[offset + i] == ROWKEY_PLACE_HOLDER_BYTE) { - break; - } - } - return new String(bytes, offset, i); - } - - public void setValueString(int col, String value) { - if (rawRecord.isMetric(col)) { - LongMutable v = rawRecord.codec(col).valueOf(value); - setValueMetrics(col, v); - } else { - final Dictionary<String> dict = info.dict(col); - if (dict != null) { - int id = dict.getIdFromValue(value); - rawRecord.setValueID(col, id); - } else { - setValueStringWithoutDictionary(col, value); - // throw new UnsupportedOperationException("cannot set value when there is no dictionary"); - } - } - } - - /** - * get value of columns which belongs to the original table columns. - * i.e. columns like min_xx, max_yy will never appear - */ - public String getValueString(int col) { - if (rawRecord.isMetric(col)) { - return getValueMetric(col); - } else { - final Dictionary<String> dict = info.dict(col); - if (dict != null) { - return dict.getValueFromId(rawRecord.getValueID(col)); - } else { - return getValueStringWithoutDictionary(col); - // throw new UnsupportedOperationException("cannot get value when there is no dictionary"); - } - } - } - - public void getValueBytes(int col, ImmutableBytesWritable bytes) { - rawRecord.getValueBytes(col, bytes); - } - - private void setValueMetrics(int col, LongMutable value) { - rawRecord.setValueMetrics(col, value); - } - - private String getValueMetric(int col) { - return rawRecord.getValueMetric(col); - } - - public short getShard() { - int timestampID = rawRecord.getValueID(info.getTimestampColumn()); - return ShardingHash.getShard(timestampID, info.getDescriptor().getSharding()); - } - - @Override - public String toString() { - StringBuilder buf = new StringBuilder("["); - for (int col = 0; col < rawRecord.getColumnCount(); col++) { - if (col > 0) - buf.append(","); - buf.append(getValueString(col)); - } - buf.append("]"); - return buf.toString(); - } - - @Override - public int hashCode() { - if (rawRecord != null) { - return rawRecord.hashCode(); - } else { - return 0; - } - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - TableRecord other = (TableRecord) obj; - return ObjectUtils.equals(other.rawRecord, this.rawRecord); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java deleted file mode 100644 index 628a08d..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfo.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.index; - -import java.util.List; - -import org.apache.kylin.common.util.Array; -import org.apache.kylin.common.util.Dictionary; -import org.apache.kylin.invertedindex.IISegment; -import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec; -import org.apache.kylin.invertedindex.model.IIDesc; -import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.model.TblColRef; - -/** - * @author yangli9 - * <p/> - * TableRecordInfo stores application-aware knowledges, while - * TableRecordInfoDigest only stores byte level knowleges - */ -public class TableRecordInfo { - - final IIDesc desc; - - final TableRecordInfoDigest digest; - final Dictionary<?>[] dictionaries; - - public TableRecordInfo(IISegment iiSegment) { - this(iiSegment.getIIDesc()); - } - - public TableRecordInfo(IIDesc desc) { - this(desc, new Dictionary<?>[desc.listAllColumns().size()]); - } - - public TableRecordInfo(IIDesc desc, Dictionary<?>[] dictionaries) { - this.desc = desc; - this.dictionaries = dictionaries; - this.digest = createDigest(desc, dictionaries); - } - - public TableRecordInfoDigest getDigest() { - return digest; - } - - private TableRecordInfoDigest createDigest(IIDesc desc, Dictionary<?>[] dictionaryMap) { - final List<TblColRef> tblColRefs = desc.listAllColumns(); - final int nColumns = tblColRefs.size(); - boolean[] isMetric = new boolean[nColumns]; - int[] lengths = new int[nColumns]; - int[] dictMaxIds = new int[nColumns]; - String[] dataTypes = new String[nColumns]; - for (int i = 0; i < nColumns; ++i) { - final TblColRef tblColRef = tblColRefs.get(i); - isMetric[i] = desc.isMetricsCol(i); - dataTypes[i] = tblColRef.getDatatype(); - if (isMetric[i]) { - lengths[i] = FixedLenMeasureCodec.get(DataType.getType(tblColRef.getColumnDesc().getDatatype())).getLength(); - } else { - if (Array.isEmpty(dictionaryMap)) { - final DataType dataType = DataType.getType(tblColRef.getColumnDesc().getDatatype()); - if (dataType.isNumberFamily()) { - lengths[i] = 16; - } else if (dataType.isStringFamily()) { - lengths[i] = 256; - } else if (dataType.isDateTimeFamily()) { - lengths[i] = 19; - } else { - throw new RuntimeException("invalid data type:" + dataType); - } - dictMaxIds[i] = Integer.MAX_VALUE; - } else { - final Dictionary<?> dictionary = dictionaryMap[i]; - lengths[i] = dictionary.getSizeOfId(); - dictMaxIds[i] = dictionary.getMaxId(); - } - } - } - // offsets - int pos = 0; - int[] offsets = new int[nColumns]; - for (int i = 0; i < nColumns; i++) { - offsets[i] = pos; - pos += lengths[i]; - } - - int byteFormLen = pos; - - return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, dataTypes); - } - - public TableRecord createTableRecord() { - return new TableRecord(digest.createTableRecordBytes(), this); - } - - public final IIDesc getDescriptor() { - return desc; - } - - public final List<TblColRef> getColumns() { - return desc.listAllColumns(); - } - - public int findColumn(TblColRef col) { - return desc.findColumn(col); - } - - public int findFactTableColumn(String columnName) { - if (columnName == null) - return -1; - for (int i = 0; i < getColumns().size(); ++i) { - TblColRef tblColRef = getColumns().get(i); - if (tblColRef.isSameAs(desc.getFactTableName(), columnName)) { - return i; - } - } - return -1; - } - - // dimensions go with dictionary - @SuppressWarnings("unchecked") - public Dictionary<String> dict(int col) { - // yes, all dictionaries are string based - return (Dictionary<String>) dictionaries[col]; - } - - public int getTimestampColumn() { - return desc.getTimestampColumn(); - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java deleted file mode 100644 index 9eebdbe..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/index/TableRecordInfoDigest.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.index; - -import java.nio.ByteBuffer; -import java.util.Arrays; - -import org.apache.kylin.common.util.BytesSerializer; -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.invertedindex.measure.FixedLenMeasureCodec; -import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.datatype.LongMutable; - -import com.google.common.base.Objects; - -/** - */ -public class TableRecordInfoDigest { - - private String[] metricDataTypes; - private int nColumns; - private int byteFormLen; - - private int[] offsets;// column offset in byte form row - private int[] dictMaxIds;// max id for each of the dict - private int[] lengths;// length of each encoded dict - private boolean[] isMetric;// whether it's metric or dict - private FixedLenMeasureCodec<?>[] measureCodecs; - - public TableRecordInfoDigest(int nColumns, int byteFormLen, // - int[] offsets, int[] dictMaxIds, int[] lengths, boolean[] isMetric, String[] metricDataTypes) { - this.nColumns = nColumns; - this.byteFormLen = byteFormLen; - this.offsets = offsets; - this.dictMaxIds = dictMaxIds; - this.lengths = lengths; - this.isMetric = isMetric; - this.metricDataTypes = metricDataTypes; - this.measureCodecs = new FixedLenMeasureCodec[nColumns]; - for (int i = 0; i < isMetric.length; i++) { - if (isMetric[i]) { - measureCodecs[i] = FixedLenMeasureCodec.get(DataType.getType(metricDataTypes[i])); - } - } - } - - private TableRecordInfoDigest() { - } - - public int getByteFormLen() { - return byteFormLen; - } - - public boolean isMetrics(int col) { - return isMetric[col]; - } - - public boolean[] isMetrics() { - return isMetric; - } - - public int getColumnCount() { - return nColumns; - } - - public int offset(int col) { - return offsets[col]; - } - - public int length(int col) { - return lengths[col]; - } - - public int getMaxID(int col) { - return dictMaxIds[col]; - } - - public boolean[] getIsMetric() { - return isMetric; - } - - public String[] getMetricDataTypes() { - return metricDataTypes; - } - - public RawTableRecord createTableRecordBytes() { - return new RawTableRecord(this); - } - - @Override - public int hashCode() { - return com.google.common.base.Objects.hashCode(nColumns, offsets, dictMaxIds, lengths, isMetric, metricDataTypes); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj instanceof TableRecordInfoDigest) { - TableRecordInfoDigest other = (TableRecordInfoDigest) obj; - return Objects.equal(this.nColumns, other.nColumns) && Arrays.equals(this.offsets, other.offsets) && Arrays.equals(this.dictMaxIds, other.dictMaxIds) && Arrays.equals(this.lengths, other.lengths) && Arrays.equals(this.isMetric, other.isMetric) && Arrays.equals(this.metricDataTypes, other.metricDataTypes); - } else { - return false; - } - } - - // metrics go with fixed-len codec - @SuppressWarnings("unchecked") - public FixedLenMeasureCodec<LongMutable> codec(int col) { - // yes, all metrics are long currently - return (FixedLenMeasureCodec<LongMutable>) measureCodecs[col]; - } - - public static byte[] serialize(TableRecordInfoDigest o) { - ByteBuffer buf = ByteBuffer.allocate(BytesSerializer.SERIALIZE_BUFFER_SIZE); - serializer.serialize(o, buf); - byte[] result = new byte[buf.position()]; - System.arraycopy(buf.array(), 0, result, 0, buf.position()); - return result; - } - - public static TableRecordInfoDigest deserialize(byte[] bytes) { - return serializer.deserialize(ByteBuffer.wrap(bytes)); - } - - public static TableRecordInfoDigest deserialize(ByteBuffer buffer) { - return serializer.deserialize(buffer); - } - - private static final BytesSerializer<TableRecordInfoDigest> serializer = new BytesSerializer<TableRecordInfoDigest>() { - - @Override - public void serialize(TableRecordInfoDigest value, ByteBuffer out) { - BytesUtil.writeVInt(value.nColumns, out); - BytesUtil.writeVInt(value.byteFormLen, out); - BytesUtil.writeIntArray(value.offsets, out); - BytesUtil.writeIntArray(value.dictMaxIds, out); - BytesUtil.writeIntArray(value.lengths, out); - BytesUtil.writeBooleanArray(value.isMetric, out); - BytesUtil.writeAsciiStringArray(value.metricDataTypes, out); - - } - - @Override - public TableRecordInfoDigest deserialize(ByteBuffer in) { - TableRecordInfoDigest result = new TableRecordInfoDigest(); - result.nColumns = BytesUtil.readVInt(in); - result.byteFormLen = BytesUtil.readVInt(in); - result.offsets = BytesUtil.readIntArray(in); - result.dictMaxIds = BytesUtil.readIntArray(in); - result.lengths = BytesUtil.readIntArray(in); - result.isMetric = BytesUtil.readBooleanArray(in); - result.metricDataTypes = BytesUtil.readAsciiStringArray(in); - return result; - } - - }; -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java deleted file mode 100644 index 0ec2638..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedHLLCodec.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -package org.apache.kylin.invertedindex.measure; - -import java.nio.ByteBuffer; - -import org.apache.kylin.measure.hllc.HyperLogLogPlusCounter; -import org.apache.kylin.metadata.datatype.DataType; - -/** - */ -public class FixedHLLCodec extends FixedLenMeasureCodec<HyperLogLogPlusCounter> { - - private DataType type; - private int presision; - private HyperLogLogPlusCounter current; - - public FixedHLLCodec(DataType type) { - this.type = type; - this.presision = type.getPrecision(); - this.current = new HyperLogLogPlusCounter(this.presision); - } - - @Override - public int getLength() { - return 1 << presision; - } - - @Override - public DataType getDataType() { - return type; - } - - @Override - public HyperLogLogPlusCounter valueOf(String value) { - current.clear(); - if (value != null) - current.add(value.getBytes()); - return current; - } - - @Override - public Object getValue() { - return current; - } - - @Override - public HyperLogLogPlusCounter read(byte[] buf, int offset) { - current.readRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset)); - return current; - } - - @Override - public void write(HyperLogLogPlusCounter v, byte[] buf, int offset) { - v.writeRegistersArray(ByteBuffer.wrap(buf, offset, buf.length - offset)); - } - - @Override - public HyperLogLogPlusCounter read(ByteBuffer buffer) { - current.readRegistersArray(buffer); - return current; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java deleted file mode 100644 index bd952a1..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedLenMeasureCodec.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.measure; - -import java.nio.ByteBuffer; - -import org.apache.kylin.measure.hllc.HLLCMeasureType; -import org.apache.kylin.metadata.datatype.DataType; - -abstract public class FixedLenMeasureCodec<T> { - - public static FixedLenMeasureCodec<?> get(DataType type) { - if (HLLCMeasureType.DATATYPE_HLLC.equals(type.getName())) { - return new FixedHLLCodec(type); - } else { - return new FixedPointLongCodec(type); - } - } - - abstract public int getLength(); - - abstract public DataType getDataType(); - - abstract public T valueOf(String value); - - abstract public Object getValue(); - - abstract public T read(byte[] buf, int offset); - - abstract public void write(T v, byte[] buf, int offset); - - abstract public T read(ByteBuffer buffer); - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java deleted file mode 100644 index 67e5158..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/measure/FixedPointLongCodec.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.measure; - -import java.nio.ByteBuffer; - -import org.apache.kylin.common.util.BytesUtil; -import org.apache.kylin.metadata.datatype.DataType; -import org.apache.kylin.metadata.datatype.LongMutable; - -public class FixedPointLongCodec extends FixedLenMeasureCodec<LongMutable> { - - private static final int SIZE = 8; - // number of digits after decimal point - int scale; - DataType type; - // avoid massive object creation - LongMutable current = new LongMutable(); - - public FixedPointLongCodec(DataType type) { - this.type = type; - this.scale = Math.max(0, type.getScale()); - } - - @Override - public int getLength() { - return SIZE; - } - - @Override - public DataType getDataType() { - return type; - } - - long getValueIgnoringDecimalPoint(String value) { - int index = value.indexOf('.'); - - if (index == 0 || index == value.length() - 1) { - throw new RuntimeException("Bad decimal format: " + value); - } else if (index < 0) { - return Long.valueOf(value) * (int) Math.pow(10, scale); - } else { - StringBuilder sb = new StringBuilder(); - sb.append(value.substring(0, index)); - - //if there are more than scale digits after the decimal point, the tail will be discarded - int end = Math.min(value.length(), index + scale + 1); - sb.append(value.substring(index + 1, end)); - int diff = index + scale + 1 - value.length(); - //if there are less than scale digits after the decimal point, the tail will be compensated - for (int i = 0; i < diff; i++) { - sb.append('0'); - } - return Long.valueOf(sb.toString()); - } - } - - String restoreDecimalPoint(long value) { - if (scale < 0) { - throw new RuntimeException("Bad scale: " + scale + " with value: " + value); - } else if (scale == 0) { - return Long.toString(value); - } else { - return String.format("%." + scale + "f", value / (Math.pow(10, scale))); - } - } - - @Override - public LongMutable valueOf(String value) { - if (value == null) - current.set(0L); - else - current.set(getValueIgnoringDecimalPoint(value)); - return current; - } - - @Override - public String getValue() { - if (scale == 0) - return current.toString(); - else - return restoreDecimalPoint(current.get()); - } - - @Override - public LongMutable read(byte[] buf, int offset) { - current.set(BytesUtil.readLong(buf, offset, SIZE)); - return current; - } - - @Override - public void write(LongMutable v, byte[] buf, int offset) { - BytesUtil.writeLong(v == null ? 0 : v.get(), buf, offset, SIZE); - } - - @Override - public LongMutable read(ByteBuffer buffer) { - current.set(BytesUtil.readLong(buffer, SIZE)); - return current; - } -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java deleted file mode 100644 index 66fb67d..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIDesc.java +++ /dev/null @@ -1,418 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.model; - -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; - -import org.apache.commons.net.util.Base64; -import org.apache.kylin.common.KylinConfig; -import org.apache.kylin.common.persistence.ResourceStore; -import org.apache.kylin.common.persistence.RootPersistentEntity; -import org.apache.kylin.common.util.Bytes; -import org.apache.kylin.common.util.JsonUtil; -import org.apache.kylin.common.util.StringUtil; -import org.apache.kylin.metadata.MetadataConstants; -import org.apache.kylin.metadata.MetadataManager; -import org.apache.kylin.metadata.model.ColumnDesc; -import org.apache.kylin.metadata.model.DataModelDesc; -import org.apache.kylin.metadata.model.FunctionDesc; -import org.apache.kylin.metadata.model.IEngineAware; -import org.apache.kylin.metadata.model.IStorageAware; -import org.apache.kylin.metadata.model.MeasureDesc; -import org.apache.kylin.metadata.model.ModelDimensionDesc; -import org.apache.kylin.metadata.model.ParameterDesc; -import org.apache.kylin.metadata.model.TableDesc; -import org.apache.kylin.metadata.model.TblColRef; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; - -/** - * @author yangli9 - */ -@SuppressWarnings("serial") -@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) -public class IIDesc extends RootPersistentEntity { - - public static final String HBASE_FAMILY = "f"; - public static final String HBASE_QUALIFIER = "c"; - public static final String HBASE_DICTIONARY = "d"; - public static final byte[] HBASE_FAMILY_BYTES = Bytes.toBytes(HBASE_FAMILY); - public static final byte[] HBASE_QUALIFIER_BYTES = Bytes.toBytes(HBASE_QUALIFIER); - public static final byte[] HBASE_DICTIONARY_BYTES = Bytes.toBytes(HBASE_DICTIONARY); - - private KylinConfig config; - private DataModelDesc model; - - @JsonProperty("name") - private String name; - @JsonProperty("model_name") - private String modelName; - @JsonProperty("timestamp_dimension") - private String timestampDimension; - @JsonProperty("bitmap_dimensions") - private List<ModelDimensionDesc> bitmapDimensions = Collections.emptyList(); - @JsonProperty("value_dimensions") - private List<ModelDimensionDesc> valueDimensions; - @JsonProperty("metrics") - private String[] metricNames; - @JsonProperty("sharding") - private short sharding = 1; // parallelism - @JsonProperty("slice_size") - private int sliceSize = 50000; // no. rows - - @JsonProperty("engine_type") - private int engineType = IEngineAware.ID_MR_II; - - @JsonProperty("storage_type") - private int storageType = IStorageAware.ID_HBASE; - - @JsonProperty("signature") - private String signature; - - // computed - private List<TableDesc> allTables = Lists.newArrayList(); - private List<TblColRef> allColumns = Lists.newArrayList(); - private List<TblColRef> allDimensions = Lists.newArrayList(); - private int tsCol; - private int[] bitmapCols; - private int[] valueCols; - private int[] metricsCols; - private BitSet metricsColSet; - private List<MeasureDesc> measureDescs; - - public void init(MetadataManager metadataManager) { - - config = metadataManager.getConfig(); - - if (this.modelName == null || this.modelName.length() == 0) { - throw new RuntimeException("The cubeDesc '" + this.getName() + "' doesn't have data model specified."); - } - - this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName); - - if (this.model == null) { - throw new RuntimeException("No data model found with name '" + modelName + "'."); - } - - timestampDimension = timestampDimension.toUpperCase(); - - // capitalize - ModelDimensionDesc.capicalizeStrings(bitmapDimensions); - ModelDimensionDesc.capicalizeStrings(valueDimensions); - StringUtil.toUpperCaseArray(metricNames, metricNames); - - // retrieve all columns and all tables, and make available measure to ii - HashSet<String> allTableNames = Sets.newHashSet(); - measureDescs = Lists.newArrayList(); - measureDescs.add(makeCountMeasure()); - for (ModelDimensionDesc modelDimensionDesc : Iterables.concat(bitmapDimensions, valueDimensions)) { - TableDesc tableDesc = this.getTableDesc(modelDimensionDesc.getTable()); - for (String column : modelDimensionDesc.getColumns()) { - ColumnDesc columnDesc = tableDesc.findColumnByName(column); - TblColRef tcr = columnDesc.getRef(); - allColumns.add(tcr); - allDimensions.add(tcr); - measureDescs.add(makeHLLMeasure(columnDesc, "hllc10")); - } - - if (!allTableNames.contains(tableDesc.getIdentity())) { - allTableNames.add(tableDesc.getIdentity()); - allTables.add(tableDesc); - } - } - for (String column : metricNames) { - TableDesc tableDesc = this.getTableDesc(this.getFactTableName()); - ColumnDesc columnDesc = tableDesc.findColumnByName(column); - allColumns.add(columnDesc.getRef()); - measureDescs.add(makeNormalMeasure("SUM", columnDesc)); - measureDescs.add(makeNormalMeasure("MIN", columnDesc)); - measureDescs.add(makeNormalMeasure("MAX", columnDesc)); - if (!allTableNames.contains(tableDesc.getIdentity())) { - allTableNames.add(tableDesc.getIdentity()); - allTables.add(tableDesc); - } - } - - // indexing for each type of columns - bitmapCols = new int[ModelDimensionDesc.getColumnCount(bitmapDimensions)]; - valueCols = new int[ModelDimensionDesc.getColumnCount(valueDimensions)]; - metricsCols = new int[metricNames.length]; - metricsColSet = new BitSet(this.getTableDesc(this.getFactTableName()).getColumnCount()); - - int totalIndex = 0; - for (int i = 0; i < bitmapCols.length; ++i, ++totalIndex) { - bitmapCols[i] = totalIndex; - } - for (int i = 0; i < valueCols.length; ++i, ++totalIndex) { - valueCols[i] = totalIndex; - } - for (int i = 0; i < metricsCols.length; ++i, ++totalIndex) { - metricsCols[i] = totalIndex; - metricsColSet.set(totalIndex); - } - - // partitioning column - tsCol = -1; - for (int i = 0; i < allColumns.size(); ++i) { - TblColRef col = allColumns.get(i); - - if (col.isSameAs(this.getFactTableName(), this.timestampDimension)) { - tsCol = i; - break; - } - } - if (tsCol < 0) - throw new RuntimeException("timestamp_dimension is not in bitmapDimensions or valueDimensions"); - } - - private TableDesc getTableDesc(String tableName) { - return MetadataManager.getInstance(this.config).getTableDesc(tableName); - } - - public String getResourcePath() { - return getIIDescResourcePath(name); - } - - public static String getIIDescResourcePath(String descName) { - return ResourceStore.II_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX; - } - - public List<MeasureDesc> getMeasures() { - return measureDescs; - } - - public List<FunctionDesc> listAllFunctions() { - List<FunctionDesc> functions = new ArrayList<FunctionDesc>(); - for (MeasureDesc m : measureDescs) { - functions.add(m.getFunction()); - } - return functions; - } - - private MeasureDesc makeNormalMeasure(String func, ColumnDesc columnDesc) { - String columnName = columnDesc.getName(); - String returnType = columnDesc.getTypeName(); - MeasureDesc measureDesc = new MeasureDesc(); - FunctionDesc f1 = new FunctionDesc(); - f1.setExpression(func); - ParameterDesc p1 = new ParameterDesc(); - p1.setType("column"); - p1.setValue(columnName); - p1.setColRefs(ImmutableList.of(columnDesc.getRef())); - f1.setParameter(p1); - f1.setReturnType(returnType); - if (f1.isSum() && f1.getReturnDataType().isIntegerFamily()) { - f1.setReturnType("bigint"); - } - - measureDesc.setFunction(f1); - measureDesc.setName(func + "_" + columnName); - return measureDesc; - } - - /** - * - * @param hllType represents the precision - */ - private MeasureDesc makeHLLMeasure(ColumnDesc columnDesc, String hllType) { - String columnName = columnDesc.getName(); - MeasureDesc measureDesc = new MeasureDesc(); - FunctionDesc f1 = new FunctionDesc(); - f1.setExpression("COUNT_DISTINCT"); - ParameterDesc p1 = new ParameterDesc(); - p1.setType("column"); - p1.setValue(columnName); - p1.setColRefs(ImmutableList.of(columnDesc.getRef())); - f1.setParameter(p1); - f1.setReturnType(hllType); - measureDesc.setFunction(f1); - measureDesc.setName("COUNT_DISTINCT" + "_" + columnName); - return measureDesc; - } - - private MeasureDesc makeCountMeasure() { - MeasureDesc measureDesc = new MeasureDesc(); - FunctionDesc f1 = new FunctionDesc(); - f1.setExpression("COUNT"); - ParameterDesc p1 = new ParameterDesc(); - p1.setType("constant"); - p1.setValue("1"); - f1.setParameter(p1); - f1.setReturnType("bigint"); - measureDesc.setFunction(f1); - measureDesc.setName("COUNT_1"); - return measureDesc; - } - - /** - * at first stage the only table in II is fact table, tables - * - * @return - */ - public List<TableDesc> listTables() { - return allTables; - } - - public List<TblColRef> listAllColumns() { - return allColumns; - } - - public List<TblColRef> listAllDimensions() { - return allDimensions; - } - - public TblColRef findColumnRef(String table, String column) { - ColumnDesc columnDesc = this.getTableDesc(table).findColumnByName(column); - return columnDesc.getRef(); - } - - public int findColumn(TblColRef col) { - return this.allColumns.indexOf(col); - } - - public KylinConfig getConfig() { - return config; - } - - public String getName() { - return name; - } - - public String getModelName() { - return modelName; - } - - public void setModelName(String modelName) { - this.modelName = modelName; - } - - public DataModelDesc getModel() { - return model; - } - - public void setModel(DataModelDesc model) { - this.model = model; - } - - public int getTimestampColumn() { - return tsCol; - } - - public int[] getBitmapColumns() { - return bitmapCols; - } - - public int[] getValueColumns() { - return valueCols; - } - - public int[] getMetricsColumns() { - return metricsCols; - } - - public short getSharding() { - return sharding; - } - - public int getSliceSize() { - return sliceSize; - } - - public String getSignature() { - return signature; - } - - public void setSignature(String signature) { - this.signature = signature; - } - - public boolean isMetricsCol(TblColRef col) { - if (!col.getTable().equalsIgnoreCase(this.getFactTableName())) - return false; - return isMetricsCol(this.findColumn(col)); - } - - public boolean isMetricsCol(int index) { - return metricsColSet.get(index); - } - - /** - * the returned fact table name is guaranteed to be in the form of db.table - * - * @return - */ - public String getFactTableName() { - return this.model.getFactTable().toUpperCase(); - } - - public String getTimestampDimension() { - return timestampDimension; - } - - public void setName(String name) { - this.name = name; - } - - public String calculateSignature() { - MessageDigest md = null; - try { - md = MessageDigest.getInstance("MD5"); - StringBuilder sigString = new StringBuilder(); - sigString.append(this.name).append("|").append(this.getFactTableName()).append("|").append(timestampDimension).append("|").append(JsonUtil.writeValueAsString(this.bitmapDimensions)).append("|").append(JsonUtil.writeValueAsString(valueDimensions)).append("|").append(JsonUtil.writeValueAsString(this.metricNames)).append("|").append(sharding).append("|").append(sliceSize); - - byte[] signature = md.digest(sigString.toString().getBytes()); - return new String(Base64.encodeBase64(signature)); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException("Failed to calculate signature"); - } catch (JsonProcessingException e) { - throw new RuntimeException("Failed to calculate signature"); - } - - } - - public int getStorageType() { - return storageType; - } - - public void setStorageType(int storageType) { - this.storageType = storageType; - } - - public int getEngineType() { - return engineType; - } - - public void setEngineType(int engineType) { - this.engineType = engineType; - } - -} http://git-wip-us.apache.org/repos/asf/kylin/blob/2cc0b9c4/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java ---------------------------------------------------------------------- diff --git a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java b/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java deleted file mode 100644 index 21e5677..0000000 --- a/invertedindex/src/main/java/org/apache/kylin/invertedindex/model/IIJoinedFlatTableDesc.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -package org.apache.kylin.invertedindex.model; - -import java.util.List; - -import org.apache.kylin.metadata.model.DataModelDesc; -import org.apache.kylin.metadata.model.IJoinedFlatTableDesc; -import org.apache.kylin.metadata.model.IntermediateColumnDesc; -import org.apache.kylin.metadata.model.TblColRef; - -import com.google.common.collect.Lists; - -/** - */ -public class IIJoinedFlatTableDesc implements IJoinedFlatTableDesc { - - private IIDesc iiDesc; - private String tableName; - private List<IntermediateColumnDesc> columnList = Lists.newArrayList(); - - public IIJoinedFlatTableDesc(IIDesc iiDesc) { - this.iiDesc = iiDesc; - parseIIDesc(); - } - - private void parseIIDesc() { - this.tableName = "kylin_intermediate_ii_" + iiDesc.getName(); - - int columnIndex = 0; - for (TblColRef col : iiDesc.listAllColumns()) { - columnList.add(new IntermediateColumnDesc(String.valueOf(columnIndex), col)); - columnIndex++; - } - } - - @Override - public String getTableName() { - return tableName + "_" + "II_Flat"; - } - - @Override - public List<IntermediateColumnDesc> getColumnList() { - return columnList; - } - - @Override - public DataModelDesc getDataModel() { - return iiDesc.getModel(); - } - -}