http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java new file mode 100644 index 0000000..e7068b0 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store; + +import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeByteMeasureChunkStore; +import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeDoubleMeasureChunkStore; +import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeIntMeasureChunkStore; +import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeLongMeasureChunkStore; +import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeShortMeasureChunkStore; +import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeByteMeasureChunkStore; +import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeDoubleMeasureChunkStore; +import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeIntMeasureChunkStore; +import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeLongMeasureChunkStore; +import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeShortMeasureChunkStore; +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.util.CarbonProperties; +import org.apache.carbondata.core.util.ValueCompressionUtil.DataType; + +/** + * Factory class for getting the measure store type + */ +public class MeasureChunkStoreFactory { + + /** + * instance type + */ + public static final MeasureChunkStoreFactory INSTANCE = new MeasureChunkStoreFactory(); + + /** + * is unsafe + */ + private static final boolean isUnsafe; + + static { + isUnsafe = Boolean.parseBoolean(CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION, + CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE)); + } + + private MeasureChunkStoreFactory() { + } + + /** + * Below method will be used to get the measure data chunk store based on data type + * + * @param dataType data type + * @param numberOfRows number of rows + * @return measure chunk store + */ + public MeasureDataChunkStore getMeasureDataChunkStore(DataType dataType, int numberOfRows) { + if (isUnsafe) { + switch (dataType) { + case DATA_BYTE: + return new SafeByteMeasureChunkStore(numberOfRows); + case DATA_SHORT: + return new SafeShortMeasureChunkStore(numberOfRows); + case DATA_INT: + return new SafeIntMeasureChunkStore(numberOfRows); + case DATA_LONG: + return new SafeLongMeasureChunkStore(numberOfRows); + case DATA_DOUBLE: + return new SafeDoubleMeasureChunkStore(numberOfRows); + default: + return new SafeDoubleMeasureChunkStore(numberOfRows); + } + } else { + switch (dataType) { + case DATA_BYTE: + return new UnsafeByteMeasureChunkStore(numberOfRows); + case DATA_SHORT: + return new UnsafeShortMeasureChunkStore(numberOfRows); + case DATA_INT: + return new UnsafeIntMeasureChunkStore(numberOfRows); + case DATA_LONG: + return new UnsafeLongMeasureChunkStore(numberOfRows); + case DATA_DOUBLE: + return new UnsafeDoubleMeasureChunkStore(numberOfRows); + default: + return new UnsafeDoubleMeasureChunkStore(numberOfRows); + } + } + } +}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java new file mode 100644 index 0000000..fed4ab4 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store; + +import java.math.BigDecimal; + +/** + * Responsibility is store the measure data in memory, + * memory can be on heap or offheap based on the user configuration + */ +public interface MeasureDataChunkStore<T> { + + /** + * Below method will be used to put the data to memory + * + * @param data + */ + void putData(T data); + + /** + * to get byte value + * + * @param index + * @return byte value based on index + */ + byte getByte(int index); + + /** + * to get the short value + * + * @param index + * @return short value based on index + */ + short getShort(int index); + + /** + * to get the int value + * + * @param index + * @return int value based on index + */ + int getInt(int index); + + /** + * to get the long value + * + * @param index + * @return long value based on index + */ + long getLong(int index); + + /** + * to get the double value + * + * @param index + * @return double value based on index + */ + double getDouble(int index); + + /** + * To get the bigdecimal value + * + * @param index + * @return bigdecimal value based on index + */ + BigDecimal getBigDecimal(int index); + + /** + * To free the occupied memory + */ + void freeMemory(); +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java new file mode 100644 index 0000000..0ed2218 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; + +import org.apache.carbondata.core.carbon.datastore.chunk.store.DimensionDataChunkStore; + +/** + * Responsibility is to store dimension data + */ +public abstract class SafeAbsractDimensionDataChunkStore implements DimensionDataChunkStore { + + /** + * data chunk for dimension column + */ + protected byte[] data; + + /** + * inverted index + */ + protected int[] invertedIndex; + + /** + * inverted index reverser + */ + protected int[] invertedIndexReverse; + + /** + * to check whether dimension column was explicitly sorted or not + */ + protected boolean isExplictSorted; + + /** + * Constructor + * + * @param totalSize total size of the data to be kept + * @param isInvertedIdex is inverted index present + * @param numberOfRows total number of rows + */ + public SafeAbsractDimensionDataChunkStore(boolean isInvertedIdex) { + this.isExplictSorted = isInvertedIdex; + } + + /** + * Below method will be used to put the rows and its metadata in offheap + * + * @param invertedIndex inverted index to be stored + * @param invertedIndexReverse inverted index reverse to be stored + * @param data data to be stored + */ + @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse, + final byte[] data) { + this.data = data; + this.invertedIndex = invertedIndex; + this.invertedIndexReverse = invertedIndexReverse; + } + + /** + * Below method will be used to free the memory occupied by the column chunk + */ + @Override public void freeMemory() { + // do nothing as GC will take care of freeing memory + } + + /** + * Below method will be used to get the inverted index + * + * @param rowId row id + * @return inverted index based on row id passed + */ + @Override public int getInvertedIndex(int rowId) { + return invertedIndex[rowId]; + } + + /** + * Below method will be used to get the surrogate key of the based on the row + * id passed + * + * @param rowId row id + * @return surrogate key + */ + @Override public int getSurrogate(int rowId) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * @return size of each column value + */ + @Override public int getColumnValueSize() { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * @return whether column was explicitly sorted or not + */ + @Override public boolean isExplicitSorted() { + return isExplictSorted; + } + + /** + * Below method will be used to fill the row values to data array + * + * @param rowId row id of the data to be filled + * @param data buffer in which data will be filled + * @param offset off the of the buffer + */ + @Override public void fillRow(int rowId, byte[] data, int offset) { + throw new UnsupportedOperationException("Operation not supported"); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java new file mode 100644 index 0000000..0daeaa4 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; + +import java.math.BigDecimal; + +import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore; + +/** + * Responsibility is store the measure data in memory, + */ +public abstract class SafeAbstractMeasureDataChunkStore<T> implements + MeasureDataChunkStore<T> { + + /** + * number of rows + */ + protected int numberOfRows; + + public SafeAbstractMeasureDataChunkStore(int numberOfRows) { + this.numberOfRows = numberOfRows; + } + + /** + * to get the byte value + * + * @param index + * @return byte value based on index + */ + @Override + public byte getByte(int index) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * to get the short value + * + * @param index + * @return short value based on index + */ + @Override + public short getShort(int index) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * to get the int value + * + * @param index + * @return int value based on index + */ + @Override + public int getInt(int index) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * to get the long value + * + * @param index + * @return long value based on index + */ + @Override + public long getLong(int index) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * to get the double value + * + * @param index + * @return double value based on index + */ + @Override + public double getDouble(int index) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * To get the bigdecimal value + * + * @param index + * @return bigdecimal value based on index + */ + @Override + public BigDecimal getBigDecimal(int index) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * To free the occupied memory + */ + @Override + public void freeMemory() { + // do nothing as GC will take care of freeing the memory + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java new file mode 100644 index 0000000..1f69cd2 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; + +/** + * Responsible for storing Byte array data to memory. + */ +public class SafeByteMeasureChunkStore extends + SafeAbstractMeasureDataChunkStore<byte[]> { + + /** + * data + */ + private byte[] data; + + public SafeByteMeasureChunkStore(int numberOfRows) { + super(numberOfRows); + } + + /** + * Below method will be used to put byte array data to memory + * + * @param data + */ + @Override + public void putData(byte[] data) { + this.data = data; + } + + /** + * to get the byte value + * + * @param index + * @return byte value based on index + */ + @Override + public byte getByte(int index) { + return this.data[index]; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java new file mode 100644 index 0000000..dc485c6 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; + +/** + * Below class will be used to store the measure values of double data type + */ +public class SafeDoubleMeasureChunkStore extends + SafeAbstractMeasureDataChunkStore<double[]> { + + /** + * data + */ + private double[] data; + + public SafeDoubleMeasureChunkStore(int numberOfRows) { + super(numberOfRows); + } + + /** + * Below method will be used to store double array data + * + * @param data + */ + @Override + public void putData(double[] data) { + this.data = data; + } + + /** + * to get the double value + * + * @param index + * @return double value based on index + */ + @Override + public double getDouble(int index) { + return this.data[index]; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java new file mode 100644 index 0000000..3f5afc5 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; + +import org.apache.carbondata.core.util.ByteUtil; + +/** + * Below class will be used to store fixed length dimension data + */ +public class SafeFixedLengthDimensionDataChunkStore extends SafeAbsractDimensionDataChunkStore { + + /** + * Size of each value + */ + private int columnValueSize; + + public SafeFixedLengthDimensionDataChunkStore(boolean isInvertedIndex, int columnValueSize) { + super(isInvertedIndex); + this.columnValueSize = columnValueSize; + } + + /** + * Below method will be used to get the row based inverted index + * + * @param rowId Inverted index + */ + @Override public byte[] getRow(int rowId) { + // if column was explicitly sorted we need to get the rowid based inverted index reverse + if (isExplictSorted) { + rowId = invertedIndexReverse[rowId]; + } + // creating a row + byte[] row = new byte[columnValueSize]; + //copy the row from data chunk based on offset + // offset position will be index * each column value length + System.arraycopy(this.data, rowId * columnValueSize, row, 0, columnValueSize); + return row; + } + + /** + * Below method will be used to get the surrogate key of the based on the row + * id passed + * + * @param rowId row id + * @return surrogate key + */ + @Override public int getSurrogate(int index) { + // if column was explicitly sorted we need to get the rowid based inverted index reverse + if (isExplictSorted) { + index = invertedIndexReverse[index]; + } + // below part is to convert the byte array to surrogate value + int startOffsetOfData = index * columnValueSize; + int surrogate = 0; + for (int i = 0; i < columnValueSize; i++) { + surrogate <<= 8; + surrogate ^= data[startOffsetOfData] & 0xFF; + startOffsetOfData++; + } + return surrogate; + } + + /** + * Below method will be used to fill the row values to buffer array + * + * @param rowId row id of the data to be filled + * @param data buffer in which data will be filled + * @param offset off the of the buffer + */ + @Override public void fillRow(int rowId, byte[] buffer, int offset) { + // if column was explicitly sorted we need to get the rowid based inverted index reverse + if (isExplictSorted) { + rowId = invertedIndexReverse[rowId]; + } + //copy the row from memory block based on offset + // offset position will be index * each column value length + System.arraycopy(data, rowId * columnValueSize, buffer, offset, columnValueSize); + } + + /** + * @return size of each column value + */ + @Override public int getColumnValueSize() { + return columnValueSize; + } + + /** + * to compare the two byte array + * + * @param index index of first byte array + * @param compareValue value of to be compared + * @return compare result + */ + @Override public int compareTo(int index, byte[] compareValue) { + return ByteUtil.UnsafeComparer.INSTANCE + .compareTo(data, index * columnValueSize, columnValueSize, compareValue, 0, + columnValueSize); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java new file mode 100644 index 0000000..29b77cb --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; + +/** + * Responsible for storing int array data to memory. + */ +public class SafeIntMeasureChunkStore extends + SafeAbstractMeasureDataChunkStore<int[]> { + + /** + * data + */ + private int[] data; + + public SafeIntMeasureChunkStore(int numberOfRows) { + super(numberOfRows); + } + + /** + * Below method will be used to put int array data to memory + * + * @param data + */ + @Override + public void putData(int[] data) { + this.data = data; + } + + /** + * to get the int value + * + * @param index + * @return int value based on index + */ + @Override + public int getInt(int index) { + return this.data[index]; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java new file mode 100644 index 0000000..3fd0370 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; + +/** + * Below class will be used to store the measure values of long data type + * + */ +public class SafeLongMeasureChunkStore extends + SafeAbstractMeasureDataChunkStore<long[]> { + + /** + * data + */ + private long[] data; + + public SafeLongMeasureChunkStore(int numberOfRows) { + super(numberOfRows); + } + + /** + * Below method will be used to store long array data + * + * @param data + */ + @Override + public void putData(long[] data) { + this.data = data; + } + + /** + * to get the long value + * + * @param index + * @return long value based on index + */ + @Override + public long getLong(int index) { + return this.data[index]; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java new file mode 100644 index 0000000..269ad79 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; + +/** + * Below class will be used to store the measure values of short data type + * + */ +public class SafeShortMeasureChunkStore extends + SafeAbstractMeasureDataChunkStore<short[]> { + + /** + * data + */ + private short[] data; + + public SafeShortMeasureChunkStore(int numberOfRows) { + super(numberOfRows); + } + + /** + * Below method will be used to put short array data + * + * @param data + */ + @Override + public void putData(short[] data) { + this.data = data; + } + + /** + * to get the short value + * + * @param index + * @return shot value based on index + */ + @Override + public short getShort(int index) { + return data[index]; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java new file mode 100644 index 0000000..7d5ccd2 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; + +import java.nio.ByteBuffer; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.util.ByteUtil; + +/** + * Below class is responsible to store variable length dimension data chunk in + * memory Memory occupied can be on heap or offheap using unsafe interface + */ +public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimensionDataChunkStore { + + /** + * total number of rows + */ + private int numberOfRows; + + /** + * offset of the data this will be used during search, as we can directly jump + * to particular location + */ + private int[] dataOffsets; + + public SafeVariableLengthDimensionDataChunkStore(boolean isInvertedIndex, int numberOfRows) { + super(isInvertedIndex); + this.numberOfRows = numberOfRows; + this.dataOffsets = new int[numberOfRows]; + } + + /** + * Below method will be used to put the rows and its metadata in offheap + * + * @param invertedIndex inverted index to be stored + * @param invertedIndexReverse inverted index reverse to be stored + * @param data data to be stored + */ + @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse, + byte[] data) { + // first put the data, inverted index and reverse inverted index to memory + super.putArray(invertedIndex, invertedIndexReverse, data); + // As data is of variable length and data format is + // <length in short><data><length in short><data> + // we need to store offset of each data so data can be accessed directly + // for example: + //data = {0,5,1,2,3,4,5,0,6,0,1,2,3,4,5,0,2,8,9} + //so value stored in offset will be position of actual data + // [2,9,17] + // to store this value we need to get the actual data length + 2 bytes used for storing the + // length + + // start position will be used to store the current data position + int startOffset = 0; + // as first position will be start from 2 byte as data is stored first in the memory block + // we need to skip first two bytes this is because first two bytes will be length of the data + // which we have to skip + dataOffsets[0] = CarbonCommonConstants.SHORT_SIZE_IN_BYTE; + // creating a byte buffer which will wrap the length of the row + ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.SHORT_SIZE_IN_BYTE); + for (int i = 1; i < numberOfRows; i++) { + buffer.put(data, startOffset, CarbonCommonConstants.SHORT_SIZE_IN_BYTE); + buffer.flip(); + // so current row position will be + // previous row length + 2 bytes used for storing previous row data + startOffset += buffer.getShort() + CarbonCommonConstants.SHORT_SIZE_IN_BYTE; + // as same byte buffer is used to avoid creating many byte buffer for each row + // we need to clear the byte buffer + buffer.clear(); + dataOffsets[i] = startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE; + } + } + + @Override public byte[] getRow(int rowId) { + // if column was explicitly sorted we need to get the rowid based inverted index reverse + if (isExplictSorted) { + rowId = invertedIndexReverse[rowId]; + } + // now to get the row from memory block we need to do following thing + // 1. first get the current offset + // 2. if it's not a last row- get the next row offset + // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset + // else subtract the current row offset with complete data + // length get the offset of set of data + int currentDataOffset = dataOffsets[rowId]; + short length = 0; + // calculating the length of data + if (rowId < numberOfRows - 1) { + length = (short) (dataOffsets[rowId + 1] - (currentDataOffset + + CarbonCommonConstants.SHORT_SIZE_IN_BYTE)); + } else { + // for last record + length = (short) (this.data.length - currentDataOffset); + } + byte[] currentRowData = new byte[length]; + System.arraycopy(data, currentDataOffset, currentRowData, 0, length); + return currentRowData; + } + + @Override public int compareTo(int index, byte[] compareValue) { + // now to get the row from memory block we need to do following thing + // 1. first get the current offset + // 2. if it's not a last row- get the next row offset + // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset + // else subtract the current row offset with complete data + // length + + // get the offset of set of data + int currentDataOffset = dataOffsets[index]; + short length = 0; + // calculating the length of data + if (index < numberOfRows - 1) { + length = (short) (dataOffsets[index + 1] - (currentDataOffset + + CarbonCommonConstants.SHORT_SIZE_IN_BYTE)); + } else { + // for last record + length = (short) (this.data.length - currentDataOffset); + } + return ByteUtil.UnsafeComparer.INSTANCE + .compareTo(data, currentDataOffset, length, compareValue, 0, compareValue.length); + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java new file mode 100644 index 0000000..044e5f4 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; + +import org.apache.carbondata.core.carbon.datastore.chunk.store.DimensionDataChunkStore; +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.memory.MemoryAllocatorFactory; +import org.apache.carbondata.core.memory.MemoryBlock; +import org.apache.carbondata.core.unsafe.CarbonUnsafe; + +/** + * Responsibility is to store dimension data in memory. storage can be on heap + * or offheap. + */ +public abstract class UnsafeAbstractDimensionDataChunkStore implements DimensionDataChunkStore { + + /** + * memory block for data page + */ + protected MemoryBlock dataPageMemoryBlock; + + /** + * to check whether dimension column was explicitly sorted or not + */ + protected boolean isExplicitSorted; + + /** + * is memory released + */ + protected boolean isMemoryReleased; + + /** + * length of the actual data + */ + protected int dataLength; + + /** + * offset of the inverted index reverse + */ + protected long invertedIndexReverseOffset; + + /** + * to validate whether data is already kept in memory or not + */ + protected boolean isMemoryOccupied; + + /** + * Constructor + * + * @param totalSize total size of the data to be kept + * @param isInvertedIdex is inverted index present + * @param numberOfRows total number of rows + */ + public UnsafeAbstractDimensionDataChunkStore(long totalSize, boolean isInvertedIdex, + int numberOfRows) { + // allocating the data page + this.dataPageMemoryBlock = + MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(totalSize); + this.isExplicitSorted = isInvertedIdex; + } + + /** + * Below method will be used to put the rows and its metadata in offheap + * + * @param invertedIndex inverted index to be stored + * @param invertedIndexReverse inverted index reverse to be stored + * @param data data to be stored + */ + @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse, + final byte[] data) { + assert (!isMemoryOccupied); + this.dataLength = data.length; + this.invertedIndexReverseOffset = dataLength; + if (isExplicitSorted) { + this.invertedIndexReverseOffset += + invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE; + } + // copy the data to memory + CarbonUnsafe.unsafe + .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset(), this.dataLength); + // if inverted index is present then copy the inverted index + // and reverse inverted index to memory + if (isExplicitSorted) { + CarbonUnsafe.unsafe.copyMemory(invertedIndex, CarbonUnsafe.INT_ARRAY_OFFSET, + dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + dataLength, + invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE); + CarbonUnsafe.unsafe.copyMemory(invertedIndexReverse, CarbonUnsafe.INT_ARRAY_OFFSET, + dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset, + invertedIndexReverse.length * CarbonCommonConstants.INT_SIZE_IN_BYTE); + } + } + + /** + * Below method will be used to free the memory occupied by the column chunk + */ + @Override public void freeMemory() { + if (isMemoryReleased) { + return; + } + // free data page memory + MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().free(dataPageMemoryBlock); + isMemoryReleased = true; + this.dataPageMemoryBlock = null; + this.isMemoryOccupied = false; + } + + /** + * Below method will be used to get the inverted index + * + * @param rowId row id + * @return inverted index based on row id passed + */ + @Override public int getInvertedIndex(int rowId) { + return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + dataLength + (rowId + * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + } + + /** + * Below method will be used to get the surrogate key of the based on the row + * id passed + * + * @param rowId row id + * @return surrogate key + */ + @Override public int getSurrogate(int rowId) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * @return size of each column value + */ + @Override public int getColumnValueSize() { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * @return whether column was explicitly sorted or not + */ + @Override public boolean isExplicitSorted() { + return isExplicitSorted; + } + + /** + * Below method will be used to fill the row values to data array + * + * @param rowId row id of the data to be filled + * @param data buffer in which data will be filled + * @param offset off the of the buffer + */ + @Override public void fillRow(int rowId, byte[] data, int offset) { + throw new UnsupportedOperationException("Operation not supported"); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java new file mode 100644 index 0000000..98cb2f6 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; + +import java.math.BigDecimal; + +import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore; +import org.apache.carbondata.core.memory.MemoryAllocatorFactory; +import org.apache.carbondata.core.memory.MemoryBlock; + +/** + * Responsibility is store the measure data in memory, memory can be on heap or + * offheap based on the user configuration using unsafe interface + */ +public abstract class UnsafeAbstractMeasureDataChunkStore<T> implements MeasureDataChunkStore<T> { + + /** + * memory block + */ + protected MemoryBlock dataPageMemoryBlock; + + /** + * number of rows + */ + protected int numberOfRows; + + /** + * to check memory is released or not + */ + protected boolean isMemoryReleased; + + /** + * to check memory is occupied or not + */ + protected boolean isMemoryOccupied; + + public UnsafeAbstractMeasureDataChunkStore(int numberOfRows) { + this.numberOfRows = numberOfRows; + } + + /** + * to get the byte value + * + * @param index + * @return byte value based on index + */ + @Override public byte getByte(int index) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * to get the short value + * + * @param index + * @return short value based on index + */ + @Override public short getShort(int index) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * to get the int value + * + * @param index + * @return int value based on index + */ + @Override public int getInt(int index) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * to get the long value + * + * @param index + * @return long value based on index + */ + @Override public long getLong(int index) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * to get the double value + * + * @param index + * @return double value based on index + */ + @Override public double getDouble(int index) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * To get the bigdecimal value + * + * @param index + * @return bigdecimal value based on index + */ + @Override public BigDecimal getBigDecimal(int index) { + throw new UnsupportedOperationException("Operation not supported"); + } + + /** + * To free the occupied memory + */ + @Override public void freeMemory() { + if (isMemoryReleased) { + return; + } + MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().free(dataPageMemoryBlock); + isMemoryReleased = true; + this.dataPageMemoryBlock = null; + this.isMemoryOccupied = false; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java new file mode 100644 index 0000000..5d0a802 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; + +import org.apache.carbondata.core.memory.MemoryAllocatorFactory; +import org.apache.carbondata.core.unsafe.CarbonUnsafe; + +/** + * Responsible for storing Byte array data to memory. memory can be on heap or + * offheap based on the user configuration using unsafe interface + */ +public class UnsafeByteMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<byte[]> { + + public UnsafeByteMeasureChunkStore(int numberOfRows) { + super(numberOfRows); + } + + /** + * Below method will be used to put byte array data to memory + * + * @param data + */ + @Override public void putData(byte[] data) { + assert (!this.isMemoryOccupied); + this.dataPageMemoryBlock = + MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(data.length); + // copy the data to memory + CarbonUnsafe.unsafe + .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size()); + this.isMemoryOccupied = true; + } + + /** + * to get the byte value + * + * @param index + * @return byte value based on index + */ + @Override public byte getByte(int index) { + return CarbonUnsafe.unsafe + .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index); + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java new file mode 100644 index 0000000..9c61ff2 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.memory.MemoryAllocatorFactory; +import org.apache.carbondata.core.unsafe.CarbonUnsafe; + +/** + * Responsible for storing double array data to memory. memory can be on heap or + * offheap based on the user configuration using unsafe interface + */ +public class UnsafeDoubleMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<double[]> { + + public UnsafeDoubleMeasureChunkStore(int numberOfRows) { + super(numberOfRows); + } + + /** + * Below method will be used to put double array data to memory + * + * @param data + */ + @Override public void putData(double[] data) { + assert (!this.isMemoryOccupied); + this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator() + .allocate(data.length * CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE); + // copy the data to memory + CarbonUnsafe.unsafe + .copyMemory(data, CarbonUnsafe.DOUBLE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size()); + this.isMemoryOccupied = true; + } + + /** + * to get the double value + * + * @param index + * @return double value based on index + */ + @Override public double getDouble(int index) { + return CarbonUnsafe.unsafe.getDouble(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE)); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java new file mode 100644 index 0000000..ff28559 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.unsafe.CarbonUnsafe; + +/** + * Below class is responsible to store fixed length dimension data chunk in + * memory Memory occupied can be on heap or offheap using unsafe interface + */ +public class UnsafeFixedLengthDimensionDataChunkStore + extends UnsafeAbstractDimensionDataChunkStore { + + /** + * Size of each value + */ + private int columnValueSize; + + /** + * Constructor + * + * @param columnValueSize value of each column + * @param isInvertedIdex is inverted index present + * @param numberOfRows total number of rows + */ + public UnsafeFixedLengthDimensionDataChunkStore(long totalDataSize, int columnValueSize, + boolean isInvertedIdex, int numberOfRows) { + super(totalDataSize, isInvertedIdex, numberOfRows); + this.columnValueSize = columnValueSize; + } + + /** + * Below method will be used to get the row based inverted index + * + * @param rowId Inverted index + */ + @Override public byte[] getRow(int rowId) { + // if column was explicitly sorted we need to get the rowid based inverted index reverse + if (isExplicitSorted) { + rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId + * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + } + // creating a row + byte[] data = new byte[columnValueSize]; + //copy the row from memory block based on offset + // offset position will be index * each column value length + CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), data, + CarbonUnsafe.BYTE_ARRAY_OFFSET, columnValueSize); + return data; + } + + /** + * Below method will be used to get the surrogate key of the based on the row + * id passed + * + * @param rowId row id + * @return surrogate key + */ + @Override public int getSurrogate(int index) { + // if column was explicitly sorted we need to get the rowid based inverted index reverse + if (isExplicitSorted) { + index = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (index + * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + } + // below part is to convert the byte array to surrogate value + int startOffsetOfData = index * columnValueSize; + int surrogate = 0; + for (int i = 0; i < columnValueSize; i++) { + surrogate <<= 8; + surrogate ^= CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + startOffsetOfData) & 0xFF; + startOffsetOfData++; + } + return surrogate; + } + + /** + * Below method will be used to fill the row values to buffer array + * + * @param rowId row id of the data to be filled + * @param data buffer in which data will be filled + * @param offset off the of the buffer + */ + @Override public void fillRow(int rowId, byte[] buffer, int offset) { + // if column was explicitly sorted we need to get the rowid based inverted index reverse + if (isExplicitSorted) { + rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId + * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + } + //copy the row from memory block based on offset + // offset position will be index * each column value length + CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), buffer, + CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, columnValueSize); + } + + /** + * @return size of each column value + */ + @Override public int getColumnValueSize() { + return columnValueSize; + } + + /** + * to compare the two byte array + * + * @param index index of first byte array + * @param compareValue value of to be compared + * @return compare result + */ + @Override public int compareTo(int index, byte[] compareValue) { + // based on index we need to calculate the actual position in memory block + index = index * columnValueSize; + int compareResult = 0; + for (int i = 0; i < compareValue.length; i++) { + compareResult = (CarbonUnsafe.unsafe + .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index) + & 0xff) - (compareValue[i] & 0xff); + if (compareResult != 0) { + break; + } + index++; + } + return compareResult; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java new file mode 100644 index 0000000..0111648 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.memory.MemoryAllocatorFactory; +import org.apache.carbondata.core.unsafe.CarbonUnsafe; + +/** + * Responsible for storing int array data to memory. memory can be on heap or + * offheap based on the user configuration using unsafe interface + */ +public class UnsafeIntMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<int[]> { + + public UnsafeIntMeasureChunkStore(int numberOfRows) { + super(numberOfRows); + } + + /** + * Below method will be used to put int array data to memory + * + * @param data + */ + @Override public void putData(int[] data) { + assert (!this.isMemoryOccupied); + this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator() + .allocate(data.length * CarbonCommonConstants.INT_SIZE_IN_BYTE); + // copy the data to memory + CarbonUnsafe.unsafe + .copyMemory(data, CarbonUnsafe.INT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size()); + this.isMemoryOccupied = true; + } + + /** + * to get the int value + * + * @param index + * @return int value based on index + */ + @Override public int getInt(int index) { + return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java new file mode 100644 index 0000000..d3bed25 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.memory.MemoryAllocatorFactory; +import org.apache.carbondata.core.unsafe.CarbonUnsafe; + +/** + * Responsible for storing long array data to memory. memory can be on heap or + * offheap based on the user configuration using unsafe interface + */ +public class UnsafeLongMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<long[]> { + + public UnsafeLongMeasureChunkStore(int numberOfRows) { + super(numberOfRows); + } + + /** + * Below method will be used to put long array data to memory + * + * @param data + */ + @Override public void putData(long[] data) { + assert (!this.isMemoryOccupied); + this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator() + .allocate(data.length * CarbonCommonConstants.LONG_SIZE_IN_BYTE); + // copy the data to memory + CarbonUnsafe.unsafe + .copyMemory(data, CarbonUnsafe.LONG_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size()); + this.isMemoryOccupied = true; + } + + /** + * to get the long value + * + * @param index + * @return long value based on index + */ + @Override public long getLong(int index) { + return CarbonUnsafe.unsafe.getLong(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.LONG_SIZE_IN_BYTE)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java new file mode 100644 index 0000000..ba865cd --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.memory.MemoryAllocatorFactory; +import org.apache.carbondata.core.unsafe.CarbonUnsafe; + +/** + * Responsible for storing short array data to memory. memory can be on heap or + * offheap based on the user configuration using unsafe interface + */ +public class UnsafeShortMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<short[]> { + + public UnsafeShortMeasureChunkStore(int numberOfRows) { + super(numberOfRows); + } + + /** + * Below method will be used to put short array data to memory + * + * @param data + */ + @Override public void putData(short[] data) { + assert (!this.isMemoryOccupied); + this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator() + .allocate(data.length * CarbonCommonConstants.SHORT_SIZE_IN_BYTE); + // copy the data to memory + CarbonUnsafe.unsafe + .copyMemory(data, CarbonUnsafe.SHORT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size()); + this.isMemoryOccupied = true; + } + + /** + * to get the short value + * + * @param index + * @return shot value based on index + */ + @Override public short getShort(int index) { + return CarbonUnsafe.unsafe.getShort(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.SHORT_SIZE_IN_BYTE)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java new file mode 100644 index 0000000..c6155e7 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; + +import java.nio.ByteBuffer; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.unsafe.CarbonUnsafe; + +/** + * Below class is responsible to store variable length dimension data chunk in + * memory Memory occupied can be on heap or offheap using unsafe interface + */ +public class UnsafeVariableLengthDimesionDataChunkStore + extends UnsafeAbstractDimensionDataChunkStore { + + /** + * total number of rows + */ + private int numberOfRows; + + /** + * pointers offsets + */ + private long dataPointersOffsets; + + public UnsafeVariableLengthDimesionDataChunkStore(long totalSize, boolean isInvertedIdex, + int numberOfRows) { + super(totalSize, isInvertedIdex, numberOfRows); + this.numberOfRows = numberOfRows; + } + + /** + * Below method will be used to put the rows and its metadata in offheap + * + * @param invertedIndex inverted index to be stored + * @param invertedIndexReverse inverted index reverse to be stored + * @param data data to be stored + */ + @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse, + byte[] data) { + // first put the data, inverted index and reverse inverted index to memory + super.putArray(invertedIndex, invertedIndexReverse, data); + // position from where offsets will start + this.dataPointersOffsets = this.invertedIndexReverseOffset; + if (isExplicitSorted) { + this.dataPointersOffsets += numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE; + } + // As data is of variable length and data format is + // <length in short><data><length in short><data> + // we need to store offset of each data so data can be accessed directly + // for example: + //data = {0,5,1,2,3,4,5,0,6,0,1,2,3,4,5,0,2,8,9} + //so value stored in offset will be position of actual data + // [2,9,17] + // to store this value we need to get the actual data length + 2 bytes used for storing the + // length + + // start position will be used to store the current data position + int startOffset = 0; + // position from where offsets will start + long pointerOffsets = this.dataPointersOffsets; + // as first position will be start from 2 byte as data is stored first in the memory block + // we need to skip first two bytes this is because first two bytes will be length of the data + // which we have to skip + CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + pointerOffsets, + CarbonCommonConstants.SHORT_SIZE_IN_BYTE); + // incrementing the pointers as first value is already filled and as we are storing as int + // we need to increment the 4 bytes to set the position of the next value to set + pointerOffsets += CarbonCommonConstants.INT_SIZE_IN_BYTE; + // creating a byte buffer which will wrap the length of the row + // using byte buffer as unsafe will return bytes in little-endian encoding + ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.SHORT_SIZE_IN_BYTE); + // store length of data + byte[] length = new byte[CarbonCommonConstants.SHORT_SIZE_IN_BYTE]; + // as first offset is already stored, we need to start from the 2nd row in data array + for (int i = 1; i < numberOfRows; i++) { + // first copy the length of previous row + CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + startOffset, length, CarbonUnsafe.BYTE_ARRAY_OFFSET, + CarbonCommonConstants.SHORT_SIZE_IN_BYTE); + buffer.put(length); + buffer.flip(); + // so current row position will be + // previous row length + 2 bytes used for storing previous row data + startOffset += CarbonCommonConstants.SHORT_SIZE_IN_BYTE + buffer.getShort(); + // as same byte buffer is used to avoid creating many byte buffer for each row + // we need to clear the byte buffer + buffer.clear(); + // now put the offset of current row, here we need to add 2 more bytes as current will + // also have length part so we have to skip length + CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + pointerOffsets, + startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE); + // incrementing the pointers as first value is already filled and as we are storing as int + // we need to increment the 4 bytes to set the position of the next value to set + pointerOffsets += CarbonCommonConstants.INT_SIZE_IN_BYTE; + } + + } + + /** + * Below method will be used to get the row based on row id passed + * + * @param index + * @return row + */ + @Override public byte[] getRow(int rowId) { + byte[] data = null; + try { + // if column was explicitly sorted we need to get the rowid based inverted index reverse + if (isExplicitSorted) { + rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId + * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + } + // now to get the row from memory block we need to do following thing + // 1. first get the current offset + // 2. if it's not a last row- get the next row offset + // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset + // else subtract the current row offset + 2 bytes(to skip the data length) + // with complete data length + int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (rowId + * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + short length = 0; + // calculating the length of data + if (rowId < numberOfRows - 1) { + int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((rowId + 1) + * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + length = (short) (OffsetOfNextdata - (currentDataOffset + + CarbonCommonConstants.SHORT_SIZE_IN_BYTE)); + } else { + // for last record we need to subtract with data length + length = (short) (this.dataLength - currentDataOffset); + } + data = new byte[length]; + CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + currentDataOffset, data, + CarbonUnsafe.BYTE_ARRAY_OFFSET, length); + } catch (Throwable t) { + System.out.println(); + } + return data; + } + + /** + * to compare the two byte array + * + * @param index index of first byte array + * @param compareValue value of to be compared + * @return compare result + */ + @Override public int compareTo(int index, byte[] compareValue) { + // now to get the row from memory block we need to do following thing + // 1. first get the current offset + // 2. if it's not a last row- get the next row offset + // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset + // else subtract the current row offset + // with complete data length get the offset of set of data + int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (index + * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + short length = 0; + // calculating the length of data + if (index < numberOfRows - 1) { + int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((index + 1) + * CarbonCommonConstants.INT_SIZE_IN_BYTE)); + length = (short) (OffsetOfNextdata - (currentDataOffset + + CarbonCommonConstants.SHORT_SIZE_IN_BYTE)); + } else { + // for last record we need to subtract with data length + length = (short) (this.dataLength - currentDataOffset); + } + // as this class handles this variable length data, so filter value can be + // smaller or bigger than than actual data, so we need to take the smaller length + int compareResult = 0; + int compareLength = length < compareValue.length ? length : compareValue.length; + for (int i = 0; i < compareLength; i++) { + compareResult = (CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(), + dataPageMemoryBlock.getBaseOffset() + currentDataOffset) & 0xff) - (compareValue[i] + & 0xff); + // if compare result is not equal we can break + if (compareResult != 0) { + break; + } + // increment the offset by one as comparison is done byte by byte + currentDataOffset++; + } + return compareResult; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java index f91a75c..041f5ed 100644 --- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java +++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java @@ -1081,6 +1081,7 @@ public final class CarbonCommonConstants { /** * Default carbon dictionary server port + */ public static final String DICTIONARY_SERVER_PORT_DEFAULT = "2030"; @@ -1094,6 +1095,26 @@ public final class CarbonCommonConstants { */ public static final int DEFAULT_COLUMN_LENGTH = 100000; + /** + * property for enabling unsafe based query processing + */ + public static final String ENABLE_UNSAFE_IN_QUERY_EXECUTION = "enable.unsafe.in.query.processing"; + + /** + * default property of unsafe processing + */ + public static final String ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE = "false"; + + /** + * property for offheap based processing + */ + public static final String USE_OFFHEAP_IN_QUERY_PROCSSING = "use.offheap.in.query.processing"; + + /** + * default value of offheap based processing + */ + public static final String USE_OFFHEAP_IN_QUERY_PROCSSING_DEFAULT = "true"; + private CarbonCommonConstants() { } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java index a174a18..e40dadd 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java @@ -31,5 +31,4 @@ public interface NodeMeasureDataStore { * @return writable array (compressed or normal) */ byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray); - } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java index a3d922d..1b90bc6 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java @@ -25,24 +25,36 @@ public interface Compressor { byte[] unCompressByte(byte[] compInput); + byte[] unCompressByte(byte[] compInput, int offset, int length); + byte[] compressShort(short[] unCompInput); short[] unCompressShort(byte[] compInput); + short[] unCompressShort(byte[] compInput, int offset, int lenght); + byte[] compressInt(int[] unCompInput); int[] unCompressInt(byte[] compInput); + int[] unCompressInt(byte[] compInput, int offset, int length); + byte[] compressLong(long[] unCompInput); long[] unCompressLong(byte[] compInput); + long[] unCompressLong(byte[] compInput, int offset, int length); + byte[] compressFloat(float[] unCompInput); float[] unCompressFloat(byte[] compInput); + float[] unCompressFloat(byte[] compInput, int offset, int length); + byte[] compressDouble(double[] unCompInput); double[] unCompressDouble(byte[] compInput); + double[] unCompressDouble(byte[] compInput, int offset, int length); + } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java index 110010f..5160879 100644 --- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java @@ -24,19 +24,29 @@ import org.apache.carbondata.core.util.CarbonProperties; public class CompressorFactory { - private static Compressor snappy; + private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory(); - public static Compressor getInstance() { - String compressor = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.COMPRESSOR, - CarbonCommonConstants.DEFAULT_COMPRESSOR); - switch (compressor) { + private final Compressor compressor; + + private CompressorFactory() { + String compressorType = CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR); + switch (compressorType) { case "snappy": - if (snappy == null) { - snappy = new SnappyCompressor(); - } - return snappy; + compressor = new SnappyCompressor(); + break; default: - throw new RuntimeException("Unsupported compressor: " + compressor); + throw new RuntimeException( + "Invalid compressor type provided! Please provide valid compressor type"); } } + + public static CompressorFactory getInstance() { + return COMPRESSOR_FACTORY; + } + + public Compressor getCompressor() { + return compressor; + } + }
