http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java deleted file mode 100644 index fed4ab4..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store; - -import java.math.BigDecimal; - -/** - * Responsibility is store the measure data in memory, - * memory can be on heap or offheap based on the user configuration - */ -public interface MeasureDataChunkStore<T> { - - /** - * Below method will be used to put the data to memory - * - * @param data - */ - void putData(T data); - - /** - * to get byte value - * - * @param index - * @return byte value based on index - */ - byte getByte(int index); - - /** - * to get the short value - * - * @param index - * @return short value based on index - */ - short getShort(int index); - - /** - * to get the int value - * - * @param index - * @return int value based on index - */ - int getInt(int index); - - /** - * to get the long value - * - * @param index - * @return long value based on index - */ - long getLong(int index); - - /** - * to get the double value - * - * @param index - * @return double value based on index - */ - double getDouble(int index); - - /** - * To get the bigdecimal value - * - * @param index - * @return bigdecimal value based on index - */ - BigDecimal getBigDecimal(int index); - - /** - * To free the occupied memory - */ - void freeMemory(); -}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java deleted file mode 100644 index 0ed2218..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; - -import org.apache.carbondata.core.carbon.datastore.chunk.store.DimensionDataChunkStore; - -/** - * Responsibility is to store dimension data - */ -public abstract class SafeAbsractDimensionDataChunkStore implements DimensionDataChunkStore { - - /** - * data chunk for dimension column - */ - protected byte[] data; - - /** - * inverted index - */ - protected int[] invertedIndex; - - /** - * inverted index reverser - */ - protected int[] invertedIndexReverse; - - /** - * to check whether dimension column was explicitly sorted or not - */ - protected boolean isExplictSorted; - - /** - * Constructor - * - * @param totalSize total size of the data to be kept - * @param isInvertedIdex is inverted index present - * @param numberOfRows total number of rows - */ - public SafeAbsractDimensionDataChunkStore(boolean isInvertedIdex) { - this.isExplictSorted = isInvertedIdex; - } - - /** - * Below method will be used to put the rows and its metadata in offheap - * - * @param invertedIndex inverted index to be stored - * @param invertedIndexReverse inverted index reverse to be stored - * @param data data to be stored - */ - @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse, - final byte[] data) { - this.data = data; - this.invertedIndex = invertedIndex; - this.invertedIndexReverse = invertedIndexReverse; - } - - /** - * Below method will be used to free the memory occupied by the column chunk - */ - @Override public void freeMemory() { - // do nothing as GC will take care of freeing memory - } - - /** - * Below method will be used to get the inverted index - * - * @param rowId row id - * @return inverted index based on row id passed - */ - @Override public int getInvertedIndex(int rowId) { - return invertedIndex[rowId]; - } - - /** - * Below method will be used to get the surrogate key of the based on the row - * id passed - * - * @param rowId row id - * @return surrogate key - */ - @Override public int getSurrogate(int rowId) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * @return size of each column value - */ - @Override public int getColumnValueSize() { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * @return whether column was explicitly sorted or not - */ - @Override public boolean isExplicitSorted() { - return isExplictSorted; - } - - /** - * Below method will be used to fill the row values to data array - * - * @param rowId row id of the data to be filled - * @param data buffer in which data will be filled - * @param offset off the of the buffer - */ - @Override public void fillRow(int rowId, byte[] data, int offset) { - throw new UnsupportedOperationException("Operation not supported"); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java deleted file mode 100644 index 0daeaa4..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; - -import java.math.BigDecimal; - -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore; - -/** - * Responsibility is store the measure data in memory, - */ -public abstract class SafeAbstractMeasureDataChunkStore<T> implements - MeasureDataChunkStore<T> { - - /** - * number of rows - */ - protected int numberOfRows; - - public SafeAbstractMeasureDataChunkStore(int numberOfRows) { - this.numberOfRows = numberOfRows; - } - - /** - * to get the byte value - * - * @param index - * @return byte value based on index - */ - @Override - public byte getByte(int index) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * to get the short value - * - * @param index - * @return short value based on index - */ - @Override - public short getShort(int index) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * to get the int value - * - * @param index - * @return int value based on index - */ - @Override - public int getInt(int index) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * to get the long value - * - * @param index - * @return long value based on index - */ - @Override - public long getLong(int index) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * to get the double value - * - * @param index - * @return double value based on index - */ - @Override - public double getDouble(int index) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * To get the bigdecimal value - * - * @param index - * @return bigdecimal value based on index - */ - @Override - public BigDecimal getBigDecimal(int index) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * To free the occupied memory - */ - @Override - public void freeMemory() { - // do nothing as GC will take care of freeing the memory - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java deleted file mode 100644 index 1f69cd2..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; - -/** - * Responsible for storing Byte array data to memory. - */ -public class SafeByteMeasureChunkStore extends - SafeAbstractMeasureDataChunkStore<byte[]> { - - /** - * data - */ - private byte[] data; - - public SafeByteMeasureChunkStore(int numberOfRows) { - super(numberOfRows); - } - - /** - * Below method will be used to put byte array data to memory - * - * @param data - */ - @Override - public void putData(byte[] data) { - this.data = data; - } - - /** - * to get the byte value - * - * @param index - * @return byte value based on index - */ - @Override - public byte getByte(int index) { - return this.data[index]; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java deleted file mode 100644 index dc485c6..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; - -/** - * Below class will be used to store the measure values of double data type - */ -public class SafeDoubleMeasureChunkStore extends - SafeAbstractMeasureDataChunkStore<double[]> { - - /** - * data - */ - private double[] data; - - public SafeDoubleMeasureChunkStore(int numberOfRows) { - super(numberOfRows); - } - - /** - * Below method will be used to store double array data - * - * @param data - */ - @Override - public void putData(double[] data) { - this.data = data; - } - - /** - * to get the double value - * - * @param index - * @return double value based on index - */ - @Override - public double getDouble(int index) { - return this.data[index]; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java deleted file mode 100644 index 3f5afc5..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; - -import org.apache.carbondata.core.util.ByteUtil; - -/** - * Below class will be used to store fixed length dimension data - */ -public class SafeFixedLengthDimensionDataChunkStore extends SafeAbsractDimensionDataChunkStore { - - /** - * Size of each value - */ - private int columnValueSize; - - public SafeFixedLengthDimensionDataChunkStore(boolean isInvertedIndex, int columnValueSize) { - super(isInvertedIndex); - this.columnValueSize = columnValueSize; - } - - /** - * Below method will be used to get the row based inverted index - * - * @param rowId Inverted index - */ - @Override public byte[] getRow(int rowId) { - // if column was explicitly sorted we need to get the rowid based inverted index reverse - if (isExplictSorted) { - rowId = invertedIndexReverse[rowId]; - } - // creating a row - byte[] row = new byte[columnValueSize]; - //copy the row from data chunk based on offset - // offset position will be index * each column value length - System.arraycopy(this.data, rowId * columnValueSize, row, 0, columnValueSize); - return row; - } - - /** - * Below method will be used to get the surrogate key of the based on the row - * id passed - * - * @param rowId row id - * @return surrogate key - */ - @Override public int getSurrogate(int index) { - // if column was explicitly sorted we need to get the rowid based inverted index reverse - if (isExplictSorted) { - index = invertedIndexReverse[index]; - } - // below part is to convert the byte array to surrogate value - int startOffsetOfData = index * columnValueSize; - int surrogate = 0; - for (int i = 0; i < columnValueSize; i++) { - surrogate <<= 8; - surrogate ^= data[startOffsetOfData] & 0xFF; - startOffsetOfData++; - } - return surrogate; - } - - /** - * Below method will be used to fill the row values to buffer array - * - * @param rowId row id of the data to be filled - * @param data buffer in which data will be filled - * @param offset off the of the buffer - */ - @Override public void fillRow(int rowId, byte[] buffer, int offset) { - // if column was explicitly sorted we need to get the rowid based inverted index reverse - if (isExplictSorted) { - rowId = invertedIndexReverse[rowId]; - } - //copy the row from memory block based on offset - // offset position will be index * each column value length - System.arraycopy(data, rowId * columnValueSize, buffer, offset, columnValueSize); - } - - /** - * @return size of each column value - */ - @Override public int getColumnValueSize() { - return columnValueSize; - } - - /** - * to compare the two byte array - * - * @param index index of first byte array - * @param compareValue value of to be compared - * @return compare result - */ - @Override public int compareTo(int index, byte[] compareValue) { - return ByteUtil.UnsafeComparer.INSTANCE - .compareTo(data, index * columnValueSize, columnValueSize, compareValue, 0, - columnValueSize); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java deleted file mode 100644 index 29b77cb..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; - -/** - * Responsible for storing int array data to memory. - */ -public class SafeIntMeasureChunkStore extends - SafeAbstractMeasureDataChunkStore<int[]> { - - /** - * data - */ - private int[] data; - - public SafeIntMeasureChunkStore(int numberOfRows) { - super(numberOfRows); - } - - /** - * Below method will be used to put int array data to memory - * - * @param data - */ - @Override - public void putData(int[] data) { - this.data = data; - } - - /** - * to get the int value - * - * @param index - * @return int value based on index - */ - @Override - public int getInt(int index) { - return this.data[index]; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java deleted file mode 100644 index 3fd0370..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; - -/** - * Below class will be used to store the measure values of long data type - * - */ -public class SafeLongMeasureChunkStore extends - SafeAbstractMeasureDataChunkStore<long[]> { - - /** - * data - */ - private long[] data; - - public SafeLongMeasureChunkStore(int numberOfRows) { - super(numberOfRows); - } - - /** - * Below method will be used to store long array data - * - * @param data - */ - @Override - public void putData(long[] data) { - this.data = data; - } - - /** - * to get the long value - * - * @param index - * @return long value based on index - */ - @Override - public long getLong(int index) { - return this.data[index]; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java deleted file mode 100644 index 269ad79..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; - -/** - * Below class will be used to store the measure values of short data type - * - */ -public class SafeShortMeasureChunkStore extends - SafeAbstractMeasureDataChunkStore<short[]> { - - /** - * data - */ - private short[] data; - - public SafeShortMeasureChunkStore(int numberOfRows) { - super(numberOfRows); - } - - /** - * Below method will be used to put short array data - * - * @param data - */ - @Override - public void putData(short[] data) { - this.data = data; - } - - /** - * to get the short value - * - * @param index - * @return shot value based on index - */ - @Override - public short getShort(int index) { - return data[index]; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java deleted file mode 100644 index 7d5ccd2..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe; - -import java.nio.ByteBuffer; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.util.ByteUtil; - -/** - * Below class is responsible to store variable length dimension data chunk in - * memory Memory occupied can be on heap or offheap using unsafe interface - */ -public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimensionDataChunkStore { - - /** - * total number of rows - */ - private int numberOfRows; - - /** - * offset of the data this will be used during search, as we can directly jump - * to particular location - */ - private int[] dataOffsets; - - public SafeVariableLengthDimensionDataChunkStore(boolean isInvertedIndex, int numberOfRows) { - super(isInvertedIndex); - this.numberOfRows = numberOfRows; - this.dataOffsets = new int[numberOfRows]; - } - - /** - * Below method will be used to put the rows and its metadata in offheap - * - * @param invertedIndex inverted index to be stored - * @param invertedIndexReverse inverted index reverse to be stored - * @param data data to be stored - */ - @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse, - byte[] data) { - // first put the data, inverted index and reverse inverted index to memory - super.putArray(invertedIndex, invertedIndexReverse, data); - // As data is of variable length and data format is - // <length in short><data><length in short><data> - // we need to store offset of each data so data can be accessed directly - // for example: - //data = {0,5,1,2,3,4,5,0,6,0,1,2,3,4,5,0,2,8,9} - //so value stored in offset will be position of actual data - // [2,9,17] - // to store this value we need to get the actual data length + 2 bytes used for storing the - // length - - // start position will be used to store the current data position - int startOffset = 0; - // as first position will be start from 2 byte as data is stored first in the memory block - // we need to skip first two bytes this is because first two bytes will be length of the data - // which we have to skip - dataOffsets[0] = CarbonCommonConstants.SHORT_SIZE_IN_BYTE; - // creating a byte buffer which will wrap the length of the row - ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.SHORT_SIZE_IN_BYTE); - for (int i = 1; i < numberOfRows; i++) { - buffer.put(data, startOffset, CarbonCommonConstants.SHORT_SIZE_IN_BYTE); - buffer.flip(); - // so current row position will be - // previous row length + 2 bytes used for storing previous row data - startOffset += buffer.getShort() + CarbonCommonConstants.SHORT_SIZE_IN_BYTE; - // as same byte buffer is used to avoid creating many byte buffer for each row - // we need to clear the byte buffer - buffer.clear(); - dataOffsets[i] = startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE; - } - } - - @Override public byte[] getRow(int rowId) { - // if column was explicitly sorted we need to get the rowid based inverted index reverse - if (isExplictSorted) { - rowId = invertedIndexReverse[rowId]; - } - // now to get the row from memory block we need to do following thing - // 1. first get the current offset - // 2. if it's not a last row- get the next row offset - // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset - // else subtract the current row offset with complete data - // length get the offset of set of data - int currentDataOffset = dataOffsets[rowId]; - short length = 0; - // calculating the length of data - if (rowId < numberOfRows - 1) { - length = (short) (dataOffsets[rowId + 1] - (currentDataOffset - + CarbonCommonConstants.SHORT_SIZE_IN_BYTE)); - } else { - // for last record - length = (short) (this.data.length - currentDataOffset); - } - byte[] currentRowData = new byte[length]; - System.arraycopy(data, currentDataOffset, currentRowData, 0, length); - return currentRowData; - } - - @Override public int compareTo(int index, byte[] compareValue) { - // now to get the row from memory block we need to do following thing - // 1. first get the current offset - // 2. if it's not a last row- get the next row offset - // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset - // else subtract the current row offset with complete data - // length - - // get the offset of set of data - int currentDataOffset = dataOffsets[index]; - short length = 0; - // calculating the length of data - if (index < numberOfRows - 1) { - length = (short) (dataOffsets[index + 1] - (currentDataOffset - + CarbonCommonConstants.SHORT_SIZE_IN_BYTE)); - } else { - // for last record - length = (short) (this.data.length - currentDataOffset); - } - return ByteUtil.UnsafeComparer.INSTANCE - .compareTo(data, currentDataOffset, length, compareValue, 0, compareValue.length); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java deleted file mode 100644 index 044e5f4..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; - -import org.apache.carbondata.core.carbon.datastore.chunk.store.DimensionDataChunkStore; -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.memory.MemoryAllocatorFactory; -import org.apache.carbondata.core.memory.MemoryBlock; -import org.apache.carbondata.core.unsafe.CarbonUnsafe; - -/** - * Responsibility is to store dimension data in memory. storage can be on heap - * or offheap. - */ -public abstract class UnsafeAbstractDimensionDataChunkStore implements DimensionDataChunkStore { - - /** - * memory block for data page - */ - protected MemoryBlock dataPageMemoryBlock; - - /** - * to check whether dimension column was explicitly sorted or not - */ - protected boolean isExplicitSorted; - - /** - * is memory released - */ - protected boolean isMemoryReleased; - - /** - * length of the actual data - */ - protected int dataLength; - - /** - * offset of the inverted index reverse - */ - protected long invertedIndexReverseOffset; - - /** - * to validate whether data is already kept in memory or not - */ - protected boolean isMemoryOccupied; - - /** - * Constructor - * - * @param totalSize total size of the data to be kept - * @param isInvertedIdex is inverted index present - * @param numberOfRows total number of rows - */ - public UnsafeAbstractDimensionDataChunkStore(long totalSize, boolean isInvertedIdex, - int numberOfRows) { - // allocating the data page - this.dataPageMemoryBlock = - MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(totalSize); - this.isExplicitSorted = isInvertedIdex; - } - - /** - * Below method will be used to put the rows and its metadata in offheap - * - * @param invertedIndex inverted index to be stored - * @param invertedIndexReverse inverted index reverse to be stored - * @param data data to be stored - */ - @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse, - final byte[] data) { - assert (!isMemoryOccupied); - this.dataLength = data.length; - this.invertedIndexReverseOffset = dataLength; - if (isExplicitSorted) { - this.invertedIndexReverseOffset += - invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE; - } - // copy the data to memory - CarbonUnsafe.unsafe - .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset(), this.dataLength); - // if inverted index is present then copy the inverted index - // and reverse inverted index to memory - if (isExplicitSorted) { - CarbonUnsafe.unsafe.copyMemory(invertedIndex, CarbonUnsafe.INT_ARRAY_OFFSET, - dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + dataLength, - invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE); - CarbonUnsafe.unsafe.copyMemory(invertedIndexReverse, CarbonUnsafe.INT_ARRAY_OFFSET, - dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset, - invertedIndexReverse.length * CarbonCommonConstants.INT_SIZE_IN_BYTE); - } - } - - /** - * Below method will be used to free the memory occupied by the column chunk - */ - @Override public void freeMemory() { - if (isMemoryReleased) { - return; - } - // free data page memory - MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().free(dataPageMemoryBlock); - isMemoryReleased = true; - this.dataPageMemoryBlock = null; - this.isMemoryOccupied = false; - } - - /** - * Below method will be used to get the inverted index - * - * @param rowId row id - * @return inverted index based on row id passed - */ - @Override public int getInvertedIndex(int rowId) { - return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + dataLength + (rowId - * CarbonCommonConstants.INT_SIZE_IN_BYTE)); - } - - /** - * Below method will be used to get the surrogate key of the based on the row - * id passed - * - * @param rowId row id - * @return surrogate key - */ - @Override public int getSurrogate(int rowId) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * @return size of each column value - */ - @Override public int getColumnValueSize() { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * @return whether column was explicitly sorted or not - */ - @Override public boolean isExplicitSorted() { - return isExplicitSorted; - } - - /** - * Below method will be used to fill the row values to data array - * - * @param rowId row id of the data to be filled - * @param data buffer in which data will be filled - * @param offset off the of the buffer - */ - @Override public void fillRow(int rowId, byte[] data, int offset) { - throw new UnsupportedOperationException("Operation not supported"); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java deleted file mode 100644 index 98cb2f6..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; - -import java.math.BigDecimal; - -import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore; -import org.apache.carbondata.core.memory.MemoryAllocatorFactory; -import org.apache.carbondata.core.memory.MemoryBlock; - -/** - * Responsibility is store the measure data in memory, memory can be on heap or - * offheap based on the user configuration using unsafe interface - */ -public abstract class UnsafeAbstractMeasureDataChunkStore<T> implements MeasureDataChunkStore<T> { - - /** - * memory block - */ - protected MemoryBlock dataPageMemoryBlock; - - /** - * number of rows - */ - protected int numberOfRows; - - /** - * to check memory is released or not - */ - protected boolean isMemoryReleased; - - /** - * to check memory is occupied or not - */ - protected boolean isMemoryOccupied; - - public UnsafeAbstractMeasureDataChunkStore(int numberOfRows) { - this.numberOfRows = numberOfRows; - } - - /** - * to get the byte value - * - * @param index - * @return byte value based on index - */ - @Override public byte getByte(int index) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * to get the short value - * - * @param index - * @return short value based on index - */ - @Override public short getShort(int index) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * to get the int value - * - * @param index - * @return int value based on index - */ - @Override public int getInt(int index) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * to get the long value - * - * @param index - * @return long value based on index - */ - @Override public long getLong(int index) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * to get the double value - * - * @param index - * @return double value based on index - */ - @Override public double getDouble(int index) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * To get the bigdecimal value - * - * @param index - * @return bigdecimal value based on index - */ - @Override public BigDecimal getBigDecimal(int index) { - throw new UnsupportedOperationException("Operation not supported"); - } - - /** - * To free the occupied memory - */ - @Override public void freeMemory() { - if (isMemoryReleased) { - return; - } - MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().free(dataPageMemoryBlock); - isMemoryReleased = true; - this.dataPageMemoryBlock = null; - this.isMemoryOccupied = false; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java deleted file mode 100644 index 5d0a802..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; - -import org.apache.carbondata.core.memory.MemoryAllocatorFactory; -import org.apache.carbondata.core.unsafe.CarbonUnsafe; - -/** - * Responsible for storing Byte array data to memory. memory can be on heap or - * offheap based on the user configuration using unsafe interface - */ -public class UnsafeByteMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<byte[]> { - - public UnsafeByteMeasureChunkStore(int numberOfRows) { - super(numberOfRows); - } - - /** - * Below method will be used to put byte array data to memory - * - * @param data - */ - @Override public void putData(byte[] data) { - assert (!this.isMemoryOccupied); - this.dataPageMemoryBlock = - MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(data.length); - // copy the data to memory - CarbonUnsafe.unsafe - .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size()); - this.isMemoryOccupied = true; - } - - /** - * to get the byte value - * - * @param index - * @return byte value based on index - */ - @Override public byte getByte(int index) { - return CarbonUnsafe.unsafe - .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java deleted file mode 100644 index 9c61ff2..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.memory.MemoryAllocatorFactory; -import org.apache.carbondata.core.unsafe.CarbonUnsafe; - -/** - * Responsible for storing double array data to memory. memory can be on heap or - * offheap based on the user configuration using unsafe interface - */ -public class UnsafeDoubleMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<double[]> { - - public UnsafeDoubleMeasureChunkStore(int numberOfRows) { - super(numberOfRows); - } - - /** - * Below method will be used to put double array data to memory - * - * @param data - */ - @Override public void putData(double[] data) { - assert (!this.isMemoryOccupied); - this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator() - .allocate(data.length * CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE); - // copy the data to memory - CarbonUnsafe.unsafe - .copyMemory(data, CarbonUnsafe.DOUBLE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size()); - this.isMemoryOccupied = true; - } - - /** - * to get the double value - * - * @param index - * @return double value based on index - */ - @Override public double getDouble(int index) { - return CarbonUnsafe.unsafe.getDouble(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE)); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java deleted file mode 100644 index ff28559..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.unsafe.CarbonUnsafe; - -/** - * Below class is responsible to store fixed length dimension data chunk in - * memory Memory occupied can be on heap or offheap using unsafe interface - */ -public class UnsafeFixedLengthDimensionDataChunkStore - extends UnsafeAbstractDimensionDataChunkStore { - - /** - * Size of each value - */ - private int columnValueSize; - - /** - * Constructor - * - * @param columnValueSize value of each column - * @param isInvertedIdex is inverted index present - * @param numberOfRows total number of rows - */ - public UnsafeFixedLengthDimensionDataChunkStore(long totalDataSize, int columnValueSize, - boolean isInvertedIdex, int numberOfRows) { - super(totalDataSize, isInvertedIdex, numberOfRows); - this.columnValueSize = columnValueSize; - } - - /** - * Below method will be used to get the row based inverted index - * - * @param rowId Inverted index - */ - @Override public byte[] getRow(int rowId) { - // if column was explicitly sorted we need to get the rowid based inverted index reverse - if (isExplicitSorted) { - rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId - * CarbonCommonConstants.INT_SIZE_IN_BYTE)); - } - // creating a row - byte[] data = new byte[columnValueSize]; - //copy the row from memory block based on offset - // offset position will be index * each column value length - CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), data, - CarbonUnsafe.BYTE_ARRAY_OFFSET, columnValueSize); - return data; - } - - /** - * Below method will be used to get the surrogate key of the based on the row - * id passed - * - * @param rowId row id - * @return surrogate key - */ - @Override public int getSurrogate(int index) { - // if column was explicitly sorted we need to get the rowid based inverted index reverse - if (isExplicitSorted) { - index = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (index - * CarbonCommonConstants.INT_SIZE_IN_BYTE)); - } - // below part is to convert the byte array to surrogate value - int startOffsetOfData = index * columnValueSize; - int surrogate = 0; - for (int i = 0; i < columnValueSize; i++) { - surrogate <<= 8; - surrogate ^= CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + startOffsetOfData) & 0xFF; - startOffsetOfData++; - } - return surrogate; - } - - /** - * Below method will be used to fill the row values to buffer array - * - * @param rowId row id of the data to be filled - * @param data buffer in which data will be filled - * @param offset off the of the buffer - */ - @Override public void fillRow(int rowId, byte[] buffer, int offset) { - // if column was explicitly sorted we need to get the rowid based inverted index reverse - if (isExplicitSorted) { - rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId - * CarbonCommonConstants.INT_SIZE_IN_BYTE)); - } - //copy the row from memory block based on offset - // offset position will be index * each column value length - CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), buffer, - CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, columnValueSize); - } - - /** - * @return size of each column value - */ - @Override public int getColumnValueSize() { - return columnValueSize; - } - - /** - * to compare the two byte array - * - * @param index index of first byte array - * @param compareValue value of to be compared - * @return compare result - */ - @Override public int compareTo(int index, byte[] compareValue) { - // based on index we need to calculate the actual position in memory block - index = index * columnValueSize; - int compareResult = 0; - for (int i = 0; i < compareValue.length; i++) { - compareResult = (CarbonUnsafe.unsafe - .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index) - & 0xff) - (compareValue[i] & 0xff); - if (compareResult != 0) { - break; - } - index++; - } - return compareResult; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java deleted file mode 100644 index 0111648..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.memory.MemoryAllocatorFactory; -import org.apache.carbondata.core.unsafe.CarbonUnsafe; - -/** - * Responsible for storing int array data to memory. memory can be on heap or - * offheap based on the user configuration using unsafe interface - */ -public class UnsafeIntMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<int[]> { - - public UnsafeIntMeasureChunkStore(int numberOfRows) { - super(numberOfRows); - } - - /** - * Below method will be used to put int array data to memory - * - * @param data - */ - @Override public void putData(int[] data) { - assert (!this.isMemoryOccupied); - this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator() - .allocate(data.length * CarbonCommonConstants.INT_SIZE_IN_BYTE); - // copy the data to memory - CarbonUnsafe.unsafe - .copyMemory(data, CarbonUnsafe.INT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size()); - this.isMemoryOccupied = true; - } - - /** - * to get the int value - * - * @param index - * @return int value based on index - */ - @Override public int getInt(int index) { - return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.INT_SIZE_IN_BYTE)); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java deleted file mode 100644 index d3bed25..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.memory.MemoryAllocatorFactory; -import org.apache.carbondata.core.unsafe.CarbonUnsafe; - -/** - * Responsible for storing long array data to memory. memory can be on heap or - * offheap based on the user configuration using unsafe interface - */ -public class UnsafeLongMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<long[]> { - - public UnsafeLongMeasureChunkStore(int numberOfRows) { - super(numberOfRows); - } - - /** - * Below method will be used to put long array data to memory - * - * @param data - */ - @Override public void putData(long[] data) { - assert (!this.isMemoryOccupied); - this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator() - .allocate(data.length * CarbonCommonConstants.LONG_SIZE_IN_BYTE); - // copy the data to memory - CarbonUnsafe.unsafe - .copyMemory(data, CarbonUnsafe.LONG_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size()); - this.isMemoryOccupied = true; - } - - /** - * to get the long value - * - * @param index - * @return long value based on index - */ - @Override public long getLong(int index) { - return CarbonUnsafe.unsafe.getLong(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.LONG_SIZE_IN_BYTE)); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java deleted file mode 100644 index ba865cd..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.memory.MemoryAllocatorFactory; -import org.apache.carbondata.core.unsafe.CarbonUnsafe; - -/** - * Responsible for storing short array data to memory. memory can be on heap or - * offheap based on the user configuration using unsafe interface - */ -public class UnsafeShortMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<short[]> { - - public UnsafeShortMeasureChunkStore(int numberOfRows) { - super(numberOfRows); - } - - /** - * Below method will be used to put short array data to memory - * - * @param data - */ - @Override public void putData(short[] data) { - assert (!this.isMemoryOccupied); - this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator() - .allocate(data.length * CarbonCommonConstants.SHORT_SIZE_IN_BYTE); - // copy the data to memory - CarbonUnsafe.unsafe - .copyMemory(data, CarbonUnsafe.SHORT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size()); - this.isMemoryOccupied = true; - } - - /** - * to get the short value - * - * @param index - * @return shot value based on index - */ - @Override public short getShort(int index) { - return CarbonUnsafe.unsafe.getShort(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.SHORT_SIZE_IN_BYTE)); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java deleted file mode 100644 index c6155e7..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe; - -import java.nio.ByteBuffer; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.unsafe.CarbonUnsafe; - -/** - * Below class is responsible to store variable length dimension data chunk in - * memory Memory occupied can be on heap or offheap using unsafe interface - */ -public class UnsafeVariableLengthDimesionDataChunkStore - extends UnsafeAbstractDimensionDataChunkStore { - - /** - * total number of rows - */ - private int numberOfRows; - - /** - * pointers offsets - */ - private long dataPointersOffsets; - - public UnsafeVariableLengthDimesionDataChunkStore(long totalSize, boolean isInvertedIdex, - int numberOfRows) { - super(totalSize, isInvertedIdex, numberOfRows); - this.numberOfRows = numberOfRows; - } - - /** - * Below method will be used to put the rows and its metadata in offheap - * - * @param invertedIndex inverted index to be stored - * @param invertedIndexReverse inverted index reverse to be stored - * @param data data to be stored - */ - @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse, - byte[] data) { - // first put the data, inverted index and reverse inverted index to memory - super.putArray(invertedIndex, invertedIndexReverse, data); - // position from where offsets will start - this.dataPointersOffsets = this.invertedIndexReverseOffset; - if (isExplicitSorted) { - this.dataPointersOffsets += numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE; - } - // As data is of variable length and data format is - // <length in short><data><length in short><data> - // we need to store offset of each data so data can be accessed directly - // for example: - //data = {0,5,1,2,3,4,5,0,6,0,1,2,3,4,5,0,2,8,9} - //so value stored in offset will be position of actual data - // [2,9,17] - // to store this value we need to get the actual data length + 2 bytes used for storing the - // length - - // start position will be used to store the current data position - int startOffset = 0; - // position from where offsets will start - long pointerOffsets = this.dataPointersOffsets; - // as first position will be start from 2 byte as data is stored first in the memory block - // we need to skip first two bytes this is because first two bytes will be length of the data - // which we have to skip - CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + pointerOffsets, - CarbonCommonConstants.SHORT_SIZE_IN_BYTE); - // incrementing the pointers as first value is already filled and as we are storing as int - // we need to increment the 4 bytes to set the position of the next value to set - pointerOffsets += CarbonCommonConstants.INT_SIZE_IN_BYTE; - // creating a byte buffer which will wrap the length of the row - // using byte buffer as unsafe will return bytes in little-endian encoding - ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.SHORT_SIZE_IN_BYTE); - // store length of data - byte[] length = new byte[CarbonCommonConstants.SHORT_SIZE_IN_BYTE]; - // as first offset is already stored, we need to start from the 2nd row in data array - for (int i = 1; i < numberOfRows; i++) { - // first copy the length of previous row - CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + startOffset, length, CarbonUnsafe.BYTE_ARRAY_OFFSET, - CarbonCommonConstants.SHORT_SIZE_IN_BYTE); - buffer.put(length); - buffer.flip(); - // so current row position will be - // previous row length + 2 bytes used for storing previous row data - startOffset += CarbonCommonConstants.SHORT_SIZE_IN_BYTE + buffer.getShort(); - // as same byte buffer is used to avoid creating many byte buffer for each row - // we need to clear the byte buffer - buffer.clear(); - // now put the offset of current row, here we need to add 2 more bytes as current will - // also have length part so we have to skip length - CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + pointerOffsets, - startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE); - // incrementing the pointers as first value is already filled and as we are storing as int - // we need to increment the 4 bytes to set the position of the next value to set - pointerOffsets += CarbonCommonConstants.INT_SIZE_IN_BYTE; - } - - } - - /** - * Below method will be used to get the row based on row id passed - * - * @param index - * @return row - */ - @Override public byte[] getRow(int rowId) { - byte[] data = null; - try { - // if column was explicitly sorted we need to get the rowid based inverted index reverse - if (isExplicitSorted) { - rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId - * CarbonCommonConstants.INT_SIZE_IN_BYTE)); - } - // now to get the row from memory block we need to do following thing - // 1. first get the current offset - // 2. if it's not a last row- get the next row offset - // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset - // else subtract the current row offset + 2 bytes(to skip the data length) - // with complete data length - int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (rowId - * CarbonCommonConstants.INT_SIZE_IN_BYTE)); - short length = 0; - // calculating the length of data - if (rowId < numberOfRows - 1) { - int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((rowId + 1) - * CarbonCommonConstants.INT_SIZE_IN_BYTE)); - length = (short) (OffsetOfNextdata - (currentDataOffset - + CarbonCommonConstants.SHORT_SIZE_IN_BYTE)); - } else { - // for last record we need to subtract with data length - length = (short) (this.dataLength - currentDataOffset); - } - data = new byte[length]; - CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + currentDataOffset, data, - CarbonUnsafe.BYTE_ARRAY_OFFSET, length); - } catch (Throwable t) { - System.out.println(); - } - return data; - } - - /** - * to compare the two byte array - * - * @param index index of first byte array - * @param compareValue value of to be compared - * @return compare result - */ - @Override public int compareTo(int index, byte[] compareValue) { - // now to get the row from memory block we need to do following thing - // 1. first get the current offset - // 2. if it's not a last row- get the next row offset - // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset - // else subtract the current row offset - // with complete data length get the offset of set of data - int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (index - * CarbonCommonConstants.INT_SIZE_IN_BYTE)); - short length = 0; - // calculating the length of data - if (index < numberOfRows - 1) { - int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((index + 1) - * CarbonCommonConstants.INT_SIZE_IN_BYTE)); - length = (short) (OffsetOfNextdata - (currentDataOffset - + CarbonCommonConstants.SHORT_SIZE_IN_BYTE)); - } else { - // for last record we need to subtract with data length - length = (short) (this.dataLength - currentDataOffset); - } - // as this class handles this variable length data, so filter value can be - // smaller or bigger than than actual data, so we need to take the smaller length - int compareResult = 0; - int compareLength = length < compareValue.length ? length : compareValue.length; - for (int i = 0; i < compareLength; i++) { - compareResult = (CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(), - dataPageMemoryBlock.getBaseOffset() + currentDataOffset) & 0xff) - (compareValue[i] - & 0xff); - // if compare result is not equal we can break - if (compareResult != 0) { - break; - } - // increment the offset by one as comparison is done byte by byte - currentDataOffset++; - } - return compareResult; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java deleted file mode 100644 index 7049d65..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.core.carbon.datastore.exception; - -import java.io.IOException; -import java.util.Locale; - -/** - * Exception class for block builder - * - * @author Administrator - */ -public class IndexBuilderException extends IOException { - /** - * default serial version ID. - */ - private static final long serialVersionUID = 1L; - - /** - * The Error message. - */ - private String msg = ""; - - /** - * Constructor - * - * @param msg The error message for this exception. - */ - public IndexBuilderException(String msg) { - super(msg); - this.msg = msg; - } - - /** - * Constructor - * - * @param msg exception message - * @param throwable detail exception - */ - public IndexBuilderException(String msg, Throwable throwable) { - super(msg, throwable); - this.msg = msg; - } - - /** - * Constructor - * - * @param throwable exception - */ - public IndexBuilderException(Throwable throwable) { - super(throwable); - } - - /** - * This method is used to get the localized message. - * - * @param locale - A Locale object represents a specific geographical, - * political, or cultural region. - * @return - Localized error message. - */ - public String getLocalizedMessage(Locale locale) { - return ""; - } - - /** - * getLocalizedMessage - */ - @Override public String getLocalizedMessage() { - return super.getLocalizedMessage(); - } - - /** - * getMessage - */ - public String getMessage() { - return this.msg; - } -} - http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java deleted file mode 100644 index 99ae8fe..0000000 --- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.core.carbon.datastore.impl.btree; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import org.apache.carbondata.core.carbon.datastore.BtreeBuilder; -import org.apache.carbondata.core.carbon.datastore.IndexKey; -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.util.CarbonProperties; - -/** - * Abstract Btree based builder - */ -public abstract class AbstractBTreeBuilder implements BtreeBuilder { - - /** - * default Number of keys per page - */ - private static final int DEFAULT_NUMBER_OF_ENTRIES_NONLEAF = 32; - - /** - * Maximum number of entries in intermediate nodes - */ - protected int maxNumberOfEntriesInNonLeafNodes; - - /** - * Number of leaf nodes - */ - protected int nLeaf; - - /** - * root node of a btree - */ - protected BTreeNode root; - - public AbstractBTreeBuilder() { - maxNumberOfEntriesInNonLeafNodes = Integer.parseInt(CarbonProperties.getInstance() - .getProperty("com.huawei.datastore.internalnodesize", - DEFAULT_NUMBER_OF_ENTRIES_NONLEAF + "")); - } - - /** - * Below method is to build the intermediate node of the btree - * - * @param curNode current node - * @param childNodeGroups children group which will have all the children for - * particular intermediate node - * @param currentGroup current group - * @param interNSKeyList list if keys - * @param numberOfInternalNode number of internal node - */ - protected void addIntermediateNode(BTreeNode curNode, List<BTreeNode[]> childNodeGroups, - BTreeNode[] currentGroup, List<List<IndexKey>> interNSKeyList, int numberOfInternalNode) { - - int groupCounter; - // Build internal nodes level by level. Each upper node can have - // upperMaxEntry keys and upperMaxEntry+1 children - int remainder; - int nHigh = numberOfInternalNode; - boolean bRootBuilt = false; - remainder = nLeaf % (maxNumberOfEntriesInNonLeafNodes); - List<IndexKey> interNSKeys = null; - while (nHigh > 1 || !bRootBuilt) { - List<BTreeNode[]> internalNodeGroups = - new ArrayList<BTreeNode[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - List<List<IndexKey>> interNSKeyTmpList = - new ArrayList<List<IndexKey>>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - numberOfInternalNode = 0; - for (int i = 0; i < nHigh; i++) { - // Create a new internal node - curNode = new BTreeNonLeafNode(); - // Allocate a new node group if current node group is full - groupCounter = i % (maxNumberOfEntriesInNonLeafNodes); - if (groupCounter == 0) { - // Create new node group - currentGroup = new BTreeNonLeafNode[maxNumberOfEntriesInNonLeafNodes]; - internalNodeGroups.add(currentGroup); - numberOfInternalNode++; - interNSKeys = new ArrayList<IndexKey>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - interNSKeyTmpList.add(interNSKeys); - } - - // Add the new internal node to current group - if (null != currentGroup) { - currentGroup[groupCounter] = curNode; - } - int nNodes; - - if (i == nHigh - 1 && remainder != 0) { - nNodes = remainder; - } else { - nNodes = maxNumberOfEntriesInNonLeafNodes; - } - // Point the internal node to its children node group - curNode.setChildren(childNodeGroups.get(i)); - // Fill the internal node with keys based on its child nodes - for (int j = 0; j < nNodes; j++) { - curNode.setKey(interNSKeyList.get(i).get(j)); - if (j == 0 && null != interNSKeys) { - interNSKeys.add(interNSKeyList.get(i).get(j)); - - } - } - } - // If nHigh is 1, we have the root node - if (nHigh == 1) { - bRootBuilt = true; - } - - remainder = nHigh % (maxNumberOfEntriesInNonLeafNodes); - nHigh = numberOfInternalNode; - childNodeGroups = internalNodeGroups; - interNSKeyList = interNSKeyTmpList; - } - root = curNode; - } - - /** - * Below method is to convert the start key - * into fixed and variable length key. - * data format<lenght><fixed length key><length><variable length key> - * - * @param startKey - * @return Index key - */ - protected IndexKey convertStartKeyToNodeEntry(byte[] startKey) { - ByteBuffer buffer = ByteBuffer.wrap(startKey); - buffer.rewind(); - int dictonaryKeySize = buffer.getInt(); - int nonDictonaryKeySize = buffer.getInt(); - byte[] dictionaryKey = new byte[dictonaryKeySize]; - buffer.get(dictionaryKey); - byte[] nonDictionaryKey = new byte[nonDictonaryKeySize]; - buffer.get(nonDictionaryKey); - return new IndexKey(dictionaryKey, nonDictionaryKey); - } - - /** - * Below method will be used to get the first data block - * in Btree case it will be root node - */ - @Override public BTreeNode get() { - return root; - } -}
