Refactor carbon-core module for code clean up (#720) * refactor core module
* fix style Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/63d3284e Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/63d3284e Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/63d3284e Branch: refs/heads/master Commit: 63d3284e4f0a09ef5248065da19c2bd3db371e08 Parents: d95742d Author: Jacky Li <jacky.li...@huawei.com> Authored: Thu Jun 23 14:19:58 2016 +0800 Committer: Ravindra Pesala <ravi.pes...@gmail.com> Committed: Thu Jun 23 11:49:58 2016 +0530 ---------------------------------------------------------------------- .../org/carbondata/common/CarbonIterator.java | 38 ++ .../store/columnar/ColumnGroupModel.java | 137 +++++ .../store/columnar/ColumnarKeyStoreInfo.java | 1 - .../store/columnar/UnBlockIndexer.java | 23 - .../fileperations/AtomicFileOperations.java | 33 -- .../fileperations/AtomicFileOperationsImpl.java | 87 ---- .../store/fileperations/FileWriteOperation.java | 25 - .../core/datastorage/util/StoreFactory.java | 143 +----- .../core/file/manager/composite/FileData.java | 74 --- .../file/manager/composite/FileManager.java | 68 --- .../composite/IFileManagerComposite.java | 59 --- .../core/iterator/CarbonIterator.java | 38 -- .../core/locks/AbstractCarbonLock.java | 77 --- .../core/locks/CarbonLockFactory.java | 72 --- .../org/carbondata/core/locks/HdfsFileLock.java | 89 ---- .../org/carbondata/core/locks/ICarbonLock.java | 40 -- .../carbondata/core/locks/LocalFileLock.java | 162 ------ .../org/carbondata/core/locks/LockUsage.java | 29 -- .../carbondata/core/locks/ZooKeeperLocking.java | 171 ------- .../carbondata/core/locks/ZookeeperInit.java | 64 --- .../core/metadata/LevelNormalizedProps.java | 107 ---- .../carbondata/core/metadata/SliceMetaData.java | 333 ------------- .../carbondata/core/util/CarbonMergerUtil.java | 75 --- .../core/util/CarbonSliceAndFiles.java | 88 ---- .../org/carbondata/core/util/CarbonUtil.java | 499 +------------------ .../carbondata/core/vo/ColumnGroupModel.java | 137 ----- .../core/writer/CarbonDataWriter.java | 482 ------------------ .../carbondata/scan/executor/QueryExecutor.java | 2 +- .../scan/executor/impl/DetailQueryExecutor.java | 2 +- .../impl/DetailRawRecordQueryExecutor.java | 2 +- .../resolver/RestructureFilterResolverImpl.java | 8 - .../processor/AbstractDataBlockIterator.java | 2 +- .../scan/processor/BlockletIterator.java | 2 +- .../org/carbondata/scan/result/BatchResult.java | 2 +- .../AbstractDetailQueryResultIterator.java | 2 +- .../scan/result/iterator/ChunkRowIterator.java | 2 +- .../scan/result/iterator/RawResultIterator.java | 2 +- .../core/locks/LocalFileLockTest.java | 48 -- .../core/locks/ZooKeeperLockingTest.java | 125 ----- .../carbondata/hadoop/CarbonRecordReader.java | 2 +- .../hadoop/test/util/StoreCreator.java | 6 +- .../spark/merger/CarbonCompactionExecutor.java | 2 +- .../spark/merger/RowResultMerger.java | 2 +- .../carbondata/spark/load/CarbonLoaderUtil.java | 6 +- .../execution/command/carbonTableSchema.scala | 3 +- .../spark/sql/hive/CarbonMetastoreCatalog.scala | 6 +- .../spark/rdd/CarbonDataRDDFactory.scala | 8 +- .../carbondata/spark/rdd/CarbonScanRDD.scala | 2 +- .../fileoperations/AtomicFileOperations.java | 33 ++ .../AtomicFileOperationsImpl.java | 87 ++++ .../lcm/fileoperations/FileWriteOperation.java | 25 + .../lcm/locks/AbstractCarbonLock.java | 77 +++ .../carbondata/lcm/locks/CarbonLockFactory.java | 72 +++ .../org/carbondata/lcm/locks/HdfsFileLock.java | 89 ++++ .../org/carbondata/lcm/locks/ICarbonLock.java | 40 ++ .../org/carbondata/lcm/locks/LocalFileLock.java | 162 ++++++ .../org/carbondata/lcm/locks/LockUsage.java | 29 ++ .../carbondata/lcm/locks/ZooKeeperLocking.java | 171 +++++++ .../org/carbondata/lcm/locks/ZookeeperInit.java | 64 +++ .../lcm/status/SegmentStatusManager.java | 12 +- .../processing/mdkeygen/MDKeyGenStep.java | 8 +- .../processing/mdkeygen/file/FileData.java | 74 +++ .../processing/mdkeygen/file/FileManager.java | 68 +++ .../mdkeygen/file/IFileManagerComposite.java | 59 +++ .../merger/util/CarbonSliceMergerUtil.java | 366 -------------- .../store/CarbonFactDataHandlerColumnar.java | 6 +- .../store/CarbonFactDataHandlerModel.java | 2 +- .../store/colgroup/ColGroupDataHolder.java | 2 +- .../store/colgroup/ColGroupMinMax.java | 2 +- .../store/writer/AbstractFactDataWriter.java | 4 +- ...actDataWriterImplForIntIndexAndAggBlock.java | 2 +- .../CarbonCSVBasedDimSurrogateKeyGen.java | 2 +- .../csvbased/CarbonCSVBasedSeqGenStep.java | 6 +- .../FileStoreSurrogateKeyGenForCSV.java | 6 +- .../carbondata/lcm/locks/LocalFileLockTest.java | 48 ++ .../lcm/locks/ZooKeeperLockingTest.java | 125 +++++ .../store/colgroup/ColGroupMinMaxTest.java | 7 +- 77 files changed, 1458 insertions(+), 3577 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/common/src/main/java/org/carbondata/common/CarbonIterator.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/carbondata/common/CarbonIterator.java b/common/src/main/java/org/carbondata/common/CarbonIterator.java new file mode 100644 index 0000000..2ac2ff3 --- /dev/null +++ b/common/src/main/java/org/carbondata/common/CarbonIterator.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.carbondata.common; + +import java.util.Iterator; + +/** + * CarbonIterator adds default implement for remove. This is required for Java 7. + * @param <E> + */ +public abstract class CarbonIterator<E> implements Iterator<E> { + + @Override public abstract boolean hasNext(); + + @Override public abstract E next(); + + @Override public void remove() { + throw new UnsupportedOperationException("remove"); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java new file mode 100644 index 0000000..532c6e9 --- /dev/null +++ b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.carbondata.core.datastorage.store.columnar; + +public class ColumnGroupModel { + + /** + * cardinality all dimension + */ + private int[] colGroupCardinality; + + /** + * number of columns in columnar block + */ + private int[] columnSplit; + + /** + * total number of columns + */ + private int noOfColumnsStore; + + /** + * whether given index is columnar or not + * true: columnar + * false: row block + */ + private boolean[] columnarStore; + + /** + * column groups + * e.g + * {{0,1,2},3,4,{5,6}} + */ + private int[][] columnGroups; + + /** + * @return cardinality of column groups + */ + public int[] getColumnGroupCardinality() { + return colGroupCardinality; + } + + /** + * set columngroup cardinality + * + * @param columnGroupCardinality + */ + public void setColumnGroupCardinality(int[] columnGroupCardinality) { + this.colGroupCardinality = columnGroupCardinality; + } + + /** + * return columnSplit + * + * @return + */ + public int[] getColumnSplit() { + return columnSplit; + } + + /** + * set columnSplit + * + * @param split + */ + public void setColumnSplit(int[] split) { + this.columnSplit = split; + } + + /** + * @return no of columnar block + */ + public int getNoOfColumnStore() { + return this.noOfColumnsStore; + } + + /** + * set no of columnar block + * + * @param noOfColumnsStore + */ + public void setNoOfColumnStore(int noOfColumnsStore) { + this.noOfColumnsStore = noOfColumnsStore; + } + + /** + * it's an identifier for row block or single column block + * + * @param columnarStore + */ + public void setColumnarStore(boolean[] columnarStore) { + this.columnarStore = columnarStore; + } + + /** + * set column groups + * + * @param columnGroups + */ + public void setColumnGroup(int[][] columnGroups) { + this.columnGroups = columnGroups; + } + + /** + * check if given column group is columnar + * + * @param colGroup + * @return true if given block is columnar + */ + public boolean isColumnar(int colGroup) { + return columnarStore[colGroup]; + } + + /** + * @return columngroups + */ + public int[][] getColumnGroup() { + return this.columnGroups; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java index c80a887..0cf0d99 100644 --- a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java +++ b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreInfo.java @@ -20,7 +20,6 @@ package org.carbondata.core.datastorage.store.columnar; import org.carbondata.core.keygenerator.mdkey.NumberCompressor; -import org.carbondata.core.vo.ColumnGroupModel; public class ColumnarKeyStoreInfo { private int numberOfKeys; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java index 26eb2ac..a20320b 100644 --- a/core/src/main/java/org/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java +++ b/core/src/main/java/org/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java @@ -27,29 +27,6 @@ public final class UnBlockIndexer { } - public static short[] uncompressIndex(short[] indexData, short[] indexMap) { - int actualSize = indexData.length; - for (int i = 0; i < indexMap.length; i++) { - actualSize += indexData[indexMap[i] + 1] - indexData[indexMap[i]] - 1; - } - short[] indexes = new short[actualSize]; - int k = 0; - for (short i = 0; i < indexData.length; i++) { - int index = Arrays.binarySearch(indexMap, i); - if (index > -1) { - for (short j = indexData[indexMap[index]]; j <= indexData[indexMap[index] + 1]; j++) { - indexes[k] = j; - k++; - } - i++; - } else { - indexes[k] = indexData[i]; - k++; - } - } - return indexes; - } - public static int[] uncompressIndex(int[] indexData, int[] indexMap) { int actualSize = indexData.length; for (int i = 0; i < indexMap.length; i++) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperations.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperations.java b/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperations.java deleted file mode 100644 index c5dcb29..0000000 --- a/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperations.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.carbondata.core.datastorage.store.fileperations; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; - -public interface AtomicFileOperations { - - DataInputStream openForRead() throws IOException; - - void close() throws IOException; - - DataOutputStream openForWrite(FileWriteOperation operation) throws IOException; -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperationsImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperationsImpl.java b/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperationsImpl.java deleted file mode 100644 index 41f4580..0000000 --- a/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/AtomicFileOperationsImpl.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.carbondata.core.datastorage.store.fileperations; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; - -import org.carbondata.core.constants.CarbonCommonConstants; -import org.carbondata.core.datastorage.store.filesystem.CarbonFile; -import org.carbondata.core.datastorage.store.impl.FileFactory; -import org.carbondata.core.datastorage.store.impl.FileFactory.FileType; - -public class AtomicFileOperationsImpl implements AtomicFileOperations { - - private String filePath; - - private FileType fileType; - - private String tempWriteFilePath; - - private DataOutputStream dataOutStream; - - public AtomicFileOperationsImpl(String filePath, FileType fileType) { - this.filePath = filePath; - - this.fileType = fileType; - } - - @Override public DataInputStream openForRead() throws IOException { - return FileFactory.getDataInputStream(filePath, fileType); - } - - @Override public DataOutputStream openForWrite(FileWriteOperation operation) throws IOException { - - filePath = filePath.replace("\\", "/"); - - tempWriteFilePath = filePath + CarbonCommonConstants.TEMPWRITEFILEEXTENSION; - - if (FileFactory.isFileExist(tempWriteFilePath, fileType)) { - FileFactory.getCarbonFile(tempWriteFilePath, fileType).delete(); - } - - FileFactory.createNewFile(tempWriteFilePath, fileType); - - dataOutStream = FileFactory.getDataOutputStream(tempWriteFilePath, fileType); - - return dataOutStream; - - } - - /* (non-Javadoc) - * @see com.huawei.unibi.carbon.datastorage.store.fileperations.AtomicFileOperations#close() - */ - @Override public void close() throws IOException { - - if (null != dataOutStream) { - dataOutStream.close(); - - CarbonFile tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType); - - if (!tempFile.renameForce(filePath)) { - throw new IOException("temporary file renaming failed, src=" - + tempFile.getPath() + ", dest=" + filePath); - } - } - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/FileWriteOperation.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/FileWriteOperation.java b/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/FileWriteOperation.java deleted file mode 100644 index 58026d9..0000000 --- a/core/src/main/java/org/carbondata/core/datastorage/store/fileperations/FileWriteOperation.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.carbondata.core.datastorage.store.fileperations; - -public enum FileWriteOperation { - - APPEND, OVERWRITE -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java b/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java index 725ee56..8fc5aac 100644 --- a/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java +++ b/core/src/main/java/org/carbondata/core/datastorage/util/StoreFactory.java @@ -20,63 +20,28 @@ package org.carbondata.core.datastorage.util; import org.carbondata.core.constants.CarbonCommonConstants; -import org.carbondata.core.datastorage.store.FileHolder; -import org.carbondata.core.datastorage.store.NodeKeyStore; import org.carbondata.core.datastorage.store.NodeMeasureDataStore; -import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStore; -import org.carbondata.core.datastorage.store.columnar.ColumnarKeyStoreInfo; import org.carbondata.core.datastorage.store.compression.ValueCompressionModel; -import org.carbondata.core.datastorage.store.impl.data.compressed.HeavyCompressedDoubleArrayDataFileStore; import org.carbondata.core.datastorage.store.impl.data.compressed.HeavyCompressedDoubleArrayDataInMemoryStore; -import org.carbondata.core.datastorage.store.impl.data.uncompressed.DoubleArrayDataFileStore; + import org.carbondata.core.datastorage.store.impl.data.uncompressed.DoubleArrayDataInMemoryStore; -import org.carbondata.core.datastorage.store.impl.key.columnar.compressed.CompressedColumnarFileKeyStore; -import org.carbondata.core.datastorage.store.impl.key.columnar.compressed.CompressedColumnarInMemoryStore; -import org.carbondata.core.datastorage.store.impl.key.columnar.uncompressed.UnCompressedColumnarFileKeyStore; -import org.carbondata.core.datastorage.store.impl.key.columnar.uncompressed.UnCompressedColumnarInMemoryStore; -import org.carbondata.core.datastorage.store.impl.key.compressed.CompressedSingleArrayKeyFileStore; -import org.carbondata.core.datastorage.store.impl.key.compressed.CompressedSingleArrayKeyInMemoryStore; -import org.carbondata.core.datastorage.store.impl.key.uncompressed.SingleArrayKeyFileStore; -import org.carbondata.core.datastorage.store.impl.key.uncompressed.SingleArrayKeyInMemoryStore; import org.carbondata.core.util.CarbonProperties; public final class StoreFactory { /** - * Single Array Key store. - */ - private static final String SINGLE_ARRAY = "SINGLE_ARRAY"; - /** - * Compressed single array key store. - */ - private static final String COMPRESSED_SINGLE_ARRAY = "COMPRESSED_SINGLE_ARRAY"; - /** * Double array data store. */ private static final String COMPRESSED_DOUBLE_ARRAY = "COMPRESSED_DOUBLE_ARRAY"; /** - * key type. - */ - private static StoreType keyType; - /** * value type. */ private static StoreType valueType; static { - String keytype = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.KEYSTORE_TYPE, - CarbonCommonConstants.KEYSTORE_TYPE_DEFAULT_VAL); String valuetype = CarbonProperties.getInstance() .getProperty(CarbonCommonConstants.VALUESTORE_TYPE, CarbonCommonConstants.VALUESTORE_TYPE_DEFAULT_VAL); - // set key type - if (COMPRESSED_SINGLE_ARRAY.equals(keytype)) { - keyType = StoreType.COMPRESSED_SINGLE_ARRAY; - } else if (SINGLE_ARRAY.equals(keytype)) { - keyType = StoreType.SINGLE_ARRAY; - } else { - keyType = StoreType.COMPRESSED_SINGLE_ARRAY; - } // set value type if (COMPRESSED_DOUBLE_ARRAY.equals(valuetype)) { valueType = StoreType.COMPRESSED_DOUBLE_ARRAY; @@ -89,111 +54,6 @@ public final class StoreFactory { } - public static NodeKeyStore createKeyStore(int size, int elementSize, boolean isLeaf, - boolean isFileStore, long offset, String fileName, int length, FileHolder fileHolder) { - switch (keyType) { - case SINGLE_ARRAY: - - if (isFileStore) { - return new SingleArrayKeyFileStore(size, elementSize, offset, fileName, length); - } else { - return new SingleArrayKeyInMemoryStore(size, elementSize, offset, fileName, fileHolder, - length); - } - default: - - if (isLeaf) { - if (isFileStore) { - return new CompressedSingleArrayKeyFileStore(size, elementSize, offset, fileName, - length); - } else { - return new CompressedSingleArrayKeyInMemoryStore(size, elementSize, offset, fileName, - fileHolder, length); - } - } else { - if (isFileStore) { - return new SingleArrayKeyFileStore(size, elementSize, offset, fileName, length); - } else { - return new SingleArrayKeyInMemoryStore(size, elementSize, offset, fileName, fileHolder, - length); - } - } - } - } - - public static NodeKeyStore createKeyStore(int size, int elementSize, boolean isLeaf) { - switch (keyType) { - case SINGLE_ARRAY: - - return new SingleArrayKeyInMemoryStore(size, elementSize); - - default: - - if (isLeaf) { - return new CompressedSingleArrayKeyInMemoryStore(size, elementSize); - } else { - return new SingleArrayKeyInMemoryStore(size, elementSize); - } - - } - } - - public static ColumnarKeyStore createColumnarKeyStore(ColumnarKeyStoreInfo columnarKeyStoreInfo, - FileHolder fileHolder, boolean isFileStore) { - switch (keyType) { - case SINGLE_ARRAY: - - if (isFileStore) { - return new UnCompressedColumnarFileKeyStore(columnarKeyStoreInfo); - } else { - return new UnCompressedColumnarInMemoryStore(columnarKeyStoreInfo, fileHolder); - } - default: - - if (isFileStore) { - return new CompressedColumnarFileKeyStore(columnarKeyStoreInfo); - } else { - return new CompressedColumnarInMemoryStore(columnarKeyStoreInfo, fileHolder); - } - } - } - - public static NodeMeasureDataStore createDataStore(boolean isFileStore, - ValueCompressionModel compressionModel, long[] offset, int[] length, String filePath, - FileHolder fileHolder) { - switch (valueType) { - - case COMPRESSED_DOUBLE_ARRAY: - - if (isFileStore) { - return new DoubleArrayDataFileStore(compressionModel, offset, filePath, length); - } else { - return new DoubleArrayDataInMemoryStore(compressionModel, offset, length, filePath, - fileHolder); - } - - case HEAVY_VALUE_COMPRESSION: - - if (isFileStore) { - return new HeavyCompressedDoubleArrayDataFileStore(compressionModel, offset, length, - filePath); - } else { - return new HeavyCompressedDoubleArrayDataInMemoryStore(compressionModel, offset, length, - filePath, fileHolder); - } - default: - - if (isFileStore) { - return new HeavyCompressedDoubleArrayDataFileStore(compressionModel, offset, length, - filePath); - } else { - return new HeavyCompressedDoubleArrayDataInMemoryStore(compressionModel, offset, length, - filePath, fileHolder); - } - - } - } - public static NodeMeasureDataStore createDataStore(ValueCompressionModel compressionModel) { switch (valueType) { case COMPRESSED_DOUBLE_ARRAY: @@ -210,7 +70,6 @@ public final class StoreFactory { * enum defined. */ public enum StoreType { - SINGLE_ARRAY, COMPRESSED_SINGLE_ARRAY, COMPRESSED_DOUBLE_ARRAY, HEAVY_VALUE_COMPRESSION http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/file/manager/composite/FileData.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/file/manager/composite/FileData.java b/core/src/main/java/org/carbondata/core/file/manager/composite/FileData.java deleted file mode 100644 index cb4efc6..0000000 --- a/core/src/main/java/org/carbondata/core/file/manager/composite/FileData.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.carbondata.core.file.manager.composite; - -import org.carbondata.core.writer.HierarchyValueWriterForCSV; - -public class FileData extends FileManager { - - /** - * Store Path - */ - private String storePath; - - /** - * hierarchyValueWriter - */ - private HierarchyValueWriterForCSV hierarchyValueWriter; - - public FileData(String fileName, String storePath) { - this.fileName = fileName; - this.storePath = storePath; - } - - /** - * @return Returns the fileName. - */ - public String getFileName() { - return fileName; - } - - /** - * @return Returns the storePath. - */ - public String getStorePath() { - return storePath; - } - - /** - * get Hierarchy Value writer - * - * @return - */ - public HierarchyValueWriterForCSV getHierarchyValueWriter() { - return hierarchyValueWriter; - } - - /** - * Set Hierarchy Value Writer. - * - * @param hierarchyValueWriter - */ - public void setHierarchyValueWriter(HierarchyValueWriterForCSV hierarchyValueWriter) { - this.hierarchyValueWriter = hierarchyValueWriter; - } - -} - http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/file/manager/composite/FileManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/file/manager/composite/FileManager.java b/core/src/main/java/org/carbondata/core/file/manager/composite/FileManager.java deleted file mode 100644 index 7e98bed..0000000 --- a/core/src/main/java/org/carbondata/core/file/manager/composite/FileManager.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.carbondata.core.file.manager.composite; - -import java.util.ArrayList; -import java.util.List; - -import org.carbondata.core.constants.CarbonCommonConstants; - -public class FileManager implements IFileManagerComposite { - /** - * listOfFileData, composite parent which holds the different objects - */ - protected List<IFileManagerComposite> listOfFileData = - new ArrayList<IFileManagerComposite>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - - protected String fileName; - - @Override public void add(IFileManagerComposite customData) { - listOfFileData.add(customData); - } - - @Override public void remove(IFileManagerComposite customData) { - listOfFileData.remove(customData); - - } - - @Override public IFileManagerComposite get(int i) { - return listOfFileData.get(i); - } - - /** - * Renames the File/Folders - */ - public boolean rename(IFileManagerComposite composite) { - return false; - } - - @Override public void setName(String name) { - this.fileName = name; - } - - /** - * Return the size - */ - public int size() { - return listOfFileData.size(); - } - -} - http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/file/manager/composite/IFileManagerComposite.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/file/manager/composite/IFileManagerComposite.java b/core/src/main/java/org/carbondata/core/file/manager/composite/IFileManagerComposite.java deleted file mode 100644 index a83b9f0..0000000 --- a/core/src/main/java/org/carbondata/core/file/manager/composite/IFileManagerComposite.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.carbondata.core.file.manager.composite; - -public interface IFileManagerComposite { - /** - * Add the data which can be either row Folder(Composite) or File - * - * @param customData - */ - void add(IFileManagerComposite customData); - - /** - * Remove the CustomData type object from the IFileManagerComposite object hierarchy. - * - * @param customData - */ - void remove(IFileManagerComposite customData); - - /** - * get the CustomData type object name - * - * @return CustomDataIntf type - */ - IFileManagerComposite get(int i); - - /** - * set the CustomData type object name - * - * @param name - */ - void setName(String name); - - /** - * Get the size - * - * @return - */ - int size(); - -} - http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/iterator/CarbonIterator.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/iterator/CarbonIterator.java b/core/src/main/java/org/carbondata/core/iterator/CarbonIterator.java deleted file mode 100644 index c651a80..0000000 --- a/core/src/main/java/org/carbondata/core/iterator/CarbonIterator.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.carbondata.core.iterator; - -import java.util.Iterator; - -/** - * CarbonIterator adds default implement for remove. This is required for Java 7. - * @param <E> - */ -public abstract class CarbonIterator<E> implements Iterator<E> { - - @Override public abstract boolean hasNext(); - - @Override public abstract E next(); - - @Override public void remove() { - throw new UnsupportedOperationException("remove"); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/AbstractCarbonLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/locks/AbstractCarbonLock.java b/core/src/main/java/org/carbondata/core/locks/AbstractCarbonLock.java deleted file mode 100644 index b77cff0..0000000 --- a/core/src/main/java/org/carbondata/core/locks/AbstractCarbonLock.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.carbondata.core.locks; - -import org.carbondata.core.constants.CarbonCommonConstants; -import org.carbondata.core.util.CarbonProperties; - -/** - * This is the abstract class of the lock implementations.This handles the - * retrying part of the locking. - */ -public abstract class AbstractCarbonLock implements ICarbonLock { - private int retryCount; - - private int retryTimeout; - - public abstract boolean lock(); - - /** - * API for enabling the locking of file with retries. - */ - public boolean lockWithRetries() { - try { - for (int i = 0; i < retryCount; i++) { - if (lock()) { - return true; - } else { - Thread.sleep(retryTimeout * 1000L); - } - } - } catch (InterruptedException e) { - return false; - } - return false; - } - - /** - * Initializes the retry count and retry timeout. - * This will determine how many times to retry to acquire lock and the retry timeout. - */ - protected void initRetry() { - String retries = CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK); - try { - retryCount = Integer.parseInt(retries); - } catch (NumberFormatException e) { - retryCount = CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK_DEFAULT; - } - - String maxTimeout = CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK); - try { - retryTimeout = Integer.parseInt(maxTimeout); - } catch (NumberFormatException e) { - retryTimeout = CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK_DEFAULT; - } - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java b/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java deleted file mode 100644 index 99f4ab1..0000000 --- a/core/src/main/java/org/carbondata/core/locks/CarbonLockFactory.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.carbondata.core.locks; - -import org.carbondata.core.constants.CarbonCommonConstants; -import org.carbondata.core.util.CarbonProperties; - -/** - * This class is a Lock factory class which is used to provide lock objects. - * Using this lock object client can request the lock and unlock. - */ -public class CarbonLockFactory { - - /** - * lockTypeConfigured to check if zookeeper feature is enabled or not for carbon. - */ - private static String lockTypeConfigured; - - static { - CarbonLockFactory.updateZooKeeperLockingStatus(); - } - - /** - * This method will determine the lock type. - * - * @param location - * @param lockUsage - * @return - */ - public static ICarbonLock getCarbonLockObj(String location, LockUsage lockUsage) { - switch (lockTypeConfigured.toUpperCase()) { - case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL: - return new LocalFileLock(location, lockUsage); - - case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER: - return new ZooKeeperLocking(location, lockUsage); - - case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS: - return new HdfsFileLock(location, lockUsage); - - default: - throw new UnsupportedOperationException("Not supported the lock type"); - } - - } - - /** - * This method will set the zookeeper status whether zookeeper to be used for locking or not. - */ - private static void updateZooKeeperLockingStatus() { - lockTypeConfigured = CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java deleted file mode 100644 index 745dfa1..0000000 --- a/core/src/main/java/org/carbondata/core/locks/HdfsFileLock.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.carbondata.core.locks; - -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; - -import org.carbondata.core.datastorage.store.impl.FileFactory; - -/** - * This class is used to handle the HDFS File locking. - * This is acheived using the concept of acquiring the data out stream using Append option. - */ -public class HdfsFileLock extends AbstractCarbonLock { - - /** - * location hdfs file location - */ - private String location; - - /** - * lockUsage is used to determine the type of the lock. according to this the lock - * folder will change. - */ - private LockUsage lockUsage; - - private DataOutputStream dataOutputStream; - - /** - * @param location - * @param lockUsage - */ - public HdfsFileLock(String location, LockUsage lockUsage) { - this.location = location; - this.lockUsage = lockUsage; - this.location = location + File.separator + this.lockUsage; - initRetry(); - } - - /* (non-Javadoc) - * @see org.carbondata.core.locks.ICarbonLock#lock() - */ - @Override public boolean lock() { - try { - if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) { - FileFactory.createNewLockFile(location, FileFactory.getFileType(location)); - } - dataOutputStream = - FileFactory.getDataOutputStreamUsingAppend(location, FileFactory.getFileType(location)); - - return true; - - } catch (IOException e) { - return false; - } - } - - /* (non-Javadoc) - * @see org.carbondata.core.locks.ICarbonLock#unlock() - */ - @Override public boolean unlock() { - if (null != dataOutputStream) { - try { - dataOutputStream.close(); - } catch (IOException e) { - return false; - } - } - return true; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/ICarbonLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/locks/ICarbonLock.java b/core/src/main/java/org/carbondata/core/locks/ICarbonLock.java deleted file mode 100644 index af97c2e..0000000 --- a/core/src/main/java/org/carbondata/core/locks/ICarbonLock.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.carbondata.core.locks; - -/** - * Carbon Lock Interface which handles the locking and unlocking. - */ -public interface ICarbonLock { - - /** - * Does the unlocking of the acquired lock. - * - * @return - */ - boolean unlock(); - - /** - * This will acquire the lock and if it doesnt get then it will retry after the confiured time. - * - * @return - */ - boolean lockWithRetries(); - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java deleted file mode 100644 index 362b86f..0000000 --- a/core/src/main/java/org/carbondata/core/locks/LocalFileLock.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.carbondata.core.locks; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.channels.FileChannel; -import java.nio.channels.FileLock; -import java.nio.channels.OverlappingFileLockException; - -import org.carbondata.common.logging.LogService; -import org.carbondata.common.logging.LogServiceFactory; -import org.carbondata.core.datastorage.store.impl.FileFactory; - -/** - * This class handles the file locking in the local file system. - * This will be handled using the file channel lock API. - */ -public class LocalFileLock extends AbstractCarbonLock { - /** - * location is the location of the lock file. - */ - private String location; - - /** - * lockUsage will determine the lock folder. so that similar locks will try to acquire - * same lock file. - */ - private LockUsage lockUsage; - - /** - * fileOutputStream of the local lock file - */ - private FileOutputStream fileOutputStream; - - /** - * channel is the FileChannel of the lock file. - */ - private FileChannel channel; - - /** - * fileLock NIO FileLock Object - */ - private FileLock fileLock; - - public static final String tmpPath; - - private String cubeName; - - private String schemaName; - - /** - * LOGGER for logging the messages. - */ - private static final LogService LOGGER = - LogServiceFactory.getLogService(LocalFileLock.class.getName()); - - static { - tmpPath = System.getProperty("java.io.tmpdir"); - } - - /** - * @param location - * @param lockUsage - */ - public LocalFileLock(String location, LockUsage lockUsage) { - this.lockUsage = lockUsage; - location = location.replace("\\", "/"); - String tempStr = location.substring(0, location.lastIndexOf('/')); - schemaName = tempStr.substring(tempStr.lastIndexOf('/') + 1, tempStr.length()); - - cubeName = location.substring(location.lastIndexOf('/') + 1, location.length()); - this.location = - tmpPath + File.separator + schemaName + File.separator + cubeName + File.separator - + this.lockUsage; - initRetry(); - } - - /** - * Lock API for locking of the file channel of the lock file. - * - * @return - */ - @Override public boolean lock() { - try { - String schemaFolderPath = tmpPath + File.separator + schemaName; - String cubeFolderPath = schemaFolderPath + File.separator + cubeName; - // create dir with schema name in tmp location. - if (!FileFactory.isFileExist(schemaFolderPath, FileFactory.getFileType(tmpPath))) { - FileFactory.mkdirs(schemaFolderPath, FileFactory.getFileType(tmpPath)); - } - - // create dir with cube name in tmp location. - if (!FileFactory.isFileExist(cubeFolderPath, FileFactory.getFileType(tmpPath))) { - FileFactory.mkdirs(cubeFolderPath, FileFactory.getFileType(tmpPath)); - } - if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) { - FileFactory.createNewLockFile(location, FileFactory.getFileType(location)); - } - - fileOutputStream = new FileOutputStream(location); - channel = fileOutputStream.getChannel(); - try { - fileLock = channel.tryLock(); - } catch (OverlappingFileLockException e) { - return false; - } - if (null != fileLock) { - return true; - } else { - return false; - } - } catch (IOException e) { - return false; - } - - } - - /** - * Unlock API for unlocking of the acquired lock. - * - * @return - */ - @Override public boolean unlock() { - boolean status; - try { - if (null != fileLock) { - fileLock.release(); - } - status = true; - } catch (IOException e) { - status = false; - } finally { - if (null != fileOutputStream) { - try { - fileOutputStream.close(); - } catch (IOException e) { - LOGGER.error(e.getMessage()); - } - } - } - return status; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/LockUsage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/locks/LockUsage.java b/core/src/main/java/org/carbondata/core/locks/LockUsage.java deleted file mode 100644 index 850b6bf..0000000 --- a/core/src/main/java/org/carbondata/core/locks/LockUsage.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.carbondata.core.locks; - -/** - * This enum is used to define the usecase of the lock. - * Each enum value is one specific lock case. - */ -public enum LockUsage { - METADATA_LOCK, - COMPACTION_LOCK; - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java b/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java deleted file mode 100644 index 8b04ac1..0000000 --- a/core/src/main/java/org/carbondata/core/locks/ZooKeeperLocking.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.carbondata.core.locks; - -import java.util.Collections; -import java.util.List; - -import org.carbondata.common.logging.LogService; -import org.carbondata.common.logging.LogServiceFactory; -import org.carbondata.core.constants.CarbonCommonConstants; - -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.ZooKeeper; - -/** - * For Handling the zookeeper locking implementation - */ -public class ZooKeeperLocking extends AbstractCarbonLock { - - private static final LogService LOGGER = - LogServiceFactory.getLogService(ZooKeeperLocking.class.getName()); - - /** - * zk is the zookeeper client instance - */ - private static ZooKeeper zk; - - /** - * zooKeeperLocation is the location in the zoo keeper file system where the locks will be - * maintained. - */ - private static final String zooKeeperLocation = CarbonCommonConstants.ZOOKEEPER_LOCATION; - - /** - * Unique folder for each cube with SchemaName_CubeName - */ - private final String tableIdFolder; - - /** - * lockName is the name of the lock to use. This name should be same for every process that want - * to share the same lock - */ - private String lockName; - - /** - * lockPath is the unique path created for the each instance of the carbon lock. - */ - private String lockPath; - - private String lockTypeFolder; - - /** - * @param lockUsage - */ - public ZooKeeperLocking(String location, LockUsage lockUsage) { - this.lockName = CarbonCommonConstants.ZOOKEEPER_LOCK; - this.lockTypeFolder = zooKeeperLocation; - location = location.replace("\\", "/"); - String tempStr = location.substring(0, location.lastIndexOf('/')); - String schemaName = tempStr.substring(tempStr.lastIndexOf('/') + 1, tempStr.length()); - - String cubeName = location.substring(location.lastIndexOf('/') + 1, location.length()); - - this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + schemaName - + '.' + cubeName; - - zk = ZookeeperInit.getInstance().getZookeeper(); - - this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + schemaName - + '.' + cubeName + CarbonCommonConstants.FILE_SEPARATOR - + lockUsage.toString(); - try { - createBaseNode(); - // if exists returns null then path doesnt exist. so creating. - if (null == zk.exists(this.tableIdFolder, true)) { - zk.create(this.tableIdFolder, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } - // if exists returns null then path doesnt exist. so creating. - if (null == zk.exists(this.lockTypeFolder, true)) { - zk.create(this.lockTypeFolder, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } - } catch (KeeperException | InterruptedException e) { - LOGGER.error(e, e.getMessage()); - } - initRetry(); - } - - /** - * Creating a znode in which all the znodes (lock files )are maintained. - */ - private void createBaseNode() throws KeeperException, InterruptedException { - if (null == zk.exists(zooKeeperLocation, true)) { - // creating a znode in which all the znodes (lock files )are maintained. - zk.create(zooKeeperLocation, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - } - } - - /** - * Handling of the locking mechanism using zoo keeper. - */ - @Override public boolean lock() { - try { - // create the lock file with lockName. - lockPath = - zk.create(this.lockTypeFolder + CarbonCommonConstants.FILE_SEPARATOR + lockName, null, - Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL); - - // get the children present in zooKeeperLocation. - List<String> nodes = zk.getChildren(this.lockTypeFolder, null); - - // sort the childrens - Collections.sort(nodes); - - // here the logic is , for each lock request zookeeper will create a file ending with - // incremental digits. - // so first request will be 00001 next is 00002 and so on. - // if the current request is 00002 and already one previous request(00001) is present then get - // children will give both nodes. - // after the sort we are checking if the lock path is first or not .if it is first then lock - // has been acquired. - - if (lockPath.endsWith(nodes.get(0))) { - return true; - } else { - // if locking failed then deleting the created lock as next time again new lock file will be - // created. - zk.delete(lockPath, -1); - return false; - } - } catch (KeeperException | InterruptedException e) { - LOGGER.error(e, e.getMessage()); - return false; - } - } - - /** - * @return status where lock file is unlocked or not. - */ - @Override public boolean unlock() { - try { - // exists will return null if the path doesn't exists. - if (null != zk.exists(lockPath, true)) { - zk.delete(lockPath, -1); - lockPath = null; - } - } catch (KeeperException | InterruptedException e) { - LOGGER.error(e, e.getMessage()); - return false; - } - return true; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/locks/ZookeeperInit.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/locks/ZookeeperInit.java b/core/src/main/java/org/carbondata/core/locks/ZookeeperInit.java deleted file mode 100644 index ba2ad57..0000000 --- a/core/src/main/java/org/carbondata/core/locks/ZookeeperInit.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.carbondata.core.locks; - -import java.io.IOException; - -import org.carbondata.common.logging.LogService; -import org.carbondata.common.logging.LogServiceFactory; - -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZooKeeper; - -/** - * This is a singleton class for initialization of zookeeper client. - */ -public class ZookeeperInit { - - private static final LogService LOGGER = - LogServiceFactory.getLogService(ZookeeperInit.class.getName()); - - private static ZookeeperInit zooKeeperInit; - /** - * zk is the zookeeper client instance - */ - private ZooKeeper zk; - - private ZookeeperInit(String zooKeeperUrl) { - - int sessionTimeOut = 100000; - try { - zk = new ZooKeeper(zooKeeperUrl, sessionTimeOut, new DummyWatcher()); - - } catch (IOException e) { - LOGGER.error(e.getMessage()); - } - - } - - public static ZookeeperInit getInstance(String zooKeeperUrl) { - - if (null == zooKeeperInit) { - synchronized (ZookeeperInit.class) { - if (null == zooKeeperInit) { - LOGGER.info("Initiating Zookeeper client."); - zooKeeperInit = new ZookeeperInit(zooKeeperUrl); - } - } - } - return zooKeeperInit; - - } - - public static ZookeeperInit getInstance() { - return zooKeeperInit; - } - - public ZooKeeper getZookeeper() { - return zk; - } - - private static class DummyWatcher implements Watcher { - public void process(WatchedEvent event) { - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/metadata/LevelNormalizedProps.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/metadata/LevelNormalizedProps.java b/core/src/main/java/org/carbondata/core/metadata/LevelNormalizedProps.java deleted file mode 100644 index d1cf630..0000000 --- a/core/src/main/java/org/carbondata/core/metadata/LevelNormalizedProps.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.carbondata.core.metadata; - -public class LevelNormalizedProps { - /** - * isLevelNormalized - */ - private boolean isLevelNormalized; - - /** - * is this level present in fact file - */ - private boolean isDimInFact; - - /** - * hierarchy Name - */ - private String hierName; - - /** - * dimension Name - */ - private String dimName; - - /** - * - */ - private boolean hasAll; - - /** - * Constructor - */ - public LevelNormalizedProps() { - - } - - /** - * @return - */ - public String getDimName() { - return dimName; - } - - /** - * @param dimName - */ - public void setDimName(String dimName) { - this.dimName = dimName; - } - - /** - * @return - */ - public String getHierName() { - return hierName; - } - - /** - * @param hierName - */ - public void setHierName(String hierName) { - this.hierName = hierName; - } - - public boolean isLevelNormalized() { - return isLevelNormalized; - } - - public void setLevelNormalized(boolean isLevelNormalized) { - this.isLevelNormalized = isLevelNormalized; - } - - public boolean isDimInFact() { - return isDimInFact; - } - - public void setDimInFact(boolean isDimInFact) { - this.isDimInFact = isDimInFact; - } - - public boolean isHasAll() { - return hasAll; - } - - public void setHasAll(boolean hasAll) { - this.hasAll = hasAll; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/metadata/SliceMetaData.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/metadata/SliceMetaData.java b/core/src/main/java/org/carbondata/core/metadata/SliceMetaData.java deleted file mode 100644 index c0dcfc4..0000000 --- a/core/src/main/java/org/carbondata/core/metadata/SliceMetaData.java +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.carbondata.core.metadata; - -import java.io.Serializable; -import java.util.Arrays; -import java.util.HashSet; - -import org.carbondata.core.keygenerator.KeyGenerator; - -public class SliceMetaData implements Serializable { - - /** - * serialVersionUID - */ - private static final long serialVersionUID = 3046237866264840878L; - - /** - * Array of dimensions declared. - */ - private String[] dimensions; - - private String complexTypeString; - /** - * actualDimensions - */ - private String[] actualDimensions; - /** - * Array of measures declared. - */ - private String[] measures; - /** - * measuresAggregator - */ - private String[] measuresAggregator; - /** - * Array of newDimensions declared. - */ - private String[] newDimensions; - /** - * actualNewDimensions - */ - private String[] newActualDimensions; - /** - * tableNamesToLoadMandatory - */ - private HashSet<String> tableNamesToLoadMandatory; - /** - * Array of newMeasures declared. - */ - private String[] newMeasures; - /** - * Array of newMsrDfts declared. - */ - private double[] newMsrDfts; - /** - * newMeasuresAggregator - */ - private String[] newMeasuresAggregator; - /** - * heirAnKeySize - */ - private String heirAnKeySize; - /** - * KeyGenerator declared. - */ - private KeyGenerator keyGenerator; - /** - * dimLens - */ - private int[] dimLens; - /** - * actualDimLens - */ - private int[] actualDimLens; - /** - * newDimLens - */ - private int[] newDimLens; - /** - * newActualDimLens - */ - private int[] newActualDimLens; - /** - * oldDimsNewCardinality - */ - private int[] oldDimsNewCardinality; - /** - * isDimCarinalityChanged - */ - private boolean isDimCarinalityChanged; - private String[] newDimsDefVals; - private int[] newDimsSurrogateKeys; - /** - * new keygenerator - */ - private KeyGenerator newKeyGenerator; - - public String getComplexTypeString() { - return complexTypeString; - } - - public void setComplexTypeString(String complexTypeString) { - this.complexTypeString = complexTypeString; - } - - public int[] getNewDimLens() { - return newDimLens; - } - - public void setNewDimLens(int[] newDimLens) { - this.newDimLens = newDimLens; - } - - public int[] getDimLens() { - return dimLens; - } - - public void setDimLens(int[] dimLens) { - this.dimLens = dimLens; - } - - public KeyGenerator getNewKeyGenerator() { - return newKeyGenerator; - } - - public void setNewKeyGenerator(KeyGenerator newKeyGenerator) { - this.newKeyGenerator = newKeyGenerator; - } - - public String[] getDimensions() { - return dimensions; - } - - public void setDimensions(String[] dimensions) { - this.dimensions = dimensions; - } - - public String[] getMeasures() { - return measures; - } - - public void setMeasures(String[] measures) { - this.measures = measures; - } - - public String[] getNewDimensions() { - return newDimensions; - } - - public void setNewDimensions(String[] newDimensions) { - this.newDimensions = newDimensions; - } - - public String[] getNewMeasures() { - return newMeasures; - } - - public void setNewMeasures(String[] newMeasures) { - this.newMeasures = newMeasures; - } - - public double[] getNewMsrDfts() { - return newMsrDfts; - } - - public void setNewMsrDfts(double[] newMsrDfts) { - this.newMsrDfts = newMsrDfts; - } - - public KeyGenerator getKeyGenerator() { - return keyGenerator; - } - - public void setKeyGenerator(KeyGenerator keyGenerator) { - this.keyGenerator = keyGenerator; - } - - @Override public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + Arrays.hashCode(dimensions); - result = prime * result + ((keyGenerator == null) ? 0 : keyGenerator.hashCode()); - result = prime * result + Arrays.hashCode(measures); - result = prime * result + Arrays.hashCode(newDimensions); - result = prime * result + Arrays.hashCode(newMeasures); - result = prime * result + Arrays.hashCode(newMsrDfts); - result = prime * result + Arrays.hashCode(dimLens); - return result; - } - - @Override public boolean equals(Object obj) { - if (obj instanceof SliceMetaData) { - SliceMetaData other = (SliceMetaData) obj; - if (Arrays.equals(dimensions, other.dimensions) && Arrays - .equals(measuresAggregator, other.measuresAggregator) && Arrays - .equals(dimLens, other.dimLens)) { - return true; - } - } - return false; - } - - public boolean isSameAs(SliceMetaData other) { - return (Arrays.equals(newDimensions, other.newDimensions) && Arrays - .equals(newMeasures, other.newMeasures)); - } - - public String[] getMeasuresAggregator() { - return measuresAggregator; - } - - public void setMeasuresAggregator(String[] measuresAggregator) { - this.measuresAggregator = measuresAggregator; - } - - public String[] getNewMeasuresAggregator() { - return newMeasuresAggregator; - } - - public void setNewMeasuresAggregator(String[] newMeasuresAggregator) { - this.newMeasuresAggregator = newMeasuresAggregator; - } - - public String getHeirAnKeySize() { - return heirAnKeySize; - } - - public void setHeirAnKeySize(String heirAnKeySize) { - this.heirAnKeySize = heirAnKeySize; - } - - public boolean isDimCarinalityChanged() { - return isDimCarinalityChanged; - } - - public void setDimCarinalityChanged(boolean isDimCarinalityChanged) { - this.isDimCarinalityChanged = isDimCarinalityChanged; - } - - public int[] getOldDimsNewCardinality() { - return oldDimsNewCardinality; - } - - public void setOldDimsNewCardinality(int[] oldDimsNewCardinality) { - this.oldDimsNewCardinality = oldDimsNewCardinality; - } - - public String[] getNewActualDimensions() { - return newActualDimensions; - } - - public void setNewActualDimensions(String[] newActualDimensions) { - this.newActualDimensions = newActualDimensions; - } - - public int[] getNewActualDimLens() { - return newActualDimLens; - } - - public void setNewActualDimLens(int[] newActualDimLens) { - this.newActualDimLens = newActualDimLens; - } - - public String[] getActualDimensions() { - return actualDimensions; - } - - public void setActualDimensions(String[] actualDimensions) { - this.actualDimensions = actualDimensions; - } - - public int[] getActualDimLens() { - return actualDimLens; - } - - public void setActualDimLens(int[] actualDimLens) { - this.actualDimLens = actualDimLens; - } - - public HashSet<String> getTableNamesToLoadMandatory() { - return tableNamesToLoadMandatory; - } - - public void setTableNamesToLoadMandatory(HashSet<String> tableNamesToLoadMandatory) { - this.tableNamesToLoadMandatory = tableNamesToLoadMandatory; - } - - /** - * return the new dimensions default values - */ - public String[] getNewDimsDefVals() { - return newDimsDefVals; - } - - /** - * set the default values of new dimensions added - */ - public void setNewDimsDefVals(String[] newDimsDefVals) { - this.newDimsDefVals = newDimsDefVals; - } - - /** - * return the surrogate keys of new dimension values - */ - public int[] getNewDimsSurrogateKeys() { - return newDimsSurrogateKeys; - } - - /** - * set the surrogate keys for the new dimension values - */ - public void setNewDimsSurrogateKeys(int[] newDimsSurrogateKeys) { - this.newDimsSurrogateKeys = newDimsSurrogateKeys; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java b/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java index a359339..e2cdf64 100644 --- a/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java +++ b/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java @@ -19,19 +19,9 @@ package org.carbondata.core.util; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - import org.carbondata.common.logging.LogService; import org.carbondata.common.logging.LogServiceFactory; import org.carbondata.core.constants.CarbonCommonConstants; -import org.carbondata.core.datastorage.store.filesystem.CarbonFile; -import org.carbondata.core.datastorage.store.impl.FileFactory; -import org.carbondata.core.datastorage.store.impl.FileFactory.FileType; - -import org.pentaho.di.core.exception.KettleException; /** * Util class for merge activities of 2 loads. @@ -44,71 +34,6 @@ public class CarbonMergerUtil { private static final LogService LOGGER = LogServiceFactory.getLogService(CarbonMergerUtil.class.getName()); - public static List<CarbonSliceAndFiles> getSliceAndFilesList(String storeLocation, - String tableName, FileType fileType, List<String> loadsToBeMerged) { - try { - if (!FileFactory.isFileExist(storeLocation, fileType)) { - return new ArrayList<CarbonSliceAndFiles>(0); - } - } catch (IOException e) { - LOGGER.error("Error occurred :: " + e.getMessage()); - } - CarbonFile file = FileFactory.getCarbonFile(storeLocation, fileType); - - CarbonFile[] listFiles = CarbonUtil.listFiles(file); - if (null == listFiles || listFiles.length < 0) { - return new ArrayList<CarbonSliceAndFiles>(0); - } - Arrays.sort(listFiles, new CarbonFileFolderComparator()); - listFiles = getMergeFilesList(loadsToBeMerged, listFiles); - - return CarbonUtil.getSliceAndFilesList(tableName, listFiles, fileType); - } - - private static CarbonFile[] getMergeFilesList(List<String> loadsToBeMerged, - CarbonFile[] listFiles) { - CarbonFile[] carbonFile = new CarbonFile[loadsToBeMerged.size()]; - int i = 0; - for (CarbonFile listFile : listFiles) { - String loadName = listFile.getName(); - for (String load : loadsToBeMerged) { - if ((CarbonCommonConstants.LOAD_FOLDER + load).equalsIgnoreCase(loadName)) { - carbonFile[i++] = listFile; - } - } - } - return carbonFile; - } - - public static int[] mergeLevelMetadata(String[] sliceLocation, String tableName, - String destinationLocation) { - int[][] cardinalityOfLoads = new int[sliceLocation.length][]; - int i = 0; - for (String loadFolderLoacation : sliceLocation) { - try { - cardinalityOfLoads[i++] = CarbonUtil.getCardinalityFromLevelMetadataFile( - loadFolderLoacation + '/' + CarbonCommonConstants.LEVEL_METADATA_FILE + tableName - + ".metadata"); - } catch (CarbonUtilException e) { - LOGGER - .error("Error occurred :: " + e.getMessage()); - } - } - int[] MaxCardinality = new int[cardinalityOfLoads[0].length]; - - for (int k = 0; k < cardinalityOfLoads[0].length; k++) { - MaxCardinality[k] = Math.max(cardinalityOfLoads[0][k], cardinalityOfLoads[1][k]); - } - - try { - CarbonUtil.writeLevelCardinalityFile(destinationLocation, tableName, MaxCardinality); - } catch (KettleException e) { - LOGGER.error("Error occurred :: " + e.getMessage()); - } - - return MaxCardinality; - } - public static int[] getCardinalityFromLevelMetadata(String path, String tableName) { int[] localCardinality = null; try { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/63d3284e/core/src/main/java/org/carbondata/core/util/CarbonSliceAndFiles.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/carbondata/core/util/CarbonSliceAndFiles.java b/core/src/main/java/org/carbondata/core/util/CarbonSliceAndFiles.java deleted file mode 100644 index 474f247..0000000 --- a/core/src/main/java/org/carbondata/core/util/CarbonSliceAndFiles.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.carbondata.core.util; - -import org.carbondata.core.datastorage.store.filesystem.CarbonFile; -import org.carbondata.core.keygenerator.KeyGenerator; - -public class CarbonSliceAndFiles { - /** - * slice path - */ - private String path; - - /** - * slice fact files - */ - private CarbonFile[] sliceFactFilesList; - - private KeyGenerator keyGen; - - /** - * This method will be used get the slice fact files - * - * @return slice fact files - */ - public CarbonFile[] getSliceFactFilesList() { - return sliceFactFilesList; - } - - /** - * This method will be used to set the slice fact files - * - * @param sliceFactFilesList - */ - public void setSliceFactFilesList(CarbonFile[] sliceFactFilesList) { - this.sliceFactFilesList = sliceFactFilesList; - } - - /** - * This method will return the slice path - * - * @return slice path - */ - public String getPath() { - return path; - } - - /** - * This method will be used to set the slice path - * - * @param path - */ - public void setPath(String path) { - this.path = path; - } - - /** - * @return the keyGen - */ - public KeyGenerator getKeyGen() { - return keyGen; - } - - /** - * @param keyGen the keyGen to set - */ - public void setKeyGen(KeyGenerator keyGen) { - this.keyGen = keyGen; - } - -}