http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java index e2be44d..ebf598f 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java @@ -22,7 +22,7 @@ package org.apache.carbondata.core.scan.filter.resolver; import java.util.ArrayList; import java.util.List; -import org.apache.carbondata.core.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.Expression;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java index 90d5db7..88dfab1 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java @@ -25,7 +25,7 @@ import java.util.Comparator; import java.util.List; import java.util.SortedMap; -import org.apache.carbondata.core.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.metadata.Encoding; import org.apache.carbondata.core.constants.CarbonCommonConstants; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java index 779b532..000825c 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java @@ -18,7 +18,7 @@ */ package org.apache.carbondata.core.scan.filter.resolver.metadata; -import org.apache.carbondata.core.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.Expression; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java index 1b295db..41afa4b 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java @@ -26,7 +26,7 @@ import java.util.List; import java.util.Map; import org.apache.carbondata.core.cache.dictionary.Dictionary; -import org.apache.carbondata.core.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.datastore.block.TableBlockInfo; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java b/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java index 4307604..eabdf82 100644 --- a/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java +++ b/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java @@ -18,8 +18,8 @@ */ package org.apache.carbondata.core.service; -import org.apache.carbondata.core.CarbonTableIdentifier; -import org.apache.carbondata.core.ColumnIdentifier; +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader; import org.apache.carbondata.core.reader.CarbonDictionaryReader; import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/service/PathService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/service/PathService.java b/core/src/main/java/org/apache/carbondata/core/service/PathService.java index 21cc349..c5143bf 100644 --- a/core/src/main/java/org/apache/carbondata/core/service/PathService.java +++ b/core/src/main/java/org/apache/carbondata/core/service/PathService.java @@ -18,7 +18,7 @@ */ package org.apache.carbondata.core.service; -import org.apache.carbondata.core.CarbonTableIdentifier; +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.path.CarbonTablePath; /** http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java b/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java index d9a6d19..114cb50 100644 --- a/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java @@ -18,8 +18,8 @@ */ package org.apache.carbondata.core.service.impl; -import org.apache.carbondata.core.CarbonTableIdentifier; -import org.apache.carbondata.core.ColumnIdentifier; +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader; import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl; import org.apache.carbondata.core.reader.CarbonDictionaryReader; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java b/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java index 9cd4b41..29b1853 100644 --- a/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java @@ -18,7 +18,7 @@ */ package org.apache.carbondata.core.service.impl; -import org.apache.carbondata.core.CarbonTableIdentifier; +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.path.CarbonStorePath; import org.apache.carbondata.core.path.CarbonTablePath; import org.apache.carbondata.core.service.PathService; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java index 317d44a..59e8c25 100644 --- a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java @@ -24,7 +24,7 @@ import java.util.*; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.path.CarbonStorePath; import org.apache.carbondata.core.path.CarbonTablePath; @@ -39,7 +39,7 @@ import org.apache.carbondata.core.updatestatus.SegmentStatusManager; import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager; import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.CarbonUtil; -import org.apache.carbondata.locks.ICarbonLock; +import org.apache.carbondata.core.updatestatus.locks.ICarbonLock; /** @@ -121,12 +121,6 @@ public class CarbonUpdateUtil { lockStatus = updateLock.lockWithRetries(); if (lockStatus) { - AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier(); - - CarbonTablePath carbonTablePath = CarbonStorePath - .getCarbonTablePath(absoluteTableIdentifier.getStorePath(), - absoluteTableIdentifier.getCarbonTableIdentifier()); - // read the existing file if present and update the same. SegmentUpdateDetails[] oldDetails = segmentUpdateStatusManager .getUpdateStatusDetails(); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java index 2015663..f7c2a2d 100644 --- a/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java +++ b/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java @@ -19,7 +19,7 @@ package org.apache.carbondata.core.update.data; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.datastore.DataRefNode; import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java index 02c0860..e01ba6d 100644 --- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java @@ -26,8 +26,8 @@ import java.util.List; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.AbsoluteTableIdentifier; -import org.apache.carbondata.core.CarbonTableIdentifier; +import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.path.CarbonStorePath; import org.apache.carbondata.core.path.CarbonTablePath; import org.apache.carbondata.core.constants.CarbonCommonConstants; @@ -35,13 +35,13 @@ import org.apache.carbondata.core.datastorage.impl.FileFactory; import org.apache.carbondata.core.load.LoadMetadataDetails; import org.apache.carbondata.core.update.CarbonUpdateUtil; import org.apache.carbondata.core.util.CarbonUtil; -import org.apache.carbondata.fileoperations.AtomicFileOperations; -import org.apache.carbondata.fileoperations.AtomicFileOperationsImpl; -import org.apache.carbondata.fileoperations.FileWriteOperation; -import org.apache.carbondata.locks.CarbonLockFactory; -import org.apache.carbondata.locks.CarbonLockUtil; -import org.apache.carbondata.locks.ICarbonLock; -import org.apache.carbondata.locks.LockUsage; +import org.apache.carbondata.core.atomic.AtomicFileOperations; +import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl; +import org.apache.carbondata.core.atomic.FileWriteOperation; +import org.apache.carbondata.core.updatestatus.locks.CarbonLockFactory; +import org.apache.carbondata.core.updatestatus.locks.CarbonLockUtil; +import org.apache.carbondata.core.updatestatus.locks.ICarbonLock; +import org.apache.carbondata.core.updatestatus.locks.LockUsage; import com.google.gson.Gson; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java index c283012..4e44661 100644 --- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java @@ -25,7 +25,7 @@ import java.util.Map; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.path.CarbonStorePath; import org.apache.carbondata.core.path.CarbonTablePath; import org.apache.carbondata.core.constants.CarbonCommonConstants; @@ -39,12 +39,12 @@ import org.apache.carbondata.core.update.SegmentUpdateDetails; import org.apache.carbondata.core.update.TupleIdEnum; import org.apache.carbondata.core.update.data.UpdateVO; import org.apache.carbondata.core.util.CarbonUtil; -import org.apache.carbondata.fileoperations.AtomicFileOperations; -import org.apache.carbondata.fileoperations.AtomicFileOperationsImpl; -import org.apache.carbondata.fileoperations.FileWriteOperation; -import org.apache.carbondata.locks.CarbonLockFactory; -import org.apache.carbondata.locks.ICarbonLock; -import org.apache.carbondata.locks.LockUsage; +import org.apache.carbondata.core.atomic.AtomicFileOperations; +import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl; +import org.apache.carbondata.core.atomic.FileWriteOperation; +import org.apache.carbondata.core.updatestatus.locks.CarbonLockFactory; +import org.apache.carbondata.core.updatestatus.locks.ICarbonLock; +import org.apache.carbondata.core.updatestatus.locks.LockUsage; import com.google.gson.Gson; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/AbstractCarbonLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/AbstractCarbonLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/AbstractCarbonLock.java new file mode 100644 index 0000000..35f5aee --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/AbstractCarbonLock.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.carbondata.core.updatestatus.locks; + +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.util.CarbonProperties; + +/** + * This is the abstract class of the lock implementations.This handles the + * retrying part of the locking. + */ +abstract class AbstractCarbonLock implements ICarbonLock { + private int retryCount; + + private int retryTimeout; + + public abstract boolean lock(); + + /** + * API for enabling the locking of file with retries. + */ + public boolean lockWithRetries() { + try { + for (int i = 0; i < retryCount; i++) { + if (lock()) { + return true; + } else { + Thread.sleep(retryTimeout * 1000L); + } + } + } catch (InterruptedException e) { + return false; + } + return false; + } + + /** + * Initializes the retry count and retry timeout. + * This will determine how many times to retry to acquire lock and the retry timeout. + */ + protected void initRetry() { + String retries = CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK); + try { + retryCount = Integer.parseInt(retries); + } catch (NumberFormatException e) { + retryCount = CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK_DEFAULT; + } + + String maxTimeout = CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK); + try { + retryTimeout = Integer.parseInt(maxTimeout); + } catch (NumberFormatException e) { + retryTimeout = CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK_DEFAULT; + } + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java new file mode 100644 index 0000000..0828ada --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.updatestatus.locks; + +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.util.CarbonProperties; + +/** + * This class is a Lock factory class which is used to provide lock objects. + * Using this lock object client can request the lock and unlock. + */ +public class CarbonLockFactory { + + /** + * lockTypeConfigured to check if zookeeper feature is enabled or not for carbon. + */ + private static String lockTypeConfigured; + + static { + CarbonLockFactory.getLockTypeConfigured(); + } + + /** + * This method will determine the lock type. + * + * @param tableIdentifier + * @param lockFile + * @return + */ + public static ICarbonLock getCarbonLockObj(CarbonTableIdentifier tableIdentifier, + String lockFile) { + switch (lockTypeConfigured.toUpperCase()) { + case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL: + return new LocalFileLock(tableIdentifier, lockFile); + + case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER: + return new ZooKeeperLocking(tableIdentifier, lockFile); + + case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS: + return new HdfsFileLock(tableIdentifier, lockFile); + + default: + throw new UnsupportedOperationException("Not supported the lock type"); + } + } + + /** + * + * @param locFileLocation + * @param lockFile + * @return carbon lock + */ + public static ICarbonLock getCarbonLockObj(String locFileLocation, String lockFile) { + switch (lockTypeConfigured.toUpperCase()) { + case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL: + return new LocalFileLock(locFileLocation, lockFile); + + case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER: + return new ZooKeeperLocking(locFileLocation, lockFile); + + case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS: + return new HdfsFileLock(locFileLocation, lockFile); + + default: + throw new UnsupportedOperationException("Not supported the lock type"); + } + } + + /** + * This method will set the zookeeper status whether zookeeper to be used for locking or not. + */ + private static void getLockTypeConfigured() { + lockTypeConfigured = CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockUtil.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockUtil.java new file mode 100644 index 0000000..b73bedd --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockUtil.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.updatestatus.locks; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; + +/** + * This class contains all carbon lock utilities + */ +public class CarbonLockUtil { + + private static final LogService LOGGER = + LogServiceFactory.getLogService(CarbonLockUtil.class.getName()); + + /** + * unlocks given file + * + * @param carbonLock + */ + public static void fileUnlock(ICarbonLock carbonLock, String locktype) { + if (carbonLock.unlock()) { + if (locktype.equals(LockUsage.METADATA_LOCK)) { + LOGGER.info("Metadata lock has been successfully released"); + } else if (locktype.equals(LockUsage.TABLE_STATUS_LOCK)) { + LOGGER.info("Table status lock has been successfully released"); + } + else if (locktype.equals(LockUsage.CLEAN_FILES_LOCK)) { + LOGGER.info("Clean files lock has been successfully released"); + } + else if (locktype.equals(LockUsage.DELETE_SEGMENT_LOCK)) { + LOGGER.info("Delete segments lock has been successfully released"); + } + } else { + if (locktype.equals(LockUsage.METADATA_LOCK)) { + LOGGER.error("Not able to release the metadata lock"); + } else if (locktype.equals(LockUsage.TABLE_STATUS_LOCK)) { + LOGGER.error("Not able to release the table status lock"); + } + else if (locktype.equals(LockUsage.CLEAN_FILES_LOCK)) { + LOGGER.info("Not able to release the clean files lock"); + } + else if (locktype.equals(LockUsage.DELETE_SEGMENT_LOCK)) { + LOGGER.info("Not able to release the delete segments lock"); + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java new file mode 100644 index 0000000..1d1de81 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.updatestatus.locks; + +import java.io.DataOutputStream; +import java.io.IOException; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; +import org.apache.carbondata.core.datastorage.impl.FileFactory; +import org.apache.carbondata.core.util.CarbonProperties; + +import org.apache.hadoop.conf.Configuration; + +/** + * This class is used to handle the HDFS File locking. + * This is acheived using the concept of acquiring the data out stream using Append option. + */ +class HdfsFileLock extends AbstractCarbonLock { + + private static final LogService LOGGER = + LogServiceFactory.getLogService(HdfsFileLock.class.getName()); + /** + * location hdfs file location + */ + private String location; + + private DataOutputStream dataOutputStream; + + public static String tmpPath; + + static { + Configuration conf = new Configuration(true); + String hdfsPath = conf.get(CarbonCommonConstants.FS_DEFAULT_FS); + // By default, we put the hdfs lock meta file for one table inside this table's store folder. + // If can not get the STORE_LOCATION, then use hadoop.tmp.dir . + tmpPath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION, + System.getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION)); + if (!tmpPath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)) { + tmpPath = hdfsPath + tmpPath; + } + } + + /** + * @param lockFileLocation + * @param lockFile + */ + public HdfsFileLock(String lockFileLocation, String lockFile) { + this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation + + CarbonCommonConstants.FILE_SEPARATOR + lockFile; + LOGGER.info("HDFS lock path:"+this.location); + initRetry(); + } + + /** + * @param tableIdentifier + * @param lockFile + */ + public HdfsFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) { + this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier + .getTableName(), lockFile); + } + + /* (non-Javadoc) + * @see org.apache.carbondata.core.locks.ICarbonLock#lock() + */ + @Override public boolean lock() { + try { + if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) { + FileFactory.createNewLockFile(location, FileFactory.getFileType(location)); + } + dataOutputStream = + FileFactory.getDataOutputStreamUsingAppend(location, FileFactory.getFileType(location)); + + return true; + + } catch (IOException e) { + return false; + } + } + + /* (non-Javadoc) + * @see org.apache.carbondata.core.locks.ICarbonLock#unlock() + */ + @Override + public boolean unlock() { + if (null != dataOutputStream) { + try { + dataOutputStream.close(); + } catch (IOException e) { + return false; + } finally { + CarbonFile carbonFile = + FileFactory.getCarbonFile(location, FileFactory.getFileType(location)); + if (carbonFile.exists()) { + if (carbonFile.delete()) { + LOGGER.info("Deleted the lock file " + location); + } else { + LOGGER.error("Not able to delete the lock file " + location); + } + } else { + LOGGER.error("Not able to delete the lock file because " + + "it is not existed in location " + location); + } + } + } + return true; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ICarbonLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ICarbonLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ICarbonLock.java new file mode 100644 index 0000000..e269c86 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ICarbonLock.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.updatestatus.locks; + +/** + * Carbon Lock Interface which handles the locking and unlocking. + */ +public interface ICarbonLock { + + /** + * Does the unlocking of the acquired lock. + * + * @return + */ + boolean unlock(); + + /** + * This will acquire the lock and if it doesnt get then it will retry after the confiured time. + * + * @return + */ + boolean lockWithRetries(); + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java new file mode 100644 index 0000000..cdee312 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.updatestatus.locks; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastorage.impl.FileFactory; + +/** + * This class handles the file locking in the local file system. + * This will be handled using the file channel lock API. + */ +class LocalFileLock extends AbstractCarbonLock { + /** + * location is the location of the lock file. + */ + private String location; + + /** + * fileOutputStream of the local lock file + */ + private FileOutputStream fileOutputStream; + + /** + * channel is the FileChannel of the lock file. + */ + private FileChannel channel; + + /** + * fileLock NIO FileLock Object + */ + private FileLock fileLock; + + /** + * lock file + */ + private String lockFile; + + public static final String tmpPath; + + private String lockFilePath; + + /** + * LOGGER for logging the messages. + */ + private static final LogService LOGGER = + LogServiceFactory.getLogService(LocalFileLock.class.getName()); + + static { + tmpPath = System.getProperty("java.io.tmpdir"); + } + + /** + * @param lockFileLocation + * @param lockFile + */ + public LocalFileLock(String lockFileLocation, String lockFile) { + this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation; + this.lockFile = lockFile; + initRetry(); + } + + /** + * @param tableIdentifier + * @param lockFile + */ + public LocalFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) { + this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier + .getTableName(), lockFile); + initRetry(); + } + + /** + * Lock API for locking of the file channel of the lock file. + * + * @return + */ + @Override public boolean lock() { + try { + if (!FileFactory.isFileExist(location, FileFactory.getFileType(tmpPath))) { + FileFactory.mkdirs(location, FileFactory.getFileType(tmpPath)); + } + lockFilePath = location + CarbonCommonConstants.FILE_SEPARATOR + + lockFile; + if (!FileFactory.isFileExist(lockFilePath, FileFactory.getFileType(location))) { + FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location)); + } + + fileOutputStream = new FileOutputStream(lockFilePath); + channel = fileOutputStream.getChannel(); + try { + fileLock = channel.tryLock(); + } catch (OverlappingFileLockException e) { + return false; + } + if (null != fileLock) { + return true; + } else { + return false; + } + } catch (IOException e) { + return false; + } + + } + + /** + * Unlock API for unlocking of the acquired lock. + * + * @return + */ + @Override public boolean unlock() { + boolean status; + try { + if (null != fileLock) { + fileLock.release(); + } + status = true; + } catch (IOException e) { + status = false; + } finally { + if (null != fileOutputStream) { + try { + fileOutputStream.close(); + // deleting the lock file after releasing the lock. + if (FileFactory.getCarbonFile(lockFilePath, FileFactory.getFileType(lockFilePath)) + .delete()) { + LOGGER.info("Successfully deleted the lock file " + lockFilePath); + } else { + LOGGER.error("Not able to delete the lock file " + lockFilePath); + } + } catch (IOException e) { + LOGGER.error(e.getMessage()); + } + } + } + return status; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LockUsage.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LockUsage.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LockUsage.java new file mode 100644 index 0000000..5a2ffe7 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LockUsage.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.updatestatus.locks; + +/** + * This enum is used to define the usecase of the lock. + * Each enum value is one specific lock case. + */ +public class LockUsage { + public static final String LOCK = ".lock"; + public static final String METADATA_LOCK = "meta.lock"; + public static final String COMPACTION_LOCK = "compaction.lock"; + public static final String SYSTEMLEVEL_COMPACTION_LOCK = "system_level_compaction.lock"; + public static final String TABLE_STATUS_LOCK = "tablestatus.lock"; + public static final String TABLE_UPDATE_STATUS_LOCK = "tableupdatestatus.lock"; + public static final String DELETE_SEGMENT_LOCK = "delete_segment.lock"; + public static final String CLEAN_FILES_LOCK = "clean_files.lock"; + public static final String DROP_TABLE_LOCK = "droptable.lock"; + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java new file mode 100644 index 0000000..2d742cf --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.updatestatus.locks; + +import java.io.File; +import java.util.Collections; +import java.util.List; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.util.CarbonProperties; + +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.ZooKeeper; + +/** + * For Handling the zookeeper locking implementation + */ +class ZooKeeperLocking extends AbstractCarbonLock { + + private static final LogService LOGGER = + LogServiceFactory.getLogService(ZooKeeperLocking.class.getName()); + + /** + * zk is the zookeeper client instance + */ + private static ZooKeeper zk; + + /** + * zooKeeperLocation is the location in the zoo keeper file system where the locks will be + * maintained. + */ + private static final String zooKeeperLocation = CarbonCommonConstants.ZOOKEEPER_LOCATION; + + /** + * Unique folder for each table with DatabaseName_TableName + */ + private final String tableIdFolder; + + /** + * lockName is the name of the lock to use. This name should be same for every process that want + * to share the same lock + */ + private String lockName; + + /** + * lockPath is the unique path created for the each instance of the carbon lock. + */ + private String lockPath; + + private String lockTypeFolder; + + public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) { + this(tableIdentifier.getDatabaseName() + File.separator + tableIdentifier.getTableName(), + lockFile); + } + + /** + * @param lockLocation + * @param lockFile + */ + public ZooKeeperLocking(String lockLocation, String lockFile) { + this.lockName = lockFile; + this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation; + + String zooKeeperUrl = + CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ZOOKEEPER_URL); + zk = ZookeeperInit.getInstance(zooKeeperUrl).getZookeeper(); + + this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation + + CarbonCommonConstants.FILE_SEPARATOR + lockFile; + try { + createBaseNode(); + // if exists returns null then path doesnt exist. so creating. + if (null == zk.exists(this.tableIdFolder, true)) { + createRecursivly(this.tableIdFolder); + } + // if exists returns null then path doesnt exist. so creating. + if (null == zk.exists(this.lockTypeFolder, true)) { + zk.create(this.lockTypeFolder, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } + } catch (KeeperException | InterruptedException e) { + LOGGER.error(e, e.getMessage()); + } + initRetry(); + } + + /** + * Creating a znode in which all the znodes (lock files )are maintained. + */ + private void createBaseNode() throws KeeperException, InterruptedException { + if (null == zk.exists(zooKeeperLocation, true)) { + // creating a znode in which all the znodes (lock files )are maintained. + zk.create(zooKeeperLocation, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } + } + + /** + * Create zookeepr node if not exist + * @param path + * @throws KeeperException + * @throws InterruptedException + */ + private void createRecursivly(String path) throws KeeperException, InterruptedException { + try { + if (zk.exists(path, true) == null && path.length() > 0) { + String temp = path.substring(0, path.lastIndexOf(File.separator)); + createRecursivly(temp); + zk.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); + } else { + return; + } + } catch (KeeperException e) { + throw e; + } catch (InterruptedException e) { + throw e; + } + + } + /** + * Handling of the locking mechanism using zoo keeper. + */ + @Override public boolean lock() { + try { + // create the lock file with lockName. + lockPath = + zk.create(this.lockTypeFolder + CarbonCommonConstants.FILE_SEPARATOR + lockName, null, + Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL); + + // get the children present in zooKeeperLocation. + List<String> nodes = zk.getChildren(this.lockTypeFolder, null); + + // sort the childrens + Collections.sort(nodes); + + // here the logic is , for each lock request zookeeper will create a file ending with + // incremental digits. + // so first request will be 00001 next is 00002 and so on. + // if the current request is 00002 and already one previous request(00001) is present then get + // children will give both nodes. + // after the sort we are checking if the lock path is first or not .if it is first then lock + // has been acquired. + + if (lockPath.endsWith(nodes.get(0))) { + return true; + } else { + // if locking failed then deleting the created lock as next time again new lock file will be + // created. + zk.delete(lockPath, -1); + return false; + } + } catch (KeeperException | InterruptedException e) { + LOGGER.error(e, e.getMessage()); + return false; + } + } + + /** + * @return status where lock file is unlocked or not. + */ + @Override public boolean unlock() { + try { + // exists will return null if the path doesn't exists. + if (null != zk.exists(lockPath, true)) { + zk.delete(lockPath, -1); + lockPath = null; + } + } catch (KeeperException | InterruptedException e) { + LOGGER.error(e, e.getMessage()); + return false; + } + return true; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZookeeperInit.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZookeeperInit.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZookeeperInit.java new file mode 100644 index 0000000..391e174 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZookeeperInit.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.updatestatus.locks; + +import java.io.IOException; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; + +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.ZooKeeper; + +/** + * This is a singleton class for initialization of zookeeper client. + */ +public class ZookeeperInit { + + private static final LogService LOGGER = + LogServiceFactory.getLogService(ZookeeperInit.class.getName()); + + private static ZookeeperInit zooKeeperInit; + /** + * zk is the zookeeper client instance + */ + private ZooKeeper zk; + + private ZookeeperInit(String zooKeeperUrl) { + + int sessionTimeOut = 100000; + try { + zk = new ZooKeeper(zooKeeperUrl, sessionTimeOut, new DummyWatcher()); + + } catch (IOException e) { + LOGGER.error(e.getMessage()); + } + + } + + public static ZookeeperInit getInstance(String zooKeeperUrl) { + + if (null == zooKeeperInit) { + synchronized (ZookeeperInit.class) { + if (null == zooKeeperInit) { + LOGGER.info("Initiating Zookeeper client."); + zooKeeperInit = new ZookeeperInit(zooKeeperUrl); + } + } + } + return zooKeeperInit; + + } + + public static ZookeeperInit getInstance() { + return zooKeeperInit; + } + + public ZooKeeper getZookeeper() { + return zk; + } + + private static class DummyWatcher implements Watcher { + public void process(WatchedEvent event) { + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java index 59305b1..fa8410a 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java +++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java @@ -24,7 +24,7 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.List; -import org.apache.carbondata.core.ColumnarFormatVersion; +import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import org.apache.carbondata.core.datastore.block.BlockInfo; import org.apache.carbondata.core.datastore.block.TableBlockInfo; import org.apache.carbondata.core.metadata.blocklet.DataFileFooter; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java index 360fd6f..79d5b7b 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java @@ -30,7 +30,7 @@ import java.util.Set; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.ColumnarFormatVersion; +import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.metadata.index.BlockIndexInfo; import org.apache.carbondata.core.constants.CarbonCommonConstants; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java index 47c9926..95d099a 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java @@ -27,7 +27,7 @@ import java.util.Properties; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.ColumnarFormatVersion; +import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import org.apache.carbondata.core.constants.CarbonCommonConstants; public final class CarbonProperties { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 2a4fb1b..34f1ac9 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -43,7 +43,7 @@ import java.util.Set; import org.apache.carbondata.core.service.CarbonCommonFactory; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.AbsoluteTableIdentifier; +import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.cache.dictionary.Dictionary; import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.datastore.block.AbstractIndex; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java index 4d9e5b8..eadac33 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java @@ -23,7 +23,7 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import org.apache.carbondata.core.ColumnarFormatVersion; +import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import org.apache.carbondata.core.datastore.block.TableBlockInfo; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; import org.apache.carbondata.core.metadata.blocklet.DataFileFooter; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java index 8ca8ddd..0808cea 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java @@ -22,7 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import org.apache.carbondata.core.ColumnarFormatVersion; +import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import org.apache.carbondata.core.datastore.block.TableBlockInfo; import org.apache.carbondata.core.metadata.blocklet.BlockletInfo; import org.apache.carbondata.core.metadata.blocklet.DataFileFooter; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java index 19df747..92a507a 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java +++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java @@ -18,7 +18,7 @@ */ package org.apache.carbondata.core.util; -import org.apache.carbondata.core.ColumnarFormatVersion; +import org.apache.carbondata.core.metadata.ColumnarFormatVersion; /** * Factory class to get the thrift reader object based on version http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java index 9fe894a..e024653 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java @@ -28,8 +28,8 @@ import java.util.List; import org.apache.carbondata.core.service.CarbonCommonFactory; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.CarbonTableIdentifier; -import org.apache.carbondata.core.ColumnIdentifier; +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.path.CarbonTablePath; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java index 9f9d7ab..1d37b5b 100644 --- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java @@ -26,8 +26,8 @@ import java.util.List; import org.apache.carbondata.core.service.CarbonCommonFactory; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.CarbonTableIdentifier; -import org.apache.carbondata.core.ColumnIdentifier; +import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.path.CarbonTablePath; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperations.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperations.java b/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperations.java deleted file mode 100644 index a1c7a2e..0000000 --- a/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperations.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.fileoperations; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; - -public interface AtomicFileOperations { - - DataInputStream openForRead() throws IOException; - - void close() throws IOException; - - DataOutputStream openForWrite(FileWriteOperation operation) throws IOException; -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java b/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java deleted file mode 100644 index 1bb8ba3..0000000 --- a/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.fileoperations; - -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.impl.FileFactory; -import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType; - -public class AtomicFileOperationsImpl implements AtomicFileOperations { - - private String filePath; - - private FileType fileType; - - private String tempWriteFilePath; - - private DataOutputStream dataOutStream; - - public AtomicFileOperationsImpl(String filePath, FileType fileType) { - this.filePath = filePath; - - this.fileType = fileType; - } - - @Override public DataInputStream openForRead() throws IOException { - return FileFactory.getDataInputStream(filePath, fileType); - } - - @Override public DataOutputStream openForWrite(FileWriteOperation operation) throws IOException { - - filePath = filePath.replace("\\", "/"); - - tempWriteFilePath = filePath + CarbonCommonConstants.TEMPWRITEFILEEXTENSION; - - if (FileFactory.isFileExist(tempWriteFilePath, fileType)) { - FileFactory.getCarbonFile(tempWriteFilePath, fileType).delete(); - } - - FileFactory.createNewFile(tempWriteFilePath, fileType); - - dataOutStream = FileFactory.getDataOutputStream(tempWriteFilePath, fileType); - - return dataOutStream; - - } - - /* (non-Javadoc) - * @see com.huawei.unibi.carbon.datastorage.store.fileperations.AtomicFileOperations#close() - */ - @Override public void close() throws IOException { - - if (null != dataOutStream) { - dataOutStream.close(); - - CarbonFile tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType); - - if (!tempFile.renameForce(filePath)) { - throw new IOException("temporary file renaming failed, src=" - + tempFile.getPath() + ", dest=" + filePath); - } - } - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/fileoperations/FileWriteOperation.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/fileoperations/FileWriteOperation.java b/core/src/main/java/org/apache/carbondata/fileoperations/FileWriteOperation.java deleted file mode 100644 index 81502f2..0000000 --- a/core/src/main/java/org/apache/carbondata/fileoperations/FileWriteOperation.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.fileoperations; - -public enum FileWriteOperation { - - APPEND, OVERWRITE -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/AbstractCarbonLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/locks/AbstractCarbonLock.java b/core/src/main/java/org/apache/carbondata/locks/AbstractCarbonLock.java deleted file mode 100644 index 934f447..0000000 --- a/core/src/main/java/org/apache/carbondata/locks/AbstractCarbonLock.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.locks; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.util.CarbonProperties; - -/** - * This is the abstract class of the lock implementations.This handles the - * retrying part of the locking. - */ -public abstract class AbstractCarbonLock implements ICarbonLock { - private int retryCount; - - private int retryTimeout; - - public abstract boolean lock(); - - /** - * API for enabling the locking of file with retries. - */ - public boolean lockWithRetries() { - try { - for (int i = 0; i < retryCount; i++) { - if (lock()) { - return true; - } else { - Thread.sleep(retryTimeout * 1000L); - } - } - } catch (InterruptedException e) { - return false; - } - return false; - } - - /** - * Initializes the retry count and retry timeout. - * This will determine how many times to retry to acquire lock and the retry timeout. - */ - protected void initRetry() { - String retries = CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK); - try { - retryCount = Integer.parseInt(retries); - } catch (NumberFormatException e) { - retryCount = CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK_DEFAULT; - } - - String maxTimeout = CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK); - try { - retryTimeout = Integer.parseInt(maxTimeout); - } catch (NumberFormatException e) { - retryTimeout = CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK_DEFAULT; - } - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/CarbonLockFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/locks/CarbonLockFactory.java b/core/src/main/java/org/apache/carbondata/locks/CarbonLockFactory.java deleted file mode 100644 index d4d60a8..0000000 --- a/core/src/main/java/org/apache/carbondata/locks/CarbonLockFactory.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.locks; - -import org.apache.carbondata.core.CarbonTableIdentifier; -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.util.CarbonProperties; - -/** - * This class is a Lock factory class which is used to provide lock objects. - * Using this lock object client can request the lock and unlock. - */ -public class CarbonLockFactory { - - /** - * lockTypeConfigured to check if zookeeper feature is enabled or not for carbon. - */ - private static String lockTypeConfigured; - - static { - CarbonLockFactory.getLockTypeConfigured(); - } - - /** - * This method will determine the lock type. - * - * @param tableIdentifier - * @param lockFile - * @return - */ - public static ICarbonLock getCarbonLockObj(CarbonTableIdentifier tableIdentifier, - String lockFile) { - switch (lockTypeConfigured.toUpperCase()) { - case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL: - return new LocalFileLock(tableIdentifier, lockFile); - - case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER: - return new ZooKeeperLocking(tableIdentifier, lockFile); - - case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS: - return new HdfsFileLock(tableIdentifier, lockFile); - - default: - throw new UnsupportedOperationException("Not supported the lock type"); - } - } - - /** - * - * @param locFileLocation - * @param lockFile - * @return carbon lock - */ - public static ICarbonLock getCarbonLockObj(String locFileLocation, String lockFile) { - switch (lockTypeConfigured.toUpperCase()) { - case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL: - return new LocalFileLock(locFileLocation, lockFile); - - case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER: - return new ZooKeeperLocking(locFileLocation, lockFile); - - case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS: - return new HdfsFileLock(locFileLocation, lockFile); - - default: - throw new UnsupportedOperationException("Not supported the lock type"); - } - } - - /** - * This method will set the zookeeper status whether zookeeper to be used for locking or not. - */ - private static void getLockTypeConfigured() { - lockTypeConfigured = CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/CarbonLockUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/locks/CarbonLockUtil.java b/core/src/main/java/org/apache/carbondata/locks/CarbonLockUtil.java deleted file mode 100644 index 181e92b..0000000 --- a/core/src/main/java/org/apache/carbondata/locks/CarbonLockUtil.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.locks; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; - -/** - * This class contains all carbon lock utilities - */ -public class CarbonLockUtil { - - private static final LogService LOGGER = - LogServiceFactory.getLogService(CarbonLockUtil.class.getName()); - - /** - * unlocks given file - * - * @param carbonLock - */ - public static void fileUnlock(ICarbonLock carbonLock, String locktype) { - if (carbonLock.unlock()) { - if (locktype.equals(LockUsage.METADATA_LOCK)) { - LOGGER.info("Metadata lock has been successfully released"); - } else if (locktype.equals(LockUsage.TABLE_STATUS_LOCK)) { - LOGGER.info("Table status lock has been successfully released"); - } - else if (locktype.equals(LockUsage.CLEAN_FILES_LOCK)) { - LOGGER.info("Clean files lock has been successfully released"); - } - else if (locktype.equals(LockUsage.DELETE_SEGMENT_LOCK)) { - LOGGER.info("Delete segments lock has been successfully released"); - } - } else { - if (locktype.equals(LockUsage.METADATA_LOCK)) { - LOGGER.error("Not able to release the metadata lock"); - } else if (locktype.equals(LockUsage.TABLE_STATUS_LOCK)) { - LOGGER.error("Not able to release the table status lock"); - } - else if (locktype.equals(LockUsage.CLEAN_FILES_LOCK)) { - LOGGER.info("Not able to release the clean files lock"); - } - else if (locktype.equals(LockUsage.DELETE_SEGMENT_LOCK)) { - LOGGER.info("Not able to release the delete segments lock"); - } - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java deleted file mode 100644 index d61df15..0000000 --- a/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.locks; - -import java.io.DataOutputStream; -import java.io.IOException; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.CarbonTableIdentifier; -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.impl.FileFactory; -import org.apache.carbondata.core.util.CarbonProperties; - -import org.apache.hadoop.conf.Configuration; - -/** - * This class is used to handle the HDFS File locking. - * This is acheived using the concept of acquiring the data out stream using Append option. - */ -public class HdfsFileLock extends AbstractCarbonLock { - - private static final LogService LOGGER = - LogServiceFactory.getLogService(HdfsFileLock.class.getName()); - /** - * location hdfs file location - */ - private String location; - - private DataOutputStream dataOutputStream; - - public static String tmpPath; - - static { - Configuration conf = new Configuration(true); - String hdfsPath = conf.get(CarbonCommonConstants.FS_DEFAULT_FS); - // By default, we put the hdfs lock meta file for one table inside this table's store folder. - // If can not get the STORE_LOCATION, then use hadoop.tmp.dir . - tmpPath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION, - System.getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION)); - if (!tmpPath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)) { - tmpPath = hdfsPath + tmpPath; - } - } - - /** - * @param lockFileLocation - * @param lockFile - */ - public HdfsFileLock(String lockFileLocation, String lockFile) { - this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation - + CarbonCommonConstants.FILE_SEPARATOR + lockFile; - LOGGER.info("HDFS lock path:"+this.location); - initRetry(); - } - - /** - * @param tableIdentifier - * @param lockFile - */ - public HdfsFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) { - this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier - .getTableName(), lockFile); - } - - /* (non-Javadoc) - * @see org.apache.carbondata.core.locks.ICarbonLock#lock() - */ - @Override public boolean lock() { - try { - if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) { - FileFactory.createNewLockFile(location, FileFactory.getFileType(location)); - } - dataOutputStream = - FileFactory.getDataOutputStreamUsingAppend(location, FileFactory.getFileType(location)); - - return true; - - } catch (IOException e) { - return false; - } - } - - /* (non-Javadoc) - * @see org.apache.carbondata.core.locks.ICarbonLock#unlock() - */ - @Override - public boolean unlock() { - if (null != dataOutputStream) { - try { - dataOutputStream.close(); - } catch (IOException e) { - return false; - } finally { - CarbonFile carbonFile = - FileFactory.getCarbonFile(location, FileFactory.getFileType(location)); - if (carbonFile.exists()) { - if (carbonFile.delete()) { - LOGGER.info("Deleted the lock file " + location); - } else { - LOGGER.error("Not able to delete the lock file " + location); - } - } else { - LOGGER.error("Not able to delete the lock file because " - + "it is not existed in location " + location); - } - } - } - return true; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/ICarbonLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/locks/ICarbonLock.java b/core/src/main/java/org/apache/carbondata/locks/ICarbonLock.java deleted file mode 100644 index fbb2030..0000000 --- a/core/src/main/java/org/apache/carbondata/locks/ICarbonLock.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.carbondata.locks; - -/** - * Carbon Lock Interface which handles the locking and unlocking. - */ -public interface ICarbonLock { - - /** - * Does the unlocking of the acquired lock. - * - * @return - */ - boolean unlock(); - - /** - * This will acquire the lock and if it doesnt get then it will retry after the confiured time. - * - * @return - */ - boolean lockWithRetries(); - -}
