fixed issue with meta lock not getting deleted for rename table
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7ae1cd8f Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7ae1cd8f Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7ae1cd8f Branch: refs/heads/12-dev Commit: 7ae1cd8fe77a391cb5f5addec4e19fbff5b06820 Parents: 9554971 Author: kunal642 <kunal.kap...@knoldus.in> Authored: Fri Mar 31 16:03:17 2017 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Thu Apr 6 15:11:22 2017 +0530 ---------------------------------------------------------------------- .../core/locks/AbstractCarbonLock.java | 9 +++++ .../carbondata/core/locks/HdfsFileLock.java | 36 +++++++++----------- .../carbondata/core/locks/ICarbonLock.java | 8 +++++ .../carbondata/core/locks/LocalFileLock.java | 5 ++- .../execution/command/AlterTableCommands.scala | 15 ++++++-- .../org/apache/spark/util/AlterTableUtil.scala | 1 + .../AlterTableValidationTestCase.scala | 8 +++++ 7 files changed, 60 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java b/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java index eeb04d5..e927a7e 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java @@ -18,6 +18,7 @@ package org.apache.carbondata.core.locks; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.util.CarbonProperties; /** @@ -72,4 +73,12 @@ public abstract class AbstractCarbonLock implements ICarbonLock { } + public boolean releaseLockManually(String lockFile) { + try { + return FileFactory.deleteFile(lockFile, FileFactory.getFileType(lockFile)); + } catch (Exception e) { + return false; + } + } + } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java index aed73b3..985ced1 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java @@ -23,6 +23,7 @@ import java.io.IOException; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.filesystem.CarbonFile; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.util.CarbonProperties; @@ -102,34 +103,31 @@ public class HdfsFileLock extends AbstractCarbonLock { */ @Override public boolean unlock() { + boolean status = false; if (null != dataOutputStream) { try { dataOutputStream.close(); + status = true; } catch (IOException e) { - try { - if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) { - return true; - } - } catch (IOException e1) { - LOGGER.error("Exception in isFileExist of the lock file " + e1.getMessage()); - } - LOGGER.error("Exception in unlocking of the lock file " + e.getMessage()); - return false; + status = false; } finally { - try { - if (FileFactory.isFileExist(location, FileFactory.getFileType(location))) { - if (FileFactory.getCarbonFile(location, FileFactory.getFileType(location)).delete()) { - LOGGER.info("Deleted the lock file " + location); - } else { - LOGGER.error("Not able to delete the lock file " + location); - } + CarbonFile carbonFile = + FileFactory.getCarbonFile(location, FileFactory.getFileType(location)); + if (carbonFile.exists()) { + if (carbonFile.delete()) { + LOGGER.info("Deleted the lock file " + location); + } else { + LOGGER.error("Not able to delete the lock file " + location); + status = false; } - } catch (IOException e) { - LOGGER.error("Exception in isFileExist of the lock file " + e.getMessage()); + } else { + LOGGER.error("Not able to delete the lock file because " + + "it is not existed in location " + location); + status = false; } } } - return true; + return status; } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java b/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java index 51c577a..27862fd 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java @@ -36,4 +36,12 @@ public interface ICarbonLock { */ boolean lockWithRetries(); + /** + * This method will delete the lock file at the specified location. + * + * @param lockFile The path of the lock file. + * @return True if the lock file is deleted, false otherwise. + */ + boolean releaseLockManually(String lockFile); + } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java index 616c9ed..2802127 100644 --- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java +++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java @@ -28,6 +28,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.util.CarbonProperties; /** * This class handles the file locking in the local file system. @@ -70,7 +71,8 @@ public class LocalFileLock extends AbstractCarbonLock { LogServiceFactory.getLogService(LocalFileLock.class.getName()); static { - tmpPath = System.getProperty("java.io.tmpdir"); + tmpPath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION, + System.getProperty("java.io.tmpdir")); } /** @@ -151,6 +153,7 @@ public class LocalFileLock extends AbstractCarbonLock { LOGGER.info("Successfully deleted the lock file " + lockFilePath); } else { LOGGER.error("Not able to delete the lock file " + lockFilePath); + status = false; } } catch (IOException e) { LOGGER.error(e.getMessage()); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala index 93a5912..88ca4af 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala @@ -33,7 +33,7 @@ import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifie import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl import org.apache.carbondata.core.metadata.encoder.Encoding import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn -import org.apache.carbondata.core.util.CarbonUtil +import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil} import org.apache.carbondata.core.util.path.CarbonStorePath import org.apache.carbondata.format.{ColumnSchema, SchemaEvolutionEntry, TableInfo} import org.apache.carbondata.spark.exception.MalformedCarbonCommandException @@ -204,7 +204,18 @@ private[sql] case class AlterTableRenameTable(alterTableRenameModel: AlterTableR if (carbonLock.unlock()) { LOGGER.info("Lock released successfully") } else { - LOGGER.error("Unable to release lock during rename table") + val storeLocation = CarbonProperties.getInstance + .getProperty(CarbonCommonConstants.STORE_LOCATION, + System.getProperty("java.io.tmpdir")) + val lockFilePath = storeLocation + CarbonCommonConstants.FILE_SEPARATOR + + oldDatabaseName + CarbonCommonConstants.FILE_SEPARATOR + newTableName + + CarbonCommonConstants.FILE_SEPARATOR + + LockUsage.METADATA_LOCK + if(carbonLock.releaseLockManually(lockFilePath)) { + LOGGER.info("Lock released successfully") + } else { + LOGGER.error("Unable to release lock during rename table") + } } } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala index 0611e99..243eeb6 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala @@ -30,6 +30,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo} object AlterTableUtil { + def validateTableAndAcquireLock(dbName: String, tableName: String, LOGGER: LogService) (sparkSession: SparkSession): ICarbonLock = { val relation = http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7ae1cd8f/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala index bd34913..914136c 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala @@ -381,11 +381,19 @@ class AlterTableValidationTestCase extends QueryTest with BeforeAndAfterAll { sql("drop database testdb") } + test("test to check if the lock file is successfully deleted") { + sql("create table lock_check(id int, name string) stored by 'carbondata'") + sql("alter table lock_check rename to lock_rename") + assert(!new File(s"${ CarbonCommonConstants.STORE_LOCATION } + /default/lock_rename/meta.lock") + .exists()) + } + override def afterAll { sql("DROP TABLE IF EXISTS restructure") sql("DROP TABLE IF EXISTS restructure_new") sql("DROP TABLE IF EXISTS restructure_test") sql("DROP TABLE IF EXISTS restructure_bad") sql("DROP TABLE IF EXISTS restructure_badnew") + sql("DROP TABLE IF EXISTS lock_rename") } }