MultiClient Load is failing
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ef583afe Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ef583afe Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ef583afe Branch: refs/heads/branch-1.1 Commit: ef583afe6968d1222553810bdc2251cef16f016c Parents: 735e477 Author: nareshpr <prnaresh.nar...@gmail.com> Authored: Tue May 30 14:48:10 2017 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Thu Jun 15 13:12:41 2017 +0530 ---------------------------------------------------------------------- .../org/apache/carbondata/spark/load/CarbonLoaderUtil.java | 9 ++------- .../apache/carbondata/spark/rdd/DataManagementFunc.scala | 4 ++-- .../apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala | 2 +- .../apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala | 2 +- .../spark/sql/execution/command/carbonTableSchema.scala | 6 +++--- .../scala/org/apache/spark/sql/hive/CarbonMetastore.scala | 2 -- 6 files changed, 9 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef583afe/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java index a4f15d2..54e12f3 100644 --- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java +++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java @@ -58,7 +58,6 @@ import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl; import org.apache.carbondata.core.fileoperations.FileWriteOperation; import org.apache.carbondata.core.locks.ICarbonLock; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; -import org.apache.carbondata.core.metadata.CarbonMetadata; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.metadata.datatype.DataType; @@ -783,14 +782,10 @@ public final class CarbonLoaderUtil { * This method will get the store location for the given path, segment id and partition id * * @param carbonStorePath - * @param dbName - * @param tableName * @param segmentId */ - public static void checkAndCreateCarbonDataLocation(String carbonStorePath, String dbName, - String tableName, String segmentId) { - CarbonTable carbonTable = CarbonMetadata.getInstance() - .getCarbonTable(dbName + CarbonCommonConstants.UNDERSCORE + tableName); + public static void checkAndCreateCarbonDataLocation(String carbonStorePath, + String segmentId, CarbonTable carbonTable) { CarbonTableIdentifier carbonTableIdentifier = carbonTable.getCarbonTableIdentifier(); CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTableIdentifier); http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef583afe/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala index 8039d24..1790ea2 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.execution.command.{CompactionCallableModel, Compacti import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage} -import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier} +import org.apache.carbondata.core.metadata.CarbonTableIdentifier import org.apache.carbondata.core.metadata.schema.table.CarbonTable import org.apache.carbondata.core.mutate.CarbonUpdateUtil import org.apache.carbondata.core.statusmanager.{LoadMetadataDetails, SegmentStatusManager} @@ -59,7 +59,7 @@ object DataManagementFunc { val sc = sqlContext // Delete the records based on data - val table = CarbonMetadata.getInstance.getCarbonTable(databaseName + "_" + tableName) + val table = schema.getCarbonTable val loadMetadataDetailsArray = SegmentStatusManager.readLoadMetadata(table.getMetaDataFilepath).toList val resultMap = new CarbonDeleteLoadByDateRDD( http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef583afe/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala index f159c61..2922365 100644 --- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala +++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala @@ -734,7 +734,7 @@ object CarbonDataRDDFactory { if (!updateModel.isDefined) { CarbonLoaderUtil.checkAndCreateCarbonDataLocation(storePath, - carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName, currentLoadCount.toString) + currentLoadCount.toString, carbonTable) } var loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS var errorMessage: String = "DataLoad failure" http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef583afe/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala index bbdbe4f..b4720a9 100644 --- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala +++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala @@ -764,7 +764,7 @@ object CarbonDataRDDFactory { if (!updateModel.isDefined) { CarbonLoaderUtil.checkAndCreateCarbonDataLocation(storePath, - carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName, currentLoadCount.toString) + currentLoadCount.toString, carbonTable) } var loadStatus = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS var errorMessage: String = "DataLoad failure" http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef583afe/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala index 5dd6832..8818c6b 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala @@ -41,7 +41,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.datastore.impl.FileFactory import org.apache.carbondata.core.dictionary.server.DictionaryServer import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage} -import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier} +import org.apache.carbondata.core.metadata.CarbonTableIdentifier import org.apache.carbondata.core.metadata.encoder.Encoding import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo} import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension @@ -89,7 +89,7 @@ case class AlterTableCompaction(alterTableModel: AlterTableModel) extends Runnab if (relation == null) { sys.error(s"Table $databaseName.$tableName does not exist") } - if (null == CarbonMetadata.getInstance.getCarbonTable(databaseName + "_" + tableName)) { + if (null == relation.tableMeta.carbonTable) { LOGGER.error(s"alter table failed. table not found: $databaseName.$tableName") sys.error(s"alter table failed. table not found: $databaseName.$tableName") } @@ -352,7 +352,7 @@ case class LoadTable( if (relation == null) { sys.error(s"Table $dbName.$tableName does not exist") } - if (null == CarbonMetadata.getInstance.getCarbonTable(dbName + "_" + tableName)) { + if (null == relation.tableMeta.carbonTable) { LOGGER.error(s"Data loading failed. table not found: $dbName.$tableName") LOGGER.audit(s"Data loading failed. table not found: $dbName.$tableName") sys.error(s"Data loading failed. table not found: $dbName.$tableName") http://git-wip-us.apache.org/repos/asf/carbondata/blob/ef583afe/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala index 1f5736e..954801a 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala @@ -40,7 +40,6 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFile import org.apache.carbondata.core.datastore.impl.FileFactory import org.apache.carbondata.core.datastore.impl.FileFactory.FileType import org.apache.carbondata.core.fileoperations.FileWriteOperation -import org.apache.carbondata.core.locks.ZookeeperInit import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier} import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl import org.apache.carbondata.core.metadata.datatype.DataType.DECIMAL @@ -529,7 +528,6 @@ class CarbonMetastore(conf: RuntimeConfig, val storePath: String) { case Some(tableMeta) => metadata.tablesMeta -= tableMeta CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName) - CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName) updateSchemasUpdatedTime(touchSchemaFileSystemTime(dbName, tableName)) case None => LOGGER.info(s"Metadata does not contain entry for table $tableName in database $dbName")