[CARBONDATA-2621][BloomDataMap] Lock problem in index datamap Problem The locking for the index Datamap is not correct. The HDFS lock is not working properly, because the lock is getting created the the local filesystem instead of HDFS.
Solution Corrected the system folder path. This closes #2387 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/254e6a53 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/254e6a53 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/254e6a53 Branch: refs/heads/branch-1.4 Commit: 254e6a535f3746bff7c70b357e74882529925372 Parents: 66285ab Author: mohammadshahidkhan <mohdshahidkhan1...@gmail.com> Authored: Wed Jun 20 15:48:17 2018 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Tue Jul 31 00:11:26 2018 +0530 ---------------------------------------------------------------------- .../apache/carbondata/core/util/CarbonProperties.java | 3 +++ .../table/DiskBasedDMSchemaStoraheProviderSuite.java | 2 +- .../carbondata/hadoop/ft/CarbonTableInputFormatTest.java | 3 +++ .../hadoop/ft/CarbonTableOutputFormatTest.java | 2 ++ .../presto/integrationtest/PrestoAllDataTypeTest.scala | 5 +++++ .../datamap/lucene/LuceneFineGrainDataMapSuite.scala | 2 -- .../LuceneFineGrainDataMapWithSearchModeSuite.scala | 6 ------ .../org/apache/spark/sql/test/TestQueryExecutor.scala | 6 ++++++ .../apache/carbondata/sdk/file/AvroCarbonWriterTest.java | 11 +++++++++++ .../apache/carbondata/sdk/file/CSVCarbonWriterTest.java | 11 +++++++++++ .../sdk/file/CSVNonTransactionalCarbonWriterTest.java | 11 +++++++++++ .../org/apache/carbondata/sdk/file/CarbonReaderTest.java | 10 ++++++++++ .../apache/carbondata/store/LocalCarbonStoreTest.java | 11 +++++++++++ 13 files changed, 74 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java index f7ace5e..004eb74 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java @@ -34,6 +34,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.constants.CarbonLoadOptionConstants; import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants; +import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import static org.apache.carbondata.core.constants.CarbonCommonConstants.BLOCKLET_SIZE; import static org.apache.carbondata.core.constants.CarbonCommonConstants.CARBON_CUSTOM_BLOCK_DISTRIBUTION; @@ -1531,6 +1532,8 @@ public final class CarbonProperties { if (systemLocation == null) { systemLocation = getStorePath(); } + systemLocation = CarbonUtil.checkAndAppendFileSystemURIScheme(systemLocation); + systemLocation = FileFactory.getUpdatedFilePath(systemLocation); return systemLocation + CarbonCommonConstants.FILE_SEPARATOR + "_system"; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStoraheProviderSuite.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStoraheProviderSuite.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStoraheProviderSuite.java index 709215a..9af7f8f 100644 --- a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStoraheProviderSuite.java +++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStoraheProviderSuite.java @@ -37,7 +37,7 @@ public class DiskBasedDMSchemaStoraheProviderSuite { @BeforeClass public static void setUp() throws IOException { String path = - new File(DiskBasedDMSchemaStorageProvider.class.getResource("/").getPath() + "../") + new File(DiskBasedDMSchemaStoraheProviderSuite.class.getResource("/").getPath() + "../") .getCanonicalPath().replaceAll("\\\\", "/"); CarbonProperties.getInstance() http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java ---------------------------------------------------------------------- diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java index ea242d1..fb9ff9c 100644 --- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java +++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableInputFormatTest.java @@ -40,6 +40,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.hadoop.CarbonProjection; import org.apache.carbondata.hadoop.api.CarbonTableInputFormat; import org.apache.carbondata.hadoop.testutil.StoreCreator; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; @@ -58,6 +59,8 @@ public class CarbonTableInputFormatTest { static { CarbonProperties.getInstance(). addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, "/tmp/carbon/badrecords"); + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, "/tmp/carbon/"); try { StoreCreator.createCarbonStore(); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java index aa2a500..0d8c38b 100644 --- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java +++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonTableOutputFormatTest.java @@ -51,6 +51,8 @@ public class CarbonTableOutputFormatTest { static { CarbonProperties.getInstance(). addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, "/tmp/carbon/badrecords"); + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, "/tmp/carbon/"); try { carbonLoadModel = StoreCreator.createTableAndLoadModel(); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala index 8793765..6995117 100644 --- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala @@ -24,8 +24,10 @@ import org.apache.hadoop.fs.permission.{FsAction, FsPermission} import org.scalatest.{BeforeAndAfterAll, FunSuiteLike} import org.apache.carbondata.common.logging.LogServiceFactory +import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.datastore.impl.FileFactory import org.apache.carbondata.core.datastore.impl.FileFactory.FileType +import org.apache.carbondata.core.util.CarbonProperties import org.apache.carbondata.presto.server.PrestoServer @@ -37,6 +39,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll { private val rootPath = new File(this.getClass.getResource("/").getPath + "../../../..").getCanonicalPath private val storePath = s"$rootPath/integration/presto/target/store" + private val systemPath = s"$rootPath/integration/presto/target/system" // Table schema: @@ -68,6 +71,8 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll { override def beforeAll: Unit = { import org.apache.carbondata.presto.util.CarbonDataStoreCreator + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, + systemPath) CarbonDataStoreCreator .createCarbonStore(storePath, s"$rootPath/integration/presto/src/test/resources/alldatatype.csv") http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala index b56701b..08e44f7 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala @@ -45,8 +45,6 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll { LuceneFineGrainDataMapSuite.createFile(file2) sql("create database if not exists lucene") CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, - CarbonEnv.getDatabaseLocation("lucene", sqlContext.sparkSession)) .addProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP, "true") sql("use lucene") sql("DROP TABLE IF EXISTS normal_test") http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala index 0ceead8..e6a2a36 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapWithSearchModeSuite.scala @@ -45,9 +45,6 @@ class LuceneFineGrainDataMapWithSearchModeSuite extends QueryTest with BeforeAnd .addProperty(CarbonCommonConstants.CARBON_SEARCH_QUERY_TIMEOUT, "100s") LuceneFineGrainDataMapSuite.createFile(file2, n) sql("create database if not exists lucene") - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, - CarbonEnv.getDatabaseLocation("lucene", sqlContext.sparkSession)) sql("use lucene") sql("DROP TABLE IF EXISTS datamap_test") sql( @@ -305,9 +302,6 @@ class LuceneFineGrainDataMapWithSearchModeSuite extends QueryTest with BeforeAnd sql("DROP TABLE IF EXISTS datamap_test") sql("DROP TABLE IF EXISTS datamap_test5") sql("use default") - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, - CarbonProperties.getStorePath) sqlContext.sparkSession.asInstanceOf[CarbonSession].stopSearchMode() } http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala index 464cca7..d3a20c3 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala @@ -128,6 +128,11 @@ object TestQueryExecutor { } else { s"$target/bad_store" } + val systemFolderPath = if (hdfsUrl.startsWith("hdfs://")) { + s"$hdfsUrl/systemfolder" + System.nanoTime() + } else { + s"$target/systemfolder" + } createDirectory(badStoreLocation) val hiveresultpath = if (hdfsUrl.startsWith("hdfs://")) { @@ -175,6 +180,7 @@ object TestQueryExecutor { (CarbonCommonConstants.DICTIONARY_SERVER_PORT_DEFAULT.toInt + Random.nextInt(100)) + "") .addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "1024") .addProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE, "1024") + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, systemFolderPath) private def lookupQueryExecutor: Class[_] = { ServiceLoader.load(classOf[TestQueryExecutorRegister], Utils.getContextOrSparkClassLoader) http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java index d0128fc..4320edc 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java @@ -27,6 +27,8 @@ import java.util.Map; import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.schema.table.DiskBasedDMSchemaStorageProvider; +import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.avro.generic.GenericData; @@ -44,6 +46,15 @@ public class AvroCarbonWriterTest { @Before public void cleanFile() { + String path = null; + try { + path = new File(AvroCarbonWriterTest.class.getResource("/").getPath() + "../") + .getCanonicalPath().replaceAll("\\\\", "/"); + } catch (IOException e) { + assert (false); + } + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path); assert (TestUtil.cleanMdtFile()); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java index 865097b..78d0240 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVCarbonWriterTest.java @@ -24,6 +24,8 @@ import java.io.IOException; import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.schema.table.DiskBasedDMSchemaStorageProvider; +import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.commons.io.FileUtils; @@ -39,6 +41,15 @@ public class CSVCarbonWriterTest { @Before public void cleanFile() { + String path = null; + try { + path = new File(CSVCarbonWriterTest.class.getResource("/").getPath() + "../") + .getCanonicalPath().replaceAll("\\\\", "/"); + } catch (IOException e) { + assert (false); + } + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path); assert (TestUtil.cleanMdtFile()); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java index 0393077..c1d5f88 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CSVNonTransactionalCarbonWriterTest.java @@ -24,6 +24,8 @@ import java.io.IOException; import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.schema.table.DiskBasedDMSchemaStorageProvider; +import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.commons.io.FileUtils; @@ -38,6 +40,15 @@ import org.junit.Test; public class CSVNonTransactionalCarbonWriterTest { @Before public void cleanFile() { + String path = null; + try { + path = new File(CSVNonTransactionalCarbonWriterTest.class.getResource("/").getPath() + "../") + .getCanonicalPath().replaceAll("\\\\", "/"); + } catch (IOException e) { + assert (false); + } + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path); assert (TestUtil.cleanMdtFile()); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java index 19bf194..279b3b6 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java @@ -28,6 +28,7 @@ import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.metadata.schema.table.DiskBasedDMSchemaStorageProvider; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.LiteralExpression; import org.apache.carbondata.core.scan.expression.conditional.*; @@ -937,6 +938,15 @@ public class CarbonReaderTest extends TestCase { @Override public void setUp() { carbonProperties = CarbonProperties.getInstance(); + String path = null; + try { + path = new File(CarbonReaderTest.class.getResource("/").getPath() + "../") + .getCanonicalPath().replaceAll("\\\\", "/"); + } catch (IOException e) { + assert (false); + } + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path); } private static final LogService LOGGER = http://git-wip-us.apache.org/repos/asf/carbondata/blob/254e6a53/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java b/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java index 2b8c708..0319df9 100644 --- a/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java +++ b/store/sdk/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java @@ -21,9 +21,11 @@ import java.io.File; import java.io.IOException; import java.util.Iterator; +import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.row.CarbonRow; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.datatype.DataTypes; +import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.sdk.file.Field; import org.apache.carbondata.sdk.file.Schema; import org.apache.carbondata.sdk.file.TestUtil; @@ -36,6 +38,15 @@ import org.junit.Test; public class LocalCarbonStoreTest { @Before public void cleanFile() { + String path = null; + try { + path = new File(LocalCarbonStoreTest.class.getResource("/").getPath() + "../") + .getCanonicalPath().replaceAll("\\\\", "/"); + } catch (IOException e) { + assert (false); + } + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_SYSTEM_FOLDER_LOCATION, path); assert (TestUtil.cleanMdtFile()); }
