[GitHub] carbondata pull request #2106: [CARBONDATA-2291] Added datamap status and re...
Github user asfgit closed the pull request at: https://github.com/apache/carbondata/pull/2106 ---
[GitHub] carbondata pull request #2106: [CARBONDATA-2291] Added datamap status and re...
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2106#discussion_r178210972
--- Diff:
integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
---
@@ -185,6 +185,16 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser
{
CarbonDataMapShowCommand(tableIdent)
}
+ /**
+ * The syntax of show datamap is used to show datamaps on the table
+ * REFRESH DATAMAP datamapname ON TABLE tableName
--- End diff --
ok
---
[GitHub] carbondata pull request #2106: [CARBONDATA-2291] Added datamap status and re...
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2106#discussion_r178210981
--- Diff:
core/src/main/java/org/apache/carbondata/core/datamap/status/DiskBasedDataMapStatusProvider.java
---
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datamap.status;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.fileoperations.FileWriteOperation;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.CarbonLockUtil;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import com.google.gson.Gson;
+
+public class DiskBasedDataMapStatusProvider implements
DataMapStatusStorageProvider {
+
+ private static final LogService LOG =
+
LogServiceFactory.getLogService(DiskBasedDataMapStatusProvider.class.getName());
+
+ private static final String DATAMAP_STATUS_FILE = "datamapstatus";
+
+ @Override public DataMapStatusDetail[] getDataMapStatusDetails() throws
IOException {
+String statusPath =
CarbonProperties.getInstance().getSystemFolderLocation()
++ CarbonCommonConstants.FILE_SEPARATOR + DATAMAP_STATUS_FILE;
+Gson gsonObjectToRead = new Gson();
+DataInputStream dataInputStream = null;
+BufferedReader buffReader = null;
+InputStreamReader inStream = null;
+DataMapStatusDetail[] dataMapStatusDetails;
+try {
+ if (!FileFactory.isFileExist(statusPath)) {
+return new DataMapStatusDetail[0];
+ }
+ dataInputStream =
+ FileFactory.getDataInputStream(statusPath,
FileFactory.getFileType(statusPath));
+ inStream = new InputStreamReader(dataInputStream,
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ buffReader = new BufferedReader(inStream);
+ dataMapStatusDetails = gsonObjectToRead.fromJson(buffReader,
DataMapStatusDetail[].class);
+} catch (IOException e) {
+ LOG.error(e, "Failed to read datamap status");
+ throw e;
+} finally {
+ CarbonUtil.closeStreams(buffReader, inStream, dataInputStream);
+}
+
+// if dataMapStatusDetails is null, return empty array
+if (null == dataMapStatusDetails) {
+ return new DataMapStatusDetail[0];
+}
+
+return dataMapStatusDetails;
+ }
+
+ @Override
+ public void updateDataMapStatus(List dataMapSchemas,
DataMapStatus dataMapStatus)
+ throws IOException {
+ICarbonLock carbonTableStatusLock = getDataMapStatusLock();
--- End diff --
ok
---
[GitHub] carbondata pull request #2106: [CARBONDATA-2291] Added datamap status and re...
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2106#discussion_r178210924
--- Diff:
core/src/main/java/org/apache/carbondata/core/datamap/status/DiskBasedDataMapStatusProvider.java
---
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datamap.status;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.fileoperations.FileWriteOperation;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.CarbonLockUtil;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import com.google.gson.Gson;
+
+public class DiskBasedDataMapStatusProvider implements
DataMapStatusStorageProvider {
--- End diff --
ok
---
[GitHub] carbondata pull request #2106: [CARBONDATA-2291] Added datamap status and re...
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2106#discussion_r178210938
--- Diff:
core/src/main/java/org/apache/carbondata/core/datamap/status/DiskBasedDataMapStatusProvider.java
---
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datamap.status;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.fileoperations.FileWriteOperation;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.CarbonLockUtil;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import com.google.gson.Gson;
+
+public class DiskBasedDataMapStatusProvider implements
DataMapStatusStorageProvider {
+
+ private static final LogService LOG =
+
LogServiceFactory.getLogService(DiskBasedDataMapStatusProvider.class.getName());
+
+ private static final String DATAMAP_STATUS_FILE = "datamapstatus";
+
+ @Override public DataMapStatusDetail[] getDataMapStatusDetails() throws
IOException {
--- End diff --
ok
---
[GitHub] carbondata pull request #2106: [CARBONDATA-2291] Added datamap status and re...
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2106#discussion_r177983852
--- Diff:
core/src/main/java/org/apache/carbondata/core/datamap/status/DiskBasedDataMapStatusProvider.java
---
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datamap.status;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.fileoperations.FileWriteOperation;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.CarbonLockUtil;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import com.google.gson.Gson;
+
+public class DiskBasedDataMapStatusProvider implements
DataMapStatusStorageProvider {
+
+ private static final LogService LOG =
+
LogServiceFactory.getLogService(DiskBasedDataMapStatusProvider.class.getName());
+
+ private static final String DATAMAP_STATUS_FILE = "datamapstatus";
+
+ @Override public DataMapStatusDetail[] getDataMapStatusDetails() throws
IOException {
+String statusPath =
CarbonProperties.getInstance().getSystemFolderLocation()
++ CarbonCommonConstants.FILE_SEPARATOR + DATAMAP_STATUS_FILE;
+Gson gsonObjectToRead = new Gson();
+DataInputStream dataInputStream = null;
+BufferedReader buffReader = null;
+InputStreamReader inStream = null;
+DataMapStatusDetail[] dataMapStatusDetails;
+try {
+ if (!FileFactory.isFileExist(statusPath)) {
+return new DataMapStatusDetail[0];
+ }
+ dataInputStream =
+ FileFactory.getDataInputStream(statusPath,
FileFactory.getFileType(statusPath));
+ inStream = new InputStreamReader(dataInputStream,
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ buffReader = new BufferedReader(inStream);
+ dataMapStatusDetails = gsonObjectToRead.fromJson(buffReader,
DataMapStatusDetail[].class);
+} catch (IOException e) {
+ LOG.error(e, "Failed to read datamap status");
+ throw e;
+} finally {
+ CarbonUtil.closeStreams(buffReader, inStream, dataInputStream);
+}
+
+// if dataMapStatusDetails is null, return empty array
+if (null == dataMapStatusDetails) {
+ return new DataMapStatusDetail[0];
+}
+
+return dataMapStatusDetails;
+ }
+
+ @Override
+ public void updateDataMapStatus(List dataMapSchemas,
DataMapStatus dataMapStatus)
+ throws IOException {
+ICarbonLock carbonTableStatusLock = getDataMapStatusLock();
--- End diff --
add some comment to tell it is going to overwrite the status file
---
[GitHub] carbondata pull request #2106: [CARBONDATA-2291] Added datamap status and re...
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2106#discussion_r177982964
--- Diff:
integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSpark2SqlParser.scala
---
@@ -185,6 +185,16 @@ class CarbonSpark2SqlParser extends CarbonDDLSqlParser
{
CarbonDataMapShowCommand(tableIdent)
}
+ /**
+ * The syntax of show datamap is used to show datamaps on the table
+ * REFRESH DATAMAP datamapname ON TABLE tableName
--- End diff --
ON TABLE is optional
---
[GitHub] carbondata pull request #2106: [CARBONDATA-2291] Added datamap status and re...
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2106#discussion_r177982171
--- Diff:
integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDataMapRefreshCommand.scala
---
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.datamap
+
+import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.execution.command.DataCommand
+
+import org.apache.carbondata.core.datamap.DataMapStoreManager
+import org.apache.carbondata.core.datamap.status.DataMapStatusManager
+import org.apache.carbondata.datamap.DataMapManager
+
+/**
+ * Refresh the datamaps while sync with main table data.
+ * @param tableIdentifier
--- End diff --
remove it
---
[GitHub] carbondata pull request #2106: [CARBONDATA-2291] Added datamap status and re...
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2106#discussion_r177981285
--- Diff:
core/src/main/java/org/apache/carbondata/core/datamap/status/DiskBasedDataMapStatusProvider.java
---
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datamap.status;
+
+import java.io.*;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
+import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.fileoperations.FileWriteOperation;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.CarbonLockUtil;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import com.google.gson.Gson;
+
+public class DiskBasedDataMapStatusProvider implements
DataMapStatusStorageProvider {
--- End diff --
add comment for this class
---
