Repository: carbondata Updated Branches: refs/heads/master 6707db689 -> 6f3b9d3b9
[HOTFIX] change log level for data loading In current data loading, many log meant for debugging purpose is logged as INFO log, in order to reduce the entry of them, In this PR they are changed to DEBUG level. This closes #2911 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6f3b9d3b Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6f3b9d3b Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6f3b9d3b Branch: refs/heads/master Commit: 6f3b9d3b99ca55a6b29e951d89e17bb60eee1f03 Parents: 6707db6 Author: Jacky Li <jacky.li...@qq.com> Authored: Fri Nov 9 14:49:15 2018 +0800 Committer: xuchuanyin <xuchuan...@hust.edu.cn> Committed: Fri Nov 9 17:06:13 2018 +0800 ---------------------------------------------------------------------- .../core/metadata/schema/table/TableInfo.java | 9 +++-- .../apache/carbondata/core/util/CarbonUtil.java | 39 +++++++++++--------- .../loading/AbstractDataLoadProcessorStep.java | 4 +- .../processing/loading/DataLoadExecutor.java | 2 - .../CarbonRowDataWriterProcessorStepImpl.java | 12 +++--- .../steps/DataWriterProcessorStepImpl.java | 12 +++--- .../store/CarbonFactDataHandlerColumnar.java | 35 ++++++++++++------ .../store/writer/AbstractFactDataWriter.java | 2 +- .../writer/v3/CarbonFactDataWriterImplV3.java | 7 +++- .../util/CarbonDataProcessorUtil.java | 1 - 10 files changed, 73 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f3b9d3b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java index b3e9e7e..3e50586 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java @@ -258,9 +258,12 @@ public class TableInfo implements Serializable, Writable { } if (null == tableBlockSize) { tableBlockSize = CarbonCommonConstants.BLOCK_SIZE_DEFAULT_VAL; - LOGGER.info("Table block size not specified for " + getTableUniqueName() - + ". Therefore considering the default value " - + CarbonCommonConstants.BLOCK_SIZE_DEFAULT_VAL + " MB"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Table block size not specified for " + getTableUniqueName() + + ". Therefore considering the default value " + + CarbonCommonConstants.BLOCK_SIZE_DEFAULT_VAL + " MB"); + } } return Integer.parseInt(tableBlockSize); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f3b9d3b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 1840ba0..2fa6260 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -2474,7 +2474,7 @@ public final class CarbonUtil { lockAcquired = carbonLock.lockWithRetries(); } if (lockAcquired) { - LOGGER.info("Acquired lock for table for table status updation"); + LOGGER.debug("Acquired lock for table for table status updation"); String metadataPath = carbonTable.getMetadataPath(); LoadMetadataDetails[] loadMetadataDetails = SegmentStatusManager.readLoadMetadata(metadataPath); @@ -2488,7 +2488,7 @@ public final class CarbonUtil { // If it is old segment, need to calculate data size and index size again if (null == dsize || null == isize) { needUpdate = true; - LOGGER.info("It is an old segment, need calculate data size and index size again"); + LOGGER.debug("It is an old segment, need calculate data size and index size again"); HashMap<String, Long> map = CarbonUtil.getDataSizeAndIndexSize( identifier.getTablePath(), loadMetadataDetail.getLoadName()); dsize = String.valueOf(map.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE)); @@ -2524,7 +2524,7 @@ public final class CarbonUtil { } } finally { if (carbonLock.unlock()) { - LOGGER.info("Table unlocked successfully after table status updation"); + LOGGER.debug("Table unlocked successfully after table status updation"); } else { LOGGER.error("Unable to unlock Table lock for table during table status updation"); } @@ -2727,7 +2727,8 @@ public final class CarbonUtil { String carbonDataDirectoryPath, long fileSizeInBytes) throws CarbonDataWriterException { long copyStartTime = System.currentTimeMillis(); - LOGGER.info("Copying " + localFilePath + " --> " + carbonDataDirectoryPath); + LOGGER.info(String.format("Copying %s to %s, operation id %d", localFilePath, + carbonDataDirectoryPath, copyStartTime)); try { CarbonFile localCarbonFile = FileFactory.getCarbonFile(localFilePath, FileFactory.getFileType(localFilePath)); @@ -2740,9 +2741,8 @@ public final class CarbonUtil { throw new CarbonDataWriterException( "Problem while copying file from local store to carbon store", e); } - LOGGER.info( - "Total copy time (ms) to copy file " + localFilePath + " is " + (System.currentTimeMillis() - - copyStartTime)); + LOGGER.info(String.format("Total copy time is %d ms, operation id %d", + System.currentTimeMillis() - copyStartTime, copyStartTime)); } /** @@ -2797,10 +2797,12 @@ public final class CarbonUtil { String readableBlockSize = ByteUtil.convertByteToReadable(blockSize); String readableFileSize = ByteUtil.convertByteToReadable(fileSize); String readableMaxSize = ByteUtil.convertByteToReadable(maxSize); - LOGGER.info( - "The configured block size is " + readableBlockSize + ", the actual carbon file size is " - + readableFileSize + ", choose the max value " + readableMaxSize - + " as the block size on HDFS"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "The configured block size is " + readableBlockSize + ", the actual carbon file size is " + + readableFileSize + ", choose the max value " + readableMaxSize + + " as the block size on HDFS"); + } return maxSize; } @@ -3117,10 +3119,11 @@ public final class CarbonUtil { } } if (islocalDictEnabled) { - LOGGER.info("Local dictionary is enabled for table: " + carbonTable.getTableUniqueName()); - LOGGER.info( - "Local dictionary threshold for table: " + carbonTable.getTableUniqueName() + " is: " - + carbonTable.getLocalDictionaryThreshold()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Local dictionary is enabled for table: " + carbonTable.getTableUniqueName()); + LOGGER.debug(String.format("Local dictionary threshold for table %s is %d", + carbonTable.getTableUniqueName(), carbonTable.getLocalDictionaryThreshold())); + } Iterator<Map.Entry<String, LocalDictionaryGenerator>> iterator = columnLocalDictGenMap.entrySet().iterator(); StringBuilder stringBuilder = new StringBuilder(); @@ -3129,8 +3132,10 @@ public final class CarbonUtil { stringBuilder.append(next.getKey()); stringBuilder.append(','); } - LOGGER.info("Local dictionary will be generated for the columns:" + stringBuilder.toString() - + " for table: " + carbonTable.getTableUniqueName()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Local dictionary will be generated for the columns: %s for" + + " table %s", stringBuilder.toString(), carbonTable.getTableUniqueName())); + } } return columnLocalDictGenMap; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f3b9d3b/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java b/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java index b93f21d..c3b587c 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/AbstractDataLoadProcessorStep.java @@ -72,13 +72,13 @@ public abstract class AbstractDataLoadProcessorStep { * @throws IOException */ public void initialize() throws IOException { - if (LOGGER.isInfoEnabled()) { + if (LOGGER.isDebugEnabled()) { // This thread prints the rows processed in each step for every 10 seconds. new Thread() { @Override public void run() { while (!closed) { try { - LOGGER.info("Rows processed in step " + getStepName() + " : " + rowCounter.get()); + LOGGER.debug("Rows processed in step " + getStepName() + " : " + rowCounter.get()); Thread.sleep(10000); } catch (InterruptedException e) { //ignore http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f3b9d3b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadExecutor.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadExecutor.java index 6550afe..11841f9 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadExecutor.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/DataLoadExecutor.java @@ -53,8 +53,6 @@ public class DataLoadExecutor { // check and remove any bad record key from bad record entry logger static map if (CarbonBadRecordUtil.hasBadRecord(loadModel)) { LOGGER.error("Data Load is partially success for table " + loadModel.getTableName()); - } else { - LOGGER.info("Data loading is successful for table " + loadModel.getTableName()); } } catch (CarbonDataLoadingException e) { if (e instanceof BadRecordFoundException) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f3b9d3b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java index d4d48f9..0d38f5c 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/CarbonRowDataWriterProcessorStepImpl.java @@ -200,11 +200,13 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces exception = new CarbonDataWriterException( "Failed for table: " + tableName + " in finishing data handler", e); } - LOGGER.info("Record Processed For table: " + tableName); - String logMessage = - "Finished Carbon DataWriterProcessorStepImpl: Read: " + readCounter[iteratorIndex] - + ": Write: " + readCounter[iteratorIndex]; - LOGGER.info(logMessage); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Record Processed For table: " + tableName); + String logMessage = + "Finished Carbon DataWriterProcessorStepImpl: Read: " + readCounter[iteratorIndex] + + ": Write: " + readCounter[iteratorIndex]; + LOGGER.debug(logMessage); + } CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get()); try { processingComplete(dataHandler); http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f3b9d3b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java index 2dc3275..6899886 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/steps/DataWriterProcessorStepImpl.java @@ -194,11 +194,11 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep { configuration.getTableIdentifier().getCarbonTableIdentifier(); String tableName = tableIdentifier.getTableName(); dataHandler.finish(); - LOGGER.info("Record Processed For table: " + tableName); - String logMessage = - "Finished Carbon DataWriterProcessorStepImpl: Read: " + readCounter + ": Write: " - + rowCounter.get(); - LOGGER.info(logMessage); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Record Processed For table: " + tableName); + LOGGER.debug("Finished Carbon DataWriterProcessorStepImpl: Read: " + readCounter + + ": Write: " + rowCounter.get()); + } CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get()); processingComplete(dataHandler); CarbonTimeStatisticsFactory.getLoadStatisticsInstance() @@ -235,7 +235,7 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep { super.close(); if (listener != null) { try { - LOGGER.info("closing all the DataMap writers registered to DataMap writer listener"); + LOGGER.debug("closing all the DataMap writers registered to DataMap writer listener"); listener.finish(); } catch (IOException e) { LOGGER.error("error while closing the datamap writers", e); http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f3b9d3b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java index 67c4463..c58f8d8 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java @@ -134,7 +134,9 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { } } - LOGGER.info("Columns considered as NoInverted Index are " + noInvertedIdxCol.toString()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Columns considered as NoInverted Index are " + noInvertedIdxCol.toString()); + } } private void initParameters(CarbonFactDataHandlerModel model) { @@ -146,7 +148,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { .getBucketId())); producerExecutorServiceTaskList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); - LOGGER.info("Initializing writer executors"); + LOGGER.debug("Initializing writer executors"); consumerExecutorService = Executors.newFixedThreadPool(1, new CarbonThreadFactory( "ConsumerPool_" + System.nanoTime() + ":" + model.getTableName() + ", range: " + model .getBucketId())); @@ -210,7 +212,10 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { blockletProcessingCount.incrementAndGet(); // set the entry count to zero processedDataCount += entryCount; - LOGGER.info("Total Number Of records added to store: " + processedDataCount); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Total Number Of records added to store: " + processedDataCount); + } dataRows = new ArrayList<>(this.pageSize); this.entryCount = 0; } catch (InterruptedException e) { @@ -256,7 +261,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { } if (SnappyCompressor.MAX_BYTE_TO_COMPRESS - (varcharColumnSizeInByte[i] + dataRows.size() * 4) < (2 << 20)) { - LOGGER.info("Limited by varchar column, page size is " + dataRows.size()); + LOGGER.debug("Limited by varchar column, page size is " + dataRows.size()); // re-init for next page varcharColumnSizeInByte = new int[model.getVarcharDimIdxInNoDict().size()]; return true; @@ -284,7 +289,9 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { tablePage.encode(); - LOGGER.info("Number Of records processed: " + dataRows.size()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Number Of records processed: " + dataRows.size()); + } return tablePage; } @@ -302,14 +309,18 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { if (producerExecutorService.isShutdown()) { return; } - LOGGER.info("Started Finish Operation"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Started Finish Operation"); + } try { semaphore.acquire(); producerExecutorServiceTaskList.add(producerExecutorService .submit(new Producer(tablePageList, dataRows, ++writerTaskSequenceCounter, true))); blockletProcessingCount.incrementAndGet(); processedDataCount += entryCount; - LOGGER.info("Total Number Of records added to store: " + processedDataCount); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Total Number Of records added to store: " + processedDataCount); + } closeWriterExecutionService(producerExecutorService); processWriteTaskSubmitList(producerExecutorServiceTaskList); processingComplete = true; @@ -377,7 +388,9 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { consumerExecutorService.shutdownNow(); processWriteTaskSubmitList(consumerExecutorServiceTaskList); this.dataWriter.writeFooter(); - LOGGER.info("All blocklets have been finished writing"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("All blocklets have been finished writing"); + } // close all the open stream for both the files this.dataWriter.closeWriter(); } @@ -402,12 +415,12 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { pageSize : CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT; } - LOGGER.info("Number of rows per column page is configured as pageSize = " + pageSize); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Number of rows per column page is configured as pageSize = " + pageSize); + } dataRows = new ArrayList<>(this.pageSize); if (model.getVarcharDimIdxInNoDict().size() > 0) { - LOGGER.info("Number of rows per column blocklet is constrained by pageSize and actual size " + - "of long string column(s)"); varcharColumnSizeInByte = new int[model.getVarcharDimIdxInNoDict().size()]; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f3b9d3b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java index 64084e8..31d3b68 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java @@ -282,7 +282,7 @@ public abstract class AbstractFactDataWriter implements CarbonFactDataWriter { .add(executorService.submit(new CompleteHdfsBackendThread(carbonDataFileTempPath))); } } catch (IOException e) { - LOGGER.error("Failed to delete carbondata file from temp location" + e.getMessage()); + LOGGER.error(e); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f3b9d3b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java index 7256c1e..48993c5 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java @@ -144,8 +144,11 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter { addPageData(tablePage); } - LOGGER.info("Number of Pages for blocklet is: " + blockletDataHolder.getNumberOfPagesAdded() - + " :Rows Added: " + blockletDataHolder.getTotalRows()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Number of Pages for blocklet is: " + + blockletDataHolder.getNumberOfPagesAdded() + + " :Rows Added: " + blockletDataHolder.getTotalRows()); + } // write the data writeBlockletToFile(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/6f3b9d3b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java index 044e4e8..b34ed13 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java @@ -522,7 +522,6 @@ public final class CarbonDataProcessorUtil { configuration.getDataLoadProperty(CarbonCommonConstants.LOAD_SORT_SCOPE) .toString()); } - LOGGER.info("sort scope is set to " + sortScope); } catch (Exception e) { sortScope = SortScopeOptions.getSortScope(CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT); LOGGER.warn("Exception occured while resolving sort scope. " +