wip clean up unused declare
fix style fix comment fix style Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/3fe69037 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/3fe69037 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/3fe69037 Branch: refs/heads/master Commit: 3fe69037e741b0fa71a1d2369eff3a8df6c4e723 Parents: 93f23cc Author: jackylk <jacky.li...@huawei.com> Authored: Wed Dec 28 01:37:36 2016 +0800 Committer: chenliang613 <chenliang...@apache.org> Committed: Thu Dec 29 16:11:07 2016 +0800 ---------------------------------------------------------------------- .../hadoop/test/util/StoreCreator.java | 12 +- .../carbondata/spark/load/CarbonLoaderUtil.java | 49 +--- .../lcm/status/SegmentStatusManager.java | 4 - .../api/dataloader/DataLoadModel.java | 56 ---- .../processing/api/dataloader/SchemaInfo.java | 76 ----- .../processing/csvload/DataGraphExecuter.java | 19 +- .../processing/csvload/GraphExecutionUtil.java | 5 +- .../processing/csvreaderstep/CsvInput.java | 201 ------------- .../processing/dataprocessor/queue/Queue.java | 47 --- .../queue/impl/DataProcessorQueue.java | 106 ------- .../holder/DataProcessorRecordHolder.java | 25 -- .../processing/datatypes/ArrayDataType.java | 8 - .../processing/datatypes/GenericDataType.java | 5 - .../processing/datatypes/PrimitiveDataType.java | 11 - .../processing/datatypes/StructDataType.java | 8 - .../graphgenerator/GraphGenerator.java | 43 +-- .../configuration/GraphConfigurationInfo.java | 153 ---------- .../processing/mdkeygen/MDKeyGenStepMeta.java | 27 -- .../processing/mdkeygen/file/FileManager.java | 7 - .../merger/step/CarbonSliceMergerStepMeta.java | 90 ------ .../processing/model/CarbonLoadModel.java | 18 -- .../newflow/CarbonDataLoadConfiguration.java | 8 - .../processing/newflow/DataField.java | 11 - .../newflow/iterator/InputIterator.java | 40 --- .../sort/impl/ParallelReadMergeSorterImpl.java | 6 - .../impl/UnsafeParallelReadMergeSorterImpl.java | 6 - .../sort/unsafe/UnsafeCarbonRowPage.java | 30 -- .../newflow/sort/unsafe/UnsafeSortDataRows.java | 39 --- .../holder/UnsafeInmemoryMergeHolder.java | 4 - .../UnsafeInMemoryIntermediateDataMerger.java | 1 - .../UnsafeSingleThreadFinalSortFilesMerger.java | 5 - .../newflow/steps/InputProcessorStepImpl.java | 5 - .../processing/schema/metadata/ColumnsInfo.java | 72 ----- .../schema/metadata/HierarchiesInfo.java | 77 ----- .../sortdata/AbstractTempSortFileReader.java | 141 --------- .../sortdata/CompressedTempSortFileReader.java | 52 ---- .../sortandgroupby/sortdata/SortParameters.java | 4 +- .../sortdata/TempSortFileReaderFactory.java | 45 --- .../UnCompressedTempSortFileReader.java | 50 ---- .../sortdatastep/SortKeyStep.java | 2 +- .../sortdatastep/SortKeyStepMeta.java | 32 --- .../store/CarbonFactDataHandlerColumnar.java | 33 --- .../store/CarbonFactDataHandlerModel.java | 64 ----- .../store/writer/AbstractFactDataWriter.java | 15 - .../store/writer/CarbonDataWriterVo.java | 21 -- .../store/writer/CarbonFactDataWriter.java | 12 - .../processing/store/writer/NodeHolder.java | 70 ----- .../CarbonCSVBasedDimSurrogateKeyGen.java | 16 -- .../csvbased/CarbonCSVBasedSeqGenData.java | 35 --- .../csvbased/CarbonCSVBasedSeqGenMeta.java | 270 ----------------- .../csvbased/CarbonCSVBasedSeqGenStep.java | 3 - .../FileStoreSurrogateKeyGenForCSV.java | 36 --- .../dbbased/CarbonDimSurrogateKeyGen.java | 286 ------------------- .../dbbased/HierarchyValueWriter.java | 155 ---------- .../dbbased/IntArrayWrapper.java | 110 ------- .../processing/util/CarbonSchemaParser.java | 223 +-------------- .../processing/util/RemoveDictionaryUtil.java | 9 - .../carbondata/test/util/StoreCreator.java | 12 +- 58 files changed, 29 insertions(+), 2941 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java ---------------------------------------------------------------------- diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java index 929e404..4519e90 100644 --- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java +++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java @@ -476,20 +476,10 @@ public class StoreCreator { model.setQuoteCharacter(dataProcessTaskStatus.getQuoteCharacter()); model.setCommentCharacter(dataProcessTaskStatus.getCommentCharacter()); model.setDateFormat(dataProcessTaskStatus.getDateFormat()); - if (null != loadMetadataDetails && !loadMetadataDetails.isEmpty()) { - model.setLoadNames( - CarbonDataProcessorUtil.getLoadNameFromLoadMetaDataDetails(loadMetadataDetails)); - model.setModificationOrDeletionTime(CarbonDataProcessorUtil - .getModificationOrDeletionTimesFromLoadMetadataDetails(loadMetadataDetails)); - } - boolean hdfsReadMode = - dataProcessTaskStatus.getCsvFilePath() != null && dataProcessTaskStatus.getCsvFilePath().startsWith("hdfs:"); - int allocate = null != dataProcessTaskStatus.getCsvFilePath() ? 1 : dataProcessTaskStatus.getFilesToProcess().size(); String outputLocation = CarbonProperties.getInstance() .getProperty("store_output_location", "../carbon-store/system/carbon/etl"); GraphGenerator generator = - new GraphGenerator(model, hdfsReadMode, partitionID, factStoreLocation, - allocate, schema, "0", outputLocation); + new GraphGenerator(model, partitionID, factStoreLocation, schema, "0", outputLocation); generator.generateGraph(); } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java index 56ddce2..eca34a6 100644 --- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java +++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java @@ -77,7 +77,6 @@ import org.apache.carbondata.processing.dataprocessor.IDataProcessStatus; import org.apache.carbondata.processing.graphgenerator.GraphGenerator; import org.apache.carbondata.processing.graphgenerator.GraphGeneratorException; import org.apache.carbondata.processing.model.CarbonLoadModel; -import org.apache.carbondata.processing.util.CarbonDataProcessorUtil; import org.apache.carbondata.spark.merger.NodeBlockRelation; import org.apache.carbondata.spark.merger.NodeMultiBlockRelation; @@ -89,25 +88,8 @@ public final class CarbonLoaderUtil { private static final LogService LOGGER = LogServiceFactory.getLogService(CarbonLoaderUtil.class.getName()); - /** - * minimum no of blocklet required for distribution - */ - private static int minBlockLetsReqForDistribution = 0; - - static { - String property = CarbonProperties.getInstance() - .getProperty(CarbonCommonConstants.CARBON_BLOCKLETDISTRIBUTION_MIN_REQUIRED_SIZE); - try { - minBlockLetsReqForDistribution = Integer.parseInt(property); - } catch (NumberFormatException ne) { - LOGGER.info("Invalid configuration. Consisering the defaul"); - minBlockLetsReqForDistribution = - CarbonCommonConstants.DEFAULT_CARBON_BLOCKLETDISTRIBUTION_MIN_REQUIRED_SIZE; - } - } private CarbonLoaderUtil() { - } private static void generateGraph(IDataProcessStatus dataProcessTaskStatus, SchemaInfo info, @@ -118,13 +100,6 @@ public final class CarbonLoaderUtil { || null != dataProcessTaskStatus.getFilesToProcess()); model.setSchemaInfo(info); model.setTableName(dataProcessTaskStatus.getTableName()); - List<LoadMetadataDetails> loadMetadataDetails = loadModel.getLoadMetadataDetails(); - if (null != loadMetadataDetails && !loadMetadataDetails.isEmpty()) { - model.setLoadNames( - CarbonDataProcessorUtil.getLoadNameFromLoadMetaDataDetails(loadMetadataDetails)); - model.setModificationOrDeletionTime(CarbonDataProcessorUtil - .getModificationOrDeletionTimesFromLoadMetadataDetails(loadMetadataDetails)); - } model.setBlocksID(dataProcessTaskStatus.getBlocksID()); model.setEscapeCharacter(dataProcessTaskStatus.getEscapeCharacter()); model.setQuoteCharacter(dataProcessTaskStatus.getQuoteCharacter()); @@ -134,15 +109,9 @@ public final class CarbonLoaderUtil { model.setFactTimeStamp(loadModel.getFactTimeStamp()); model.setMaxColumns(loadModel.getMaxColumns()); model.setDateFormat(loadModel.getDateFormat()); - boolean hdfsReadMode = - dataProcessTaskStatus.getCsvFilePath() != null - && dataProcessTaskStatus.getCsvFilePath().startsWith("hdfs:"); - int allocate = - null != dataProcessTaskStatus.getCsvFilePath() - ? 1 : dataProcessTaskStatus.getFilesToProcess().size(); - GraphGenerator generator = new GraphGenerator(model, hdfsReadMode, loadModel.getPartitionId(), - loadModel.getStorePath(), allocate, - loadModel.getCarbonDataLoadSchema(), loadModel.getSegmentId(), outputLocation); + GraphGenerator generator = new GraphGenerator(model, loadModel.getPartitionId(), + loadModel.getStorePath(), loadModel.getCarbonDataLoadSchema(), loadModel.getSegmentId(), + outputLocation); generator.generateGraph(); } @@ -160,7 +129,6 @@ public final class CarbonLoaderUtil { CarbonProperties.getInstance().addProperty(tempLocationKey, storeLocation); CarbonProperties.getInstance() .addProperty(CarbonCommonConstants.STORE_LOCATION_HDFS, storePath); - // CarbonProperties.getInstance().addProperty("store_output_location", outPutLoc); CarbonProperties.getInstance().addProperty("send.signal.load", "false"); String fileNamePrefix = ""; @@ -542,17 +510,6 @@ public final class CarbonLoaderUtil { } /** - * This method will divide the blocks among the nodes as per the data locality - * - * @param blockInfos - * @return - */ - public static Map<String, List<Distributable>> nodeBlockMapping(List<Distributable> blockInfos) { - // -1 if number of nodes has to be decided based on block location information - return nodeBlockMapping(blockInfos, -1); - } - - /** * the method returns the number of required executors * * @param blockInfos http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java b/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java index 030ee0b..b8b86b6 100644 --- a/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java +++ b/processing/src/main/java/org/apache/carbondata/lcm/status/SegmentStatusManager.java @@ -582,10 +582,6 @@ public class SegmentStatusManager { return validSegments; } - public List<String> getUpadtedSegments() { - return validUpdatedSegments; - } - public List<String> getInvalidSegments() { return invalidSegments; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/DataLoadModel.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/DataLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/DataLoadModel.java index 5a7f14d..3971c97 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/DataLoadModel.java +++ b/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/DataLoadModel.java @@ -35,20 +35,6 @@ public class DataLoadModel { */ private boolean isCsvLoad; - /** - * Modified Dimension - */ - private String[] modifiedDimesion; - - /** - * loadNames separated by HASH_SPC_CHARACTER - */ - private String loadNames; - /** - * modificationOrDeletionTime separated by HASH_SPC_CHARACTER - */ - private String modificationOrDeletionTime; - private String blocksID; /** * task id, each spark task has a unique id @@ -113,48 +99,6 @@ public class DataLoadModel { } /** - * @return Returns the modifiedDimesion. - */ - public String[] getModifiedDimesion() { - return modifiedDimesion; - } - - /** - * @param modifiedDimesion The modifiedDimesion to set. - */ - public void setModifiedDimesion(String[] modifiedDimesion) { - this.modifiedDimesion = modifiedDimesion; - } - - /** - * return modificationOrDeletionTime separated by HASH_SPC_CHARACTER - */ - public String getModificationOrDeletionTime() { - return modificationOrDeletionTime; - } - - /** - * set modificationOrDeletionTime separated by HASH_SPC_CHARACTER - */ - public void setModificationOrDeletionTime(String modificationOrDeletionTime) { - this.modificationOrDeletionTime = modificationOrDeletionTime; - } - - /** - * return loadNames separated by HASH_SPC_CHARACTER - */ - public String getLoadNames() { - return loadNames; - } - - /** - * set loadNames separated by HASH_SPC_CHARACTER - */ - public void setLoadNames(String loadNames) { - this.loadNames = loadNames; - } - - /** * get block id * * @return http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/SchemaInfo.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/SchemaInfo.java b/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/SchemaInfo.java index a00e913..4b0d8bf 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/SchemaInfo.java +++ b/processing/src/main/java/org/apache/carbondata/processing/api/dataloader/SchemaInfo.java @@ -27,26 +27,6 @@ public class SchemaInfo { private String databaseName; /** - * srcDriverName - */ - private String srcDriverName; - - /** - * srcConUrl - */ - private String srcConUrl; - - /** - * srcUserName - */ - private String srcUserName; - - /** - * srcPwd - */ - private String srcPwd; - - /** * tableName */ private String tableName; @@ -90,62 +70,6 @@ public class SchemaInfo { this.complexDelimiterLevel2 = complexDelimiterLevel2; } - /** - * @return Returns the srcDriverName. - */ - public String getSrcDriverName() { - return srcDriverName; - } - - /** - * @param srcDriverName The srcDriverName to set. - */ - public void setSrcDriverName(String srcDriverName) { - this.srcDriverName = srcDriverName; - } - - /** - * @return Returns the srcConUrl. - */ - public String getSrcConUrl() { - return srcConUrl; - } - - /** - * @param srcConUrl The srcConUrl to set. - */ - public void setSrcConUrl(String srcConUrl) { - this.srcConUrl = srcConUrl; - } - - /** - * @return Returns the srcUserName. - */ - public String getSrcUserName() { - return srcUserName; - } - - /** - * @param srcUserName The srcUserName to set. - */ - public void setSrcUserName(String srcUserName) { - this.srcUserName = srcUserName; - } - - /** - * @return Returns the srcPwd. - */ - public String getSrcPwd() { - return srcPwd; - } - - /** - * @param srcPwd The srcPwd to set. - */ - public void setSrcPwd(String srcPwd) { - this.srcPwd = srcPwd; - } - public String getTableName() { return tableName; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java index 4765277..3d35837 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java +++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java @@ -260,7 +260,7 @@ public class DataGraphExecuter { CarbonFile csvFileToRead = GraphExecutionUtil.getCsvFileToRead(model.getCsvFilePath()); TextFileInputField[] inputFields = GraphExecutionUtil - .getTextInputFiles(csvFileToRead, measureColumns, builder, measuresInCSVFile, ","); + .getTextInputFiles(csvFileToRead, builder, measuresInCSVFile, ","); stepMetaInterface.setInputFields(inputFields); } else if (model.isDirectLoad()) { String[] files = new String[model.getFilesToProcess().size()]; @@ -293,7 +293,7 @@ public class DataGraphExecuter { CarbonFile csvFile = GraphExecutionUtil.getCsvFileToRead(model.getFilesToProcess().get(0)); TextFileInputField[] inputFields = GraphExecutionUtil - .getTextInputFiles(csvFile, measureColumns, builder, measuresInCSVFile, + .getTextInputFiles(csvFile, builder, measuresInCSVFile, model.getCsvDelimiter()); ((CsvInputMeta) step.getStepMetaInterface()).setInputFields(inputFields); ((CsvInputMeta) step.getStepMetaInterface()).setDelimiter(model.getCsvDelimiter()); @@ -369,7 +369,7 @@ public class DataGraphExecuter { if (null != model.getCsvFilePath() && model.getRddIteratorKey() == null) { CarbonFile csvFileToRead = GraphExecutionUtil.getCsvFileToRead(model.getCsvFilePath()); TextFileInputField[] inputFields = GraphExecutionUtil - .getTextInputFiles(csvFileToRead, measureColumns, builder, measuresInCSVFile, ","); + .getTextInputFiles(csvFileToRead, builder, measuresInCSVFile, ","); ((CsvInputMeta) step.getStepMetaInterface()).setInputFields(inputFields); } else if (model.isDirectLoad()) { if (null != model.getCsvHeader() && !model.getCsvHeader().isEmpty()) { @@ -385,7 +385,7 @@ public class DataGraphExecuter { CarbonFile csvFileToRead = GraphExecutionUtil.getCsvFileToRead(model.getFilesToProcess().get(0)); TextFileInputField[] inputFields = GraphExecutionUtil - .getTextInputFiles(csvFileToRead, measureColumns, builder, measuresInCSVFile, + .getTextInputFiles(csvFileToRead, builder, measuresInCSVFile, model.getCsvDelimiter()); ((CsvInputMeta) step.getStepMetaInterface()).setInputFields(inputFields); ((CsvInputMeta) step.getStepMetaInterface()).setDelimiter(model.getCsvDelimiter()); @@ -541,15 +541,4 @@ public class DataGraphExecuter { return GraphExecutionUtil.checkCSVAndRequestedTableColumns(dimFilePath, columnNames, delimiter); } - /** - * Interrupts all child threads run by kettle to execute the graph - */ - public void interruptGraphExecution() { - LOGGER.error("Graph Execution is interrupted"); - if (null != trans) { - trans.killAll(); - LOGGER.info("Graph execution steps are killed."); - } - } - } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java index 1dd9bdf..2148901 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java @@ -93,9 +93,8 @@ public final class GraphExecutionUtil { * @param measuresInCSVFile * @throws DataLoadingException */ - public static TextFileInputField[] getTextInputFiles(CarbonFile csvFile, - List<String> measureColumns, StringBuilder builder, StringBuilder measuresInCSVFile, - String delimiter) throws DataLoadingException { + public static TextFileInputField[] getTextInputFiles(CarbonFile csvFile, StringBuilder builder, + StringBuilder measuresInCSVFile, String delimiter) throws DataLoadingException { DataInputStream fileReader = null; BufferedReader bufferedReader = null; String readLine = null; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInput.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInput.java b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInput.java index c64c504..11acd56 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInput.java +++ b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInput.java @@ -41,7 +41,6 @@ import org.apache.carbondata.processing.graphgenerator.GraphGenerator; import org.apache.commons.lang3.StringUtils; import org.pentaho.di.core.Const; import org.pentaho.di.core.exception.KettleException; -import org.pentaho.di.core.logging.LogChannelInterface; import org.pentaho.di.core.row.RowMeta; import org.pentaho.di.core.row.ValueMetaInterface; import org.pentaho.di.i18n.BaseMessages; @@ -81,7 +80,6 @@ public class CsvInput extends BaseStep implements StepInterface { * resultArray */ private Future[] resultArray; - private boolean isTerminated; private List<List<BlockDetails>> threadBlockList = new ArrayList<>(); private ExecutorService exec; @@ -97,205 +95,6 @@ public class CsvInput extends BaseStep implements StepInterface { LOGGER.info("** Using csv file **"); } - /** - * This method is borrowed from TextFileInput - * - * @param log - * @param line - * @param delimiter - * @param enclosure - * @param escapeCharacter - * @return - * @throws KettleException - */ - public static final String[] guessStringsFromLine(LogChannelInterface log, String line, - String delimiter, String enclosure, String escapeCharacter) throws KettleException { - List<String> strings = new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN); - - String pol; // piece of line - - try { - if (line == null) { - return null; - } - - // Split string in pieces, only for CSV! - int pos = 0; - int length = line.length(); - boolean dencl = false; - - int lenEncl = (enclosure == null ? 0 : enclosure.length()); - int lenEsc = (escapeCharacter == null ? 0 : escapeCharacter.length()); - - while (pos < length) { - int from = pos; - int next; - - boolean enclFound; - boolean containsEscapedEnclosures = false; - boolean containsEscapedSeparators = false; - - // Is the field beginning with an enclosure? - // "aa;aa";123;"aaa-aaa";000;... - if (lenEncl > 0 && line.substring(from, from + lenEncl).equalsIgnoreCase(enclosure)) { - if (log.isRowLevel()) { - log.logRowlevel(BaseMessages.getString(PKG, "CsvInput.Log.ConvertLineToRowTitle"), - BaseMessages.getString(PKG, "CsvInput.Log.ConvertLineToRow", - line.substring(from, from + lenEncl))); //$NON-NLS-1$ //$NON-NLS-2$ - } - enclFound = true; - int p = from + lenEncl; - - boolean isEnclosure = - lenEncl > 0 && p + lenEncl < length && line.substring(p, p + lenEncl) - .equalsIgnoreCase(enclosure); - boolean isEscape = lenEsc > 0 && p + lenEsc < length && line.substring(p, p + lenEsc) - .equalsIgnoreCase(escapeCharacter); - - boolean enclosureAfter = false; - - // Is it really an enclosure? See if it's not repeated twice or escaped! - if ((isEnclosure || isEscape) && p < length - 1) { - String strnext = line.substring(p + lenEncl, p + 2 * lenEncl); - if (strnext.equalsIgnoreCase(enclosure)) { - p++; - enclosureAfter = true; - dencl = true; - - // Remember to replace them later on! - if (isEscape) { - containsEscapedEnclosures = true; - } - } - } - - // Look for a closing enclosure! - while ((!isEnclosure || enclosureAfter) && p < line.length()) { - p++; - enclosureAfter = false; - isEnclosure = lenEncl > 0 && p + lenEncl < length && line.substring(p, p + lenEncl) - .equals(enclosure); - isEscape = lenEsc > 0 && p + lenEsc < length && line.substring(p, p + lenEsc) - .equals(escapeCharacter); - - // Is it really an enclosure? See if it's not repeated twice or escaped! - if ((isEnclosure || isEscape) && p < length - 1) // Is - { - String strnext = line.substring(p + lenEncl, p + 2 * lenEncl); - if (strnext.equals(enclosure)) { - p++; - enclosureAfter = true; - dencl = true; - - // Remember to replace them later on! - if (isEscape) { - containsEscapedEnclosures = true; // remember - } - } - } - } - - if (p >= length) { - next = p; - } else { - next = p + lenEncl; - } - - if (log.isRowLevel()) { - log.logRowlevel(BaseMessages.getString(PKG, "CsvInput.Log.ConvertLineToRowTitle"), - BaseMessages.getString(PKG, "CsvInput.Log.EndOfEnclosure", - "" + p)); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - } - } else { - enclFound = false; - boolean found = false; - int startpoint = from; - do { - next = line.indexOf(delimiter, startpoint); - - // See if this position is preceded by an escape character. - if (lenEsc > 0 && next - lenEsc > 0) { - String before = line.substring(next - lenEsc, next); - - if (escapeCharacter != null && escapeCharacter.equals(before)) { - // take the next separator, this one is escaped... - startpoint = next + 1; - containsEscapedSeparators = true; - } else { - found = true; - } - } else { - found = true; - } - } while (!found && next >= 0); - } - if (next == -1) { - next = length; - } - - if (enclFound) { - pol = line.substring(from + lenEncl, next - lenEncl); - if (log.isRowLevel()) { - log.logRowlevel(BaseMessages.getString(PKG, "CsvInput.Log.ConvertLineToRowTitle"), - BaseMessages.getString(PKG, "CsvInput.Log.EnclosureFieldFound", - "" + pol)); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - } - } else { - pol = line.substring(from, next); - if (log.isRowLevel()) { - log.logRowlevel(BaseMessages.getString(PKG, "CsvInput.Log.ConvertLineToRowTitle"), - BaseMessages.getString(PKG, "CsvInput.Log.NormalFieldFound", - "" + pol)); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ - } - } - - if (dencl) { - StringBuilder sbpol = new StringBuilder(pol); - int idx = sbpol.indexOf(enclosure + enclosure); - while (idx >= 0) { - sbpol.delete(idx, idx + (enclosure == null ? 0 : enclosure.length())); - idx = sbpol.indexOf(enclosure + enclosure); - } - pol = sbpol.toString(); - } - - // replace the escaped enclosures with enclosures... - if (containsEscapedEnclosures) { - String replace = escapeCharacter + enclosure; - String replaceWith = enclosure; - - pol = Const.replace(pol, replace, replaceWith); - } - - //replace the escaped separators with separators... - if (containsEscapedSeparators) { - String replace = escapeCharacter + delimiter; - String replaceWith = delimiter; - - pol = Const.replace(pol, replace, replaceWith); - } - - // Now add pol to the strings found! - strings.add(pol); - - pos = next + delimiter.length(); - } - if (pos == length) { - if (log.isRowLevel()) { - log.logRowlevel(BaseMessages.getString(PKG, "CsvInput.Log.ConvertLineToRowTitle"), - BaseMessages.getString(PKG, "CsvInput.Log.EndOfEmptyLineFound")); - } - strings.add(""); //$NON-NLS-1$ - } - } catch (Exception e) { - throw new KettleException( - BaseMessages.getString(PKG, "CsvInput.Log.Error.ErrorConvertingLine", e.toString()), - e); //$NON-NLS-1$ - } - - return strings.toArray(new String[strings.size()]); - } - public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException { meta = (CsvInputMeta) smi; data = (CsvInputData) sdi; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/queue/Queue.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/queue/Queue.java b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/queue/Queue.java deleted file mode 100644 index 91c78ad..0000000 --- a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/queue/Queue.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.processing.dataprocessor.queue; - -public interface Queue<E> { - /** - * This method insert the element in the queue, and return false if it fails - * to isert in the queue. - * - * @param obj - * @return true if insererted properly false otherwise. - */ - boolean offer(E obj); - - /** - * This method get the element from the head of the queue and - * remove the element from the queue. - * - * @return - */ - E poll(); - - /** - * This method just return the element in the head of the queue. - * - * @return - */ - E peek(); - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/queue/impl/DataProcessorQueue.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/queue/impl/DataProcessorQueue.java b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/queue/impl/DataProcessorQueue.java deleted file mode 100644 index d73700e..0000000 --- a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/queue/impl/DataProcessorQueue.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.processing.dataprocessor.queue.impl; - -import java.util.concurrent.PriorityBlockingQueue; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.carbondata.processing.dataprocessor.queue.Queue; -import org.apache.carbondata.processing.dataprocessor.record.holder.DataProcessorRecordHolder; - -public class DataProcessorQueue implements Queue<DataProcessorRecordHolder> { - /** - * Size of the queue - */ - private int qSize; - - /** - * Counter to maintain state of the queue. - */ - private AtomicInteger counter; - - /** - * Queue that holds the data. - */ - private PriorityBlockingQueue<DataProcessorRecordHolder> priorityQueue; - - public DataProcessorQueue(int size) { - this.counter = new AtomicInteger(); - this.qSize = size; - this.priorityQueue = - new PriorityBlockingQueue<DataProcessorRecordHolder>(size, new RecordComparator()); - - } - - @Override public boolean offer(DataProcessorRecordHolder obj) { - if (counter.get() == qSize) { - return false; - } else { - - priorityQueue.offer(obj); - counter.getAndIncrement(); - return true; - } - } - - @SuppressWarnings("unchecked") - @Override public DataProcessorRecordHolder poll() { - if (priorityQueue.isEmpty()) { - return null; - } else { - counter.getAndDecrement(); - return (DataProcessorRecordHolder) priorityQueue.poll(); - } - } - - @SuppressWarnings("unchecked") - @Override public DataProcessorRecordHolder peek() { - if (priorityQueue.isEmpty()) { - return null; - } else { - return priorityQueue.peek(); - } - } - - /** - * Is Queue is Full. - * - * @return - */ - public boolean isFull() { - return counter.get() == qSize; - } - - /** - * Is queue is Empty - * - * @return - */ - public boolean isEmpty() { - return priorityQueue.isEmpty(); - } - - /** - * return the size (i.e. Elements present in the Queue) - */ - public int size() { - return counter.get(); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/record/holder/DataProcessorRecordHolder.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/record/holder/DataProcessorRecordHolder.java b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/record/holder/DataProcessorRecordHolder.java index d34803a..f7994dc 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/record/holder/DataProcessorRecordHolder.java +++ b/processing/src/main/java/org/apache/carbondata/processing/dataprocessor/record/holder/DataProcessorRecordHolder.java @@ -24,34 +24,12 @@ package org.apache.carbondata.processing.dataprocessor.record.holder; */ public class DataProcessorRecordHolder { - private Object[][] processedRecords; - - private Object[][] originalRecords; - private int seqNumber; - private int counter; - - private int processCounter; - public DataProcessorRecordHolder(int size, int seqNumber) { - this.originalRecords = new Object[size][]; - this.processedRecords = new Object[size][]; this.seqNumber = seqNumber; } - public Object[][] getOriginalRow() { - return originalRecords; - } - - public void addRow(Object[] oriRow) { - originalRecords[counter++] = oriRow; - } - - public void addProcessedRows(Object[] processedRows) { - processedRecords[processCounter++] = processedRows; - } - /** * Returns the sequence number. */ @@ -59,7 +37,4 @@ public class DataProcessorRecordHolder { return seqNumber; } - public Object[][] getProcessedRow() { - return processedRecords; - } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/datatypes/ArrayDataType.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/ArrayDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/ArrayDataType.java index 9843c2e..14553ec 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/ArrayDataType.java +++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/ArrayDataType.java @@ -110,14 +110,6 @@ public class ArrayDataType implements GenericDataType<ArrayObject> { } /* - * set column name - */ - @Override - public void setName(String name) { - this.name = name; - } - - /* * set parent name */ @Override http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/datatypes/GenericDataType.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/GenericDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/GenericDataType.java index f8c765b..1be004b 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/GenericDataType.java +++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/GenericDataType.java @@ -44,11 +44,6 @@ public interface GenericDataType<T> { String getName(); /** - * @param name - To set column name - */ - void setName(String name); - - /** * @return - columns parent name */ String getParentname(); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java index 610366c..38f8c79 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java +++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java @@ -155,14 +155,6 @@ public class PrimitiveDataType implements GenericDataType<Object> { } /* - * set column name - */ - @Override - public void setName(String name) { - this.name = name; - } - - /* * get column parent name */ @Override @@ -245,9 +237,6 @@ public class PrimitiveDataType implements GenericDataType<Object> { dimCardWithComplex.add(dictionaryGenerator.size()); } - /* - * parse bytearray and bit pack - */ @Override public void parseAndBitPack(ByteBuffer byteArrayInput, DataOutputStream dataOutputStream, KeyGenerator[] generator) throws IOException, KeyGenException { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java index f034895..09cd421 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java +++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java @@ -100,14 +100,6 @@ public class StructDataType implements GenericDataType<StructObject> { } /* - * set column name - */ - @Override - public void setName(String name) { - this.name = name; - } - - /* * get parent column name */ @Override http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java index eb10549..fb9599e 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java +++ b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java @@ -207,16 +207,13 @@ public class GraphGenerator { private String rddIteratorKey; - public GraphGenerator(DataLoadModel dataLoadModel, boolean isHDFSReadMode, String partitionID, - String factStoreLocation, int allocate, + public GraphGenerator(DataLoadModel dataLoadModel, String partitionID, String factStoreLocation, CarbonDataLoadSchema carbonDataLoadSchema, String segmentId) { CarbonMetadata.getInstance().addCarbonTable(carbonDataLoadSchema.getCarbonTable()); this.schemaInfo = dataLoadModel.getSchemaInfo(); this.tableName = dataLoadModel.getTableName(); this.isCSVLoad = dataLoadModel.isCsvLoad(); - this.modifiedDimension = dataLoadModel.getModifiedDimesion(); this.isAutoAggRequest = schemaInfo.isAutoAggregateRequest(); - //this.schema = schema; this.carbonDataLoadSchema = carbonDataLoadSchema; this.databaseName = carbonDataLoadSchema.getCarbonTable().getDatabaseName(); this.partitionID = partitionID; @@ -235,11 +232,9 @@ public class GraphGenerator { LOGGER.info("************* Is Columnar Storage" + isColumnar); } - public GraphGenerator(DataLoadModel dataLoadModel, boolean isHDFSReadMode, String partitionID, - String factStoreLocation, int allocate, CarbonDataLoadSchema carbonDataLoadSchema, - String segmentId, String outputLocation) { - this(dataLoadModel, isHDFSReadMode, partitionID, factStoreLocation, allocate, - carbonDataLoadSchema, segmentId); + public GraphGenerator(DataLoadModel dataLoadModel, String partitionID, String factStoreLocation, + CarbonDataLoadSchema carbonDataLoadSchema, String segmentId, String outputLocation) { + this(dataLoadModel, partitionID, factStoreLocation, carbonDataLoadSchema, segmentId); this.outputLocation = outputLocation; this.rddIteratorKey = dataLoadModel.getRddIteratorKey(); } @@ -561,7 +556,6 @@ public class GraphGenerator { seqMeta.setDatabaseName(schemaInfo.getDatabaseName()); seqMeta.setComplexDelimiterLevel1(schemaInfo.getComplexDelimiterLevel1()); seqMeta.setComplexDelimiterLevel2(schemaInfo.getComplexDelimiterLevel2()); - seqMeta.setCarbonMetaHier(graphConfiguration.getMetaHeirString()); seqMeta.setCarbonmsr(graphConfiguration.getMeasuresString()); seqMeta.setCarbonProps(graphConfiguration.getPropertiesString()); seqMeta.setCarbonhier(graphConfiguration.getHiersString()); @@ -823,7 +817,7 @@ public class GraphGenerator { graphConfiguration.setNoDictionaryDims(noDictionarydimString.toString()); String tableString = - CarbonSchemaParser.getTableNameString(factTableName, dimensions, carbonDataLoadSchema); + CarbonSchemaParser.getTableNameString(dimensions, carbonDataLoadSchema); String dimensionColumnIds = CarbonSchemaParser.getColumnIdString(dimensions); graphConfiguration.setDimensionTableNames(tableString); graphConfiguration.setDimensionString(dimString.toString()); @@ -889,9 +883,6 @@ public class GraphGenerator { // check quotes required in query or Not boolean isQuotesRequired = true; String quote = CarbonSchemaParser.QUOTES; - if (null != schemaInfo.getSrcDriverName()) { - quote = getQuoteType(schemaInfo); - } graphConfiguration.setTableInputSqlQuery(CarbonSchemaParser .getTableInputSQLQuery(dimensions, measures, carbonDataLoadSchema.getCarbonTable().getFactTableName(), isQuotesRequired, @@ -908,10 +899,6 @@ public class GraphGenerator { graphConfiguration.setMeasures(CarbonSchemaParser.getMeasures(measures)); graphConfiguration.setAGG(false); - graphConfiguration.setUsername(schemaInfo.getSrcUserName()); - graphConfiguration.setPassword(schemaInfo.getSrcPwd()); - graphConfiguration.setDriverclass(schemaInfo.getSrcDriverName()); - graphConfiguration.setConnectionUrl(schemaInfo.getSrcConUrl()); return graphConfiguration; } @@ -928,25 +915,6 @@ public class GraphGenerator { return tableOptionWrapper; } - private String getQuoteType(SchemaInfo schemaInfo) throws GraphGeneratorException { - String driverClass = schemaInfo.getSrcDriverName(); - String type = DRIVERS.get(driverClass); - - if (null == type) { - LOGGER.error("Driver : \"" + driverClass + " \"Not Supported."); - throw new GraphGeneratorException("Driver : \"" + driverClass + " \"Not Supported."); - } - - if (type.equals(CarbonCommonConstants.TYPE_ORACLE) || type - .equals(CarbonCommonConstants.TYPE_MSSQL)) { - return CarbonSchemaParser.QUOTES; - } else if (type.equals(CarbonCommonConstants.TYPE_MYSQL)) { - return CarbonSchemaParser.QUOTES; - } - - return CarbonSchemaParser.QUOTES; - } - public CarbonTable getTable() { return carbonDataLoadSchema.getCarbonTable(); } @@ -959,7 +927,6 @@ public class GraphGenerator { */ private void prepareNoDictionaryMapping(List<CarbonDimension> dims, GraphConfigurationInfo graphConfig) { - // boolean[] NoDictionaryMapping = new boolean[dims.size()]; List<Boolean> noDictionaryMapping = new ArrayList<Boolean>(); for (CarbonDimension dimension : dims) { // for complex type need to break the loop http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java index 7c428a7..d1ada49 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java +++ b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java @@ -228,13 +228,6 @@ public class GraphConfigurationInfo { } /** - * @return the connectionName - */ - public String getConnectionName() { - return connectionName; - } - - /** * @param connectionName the connectionName to set */ public void setConnectionName(String connectionName) { @@ -242,20 +235,6 @@ public class GraphConfigurationInfo { } /** - * @return the dbType - */ - public String getDbType() { - return dbType; - } - - /** - * @param dbType the dbType to set - */ - public void setDbType(String dbType) { - this.dbType = dbType; - } - - /** * @return the numberOfCores */ public String getNumberOfCores() { @@ -270,13 +249,6 @@ public class GraphConfigurationInfo { } /** - * @return the storeLocation - */ - public String getStoreLocation() { - return storeLocation; - } - - /** * @param storeLocation the storeLocation to set */ public void setStoreLocation(String storeLocation) { @@ -306,13 +278,6 @@ public class GraphConfigurationInfo { } /** - * @return the blockletSize - */ - public String getBlockletSize() { - return blockletSize; - } - - /** * @param blockletSize the blockletSize to set */ public void setBlockletSize(String blockletSize) { @@ -320,13 +285,6 @@ public class GraphConfigurationInfo { } /** - * @return the maxBlockletInFile - */ - public String getMaxBlockletInFile() { - return maxBlockletInFile; - } - - /** * @param maxBlockletInFile the maxBlockletInFile to set */ public void setMaxBlockletInFile(String maxBlockletInFile) { @@ -413,15 +371,6 @@ public class GraphConfigurationInfo { } /** - * setNormHiers - * - * @param normHiers void - */ - public void setNormHiers(String normHiers) { - this.normHiers = normHiers; - } - - /** * @return the hiersString */ public String getHiersString() { @@ -457,27 +406,6 @@ public class GraphConfigurationInfo { } /** - * @param propertiesString the propertiesString to set - */ - public void setPropertiesString(String propertiesString) { - this.propertiesString = propertiesString; - } - - /** - * @return the timeHeirString - */ - public String getTimeHeirString() { - return timeHeirString; - } - - /** - * @param timeHeirString the timeHeirString to set - */ - public void setTimeHeirString(String timeHeirString) { - this.timeHeirString = timeHeirString; - } - - /** * @return the metaHeirString */ public String getMetaHeirString() { @@ -492,48 +420,6 @@ public class GraphConfigurationInfo { } /** - * @return the metaHeirQueryString - */ - public String getMetaHeirQueryString() { - return metaHeirQueryString; - } - - /** - * @param metaHeirQueryString the metaHeirQueryString to set - */ - public void setMetaHeirQueryString(String metaHeirQueryString) { - this.metaHeirQueryString = metaHeirQueryString; - } - - /** - * @return the jndiName - */ - public String getJndiName() { - return jndiName; - } - - /** - * @param jndiName the jndiName to set - */ - public void setJndiName(String jndiName) { - this.jndiName = jndiName; - } - - /** - * @return the tableMeasuresAndDataTypeMap - */ - public Map<String, String> getTableMeasuresAndDataTypeMap() { - return tableMeasuresAndDataTypeMap; - } - - /** - * @param tableMeasuresAndDataTypeMap the tableMeasuresAndDataTypeMap to set - */ - public void setTableMeasuresAndDataTypeMap(Map<String, String> tableMeasuresAndDataTypeMap) { - this.tableMeasuresAndDataTypeMap = tableMeasuresAndDataTypeMap; - } - - /** * @return the tableInputSqlQuery */ public String getTableInputSqlQuery() { @@ -562,13 +448,6 @@ public class GraphConfigurationInfo { } /** - * @return the sortSize - */ - public String getSortSize() { - return sortSize; - } - - /** * @param sortSize the sortSize to set */ public void setSortSize(String sortSize) { @@ -756,13 +635,6 @@ public class GraphConfigurationInfo { } /** - * @return Returns the forignKey. - */ - public String[] getForignKey() { - return forignKey; - } - - /** * @param forignKey The forignKey to set. */ public void setForignKey(String[] forignKey) { @@ -889,10 +761,6 @@ public class GraphConfigurationInfo { return columnAndTableNameColumnMapForAgg; } - public void setColumnAndTableNameColumnMapForAgg(String columnAndTableNameColumnMapForAgg) { - this.columnAndTableNameColumnMapForAgg = columnAndTableNameColumnMapForAgg; - } - /** * @return Returns the denormColumns. */ @@ -915,20 +783,6 @@ public class GraphConfigurationInfo { } /** - * @param aggClass the aggClass to set - */ - public void setAggClass(String[] aggClass) { - this.aggClass = aggClass; - } - - /** - * @return the measureUniqueColumnNamesString - */ - public String getMeasureUniqueColumnNamesString() { - return measureUniqueColumnNamesString; - } - - /** * @param measureUniqueColumnNamesString the measureUniqueColumnNamesString to set */ public void setMeasureUniqueColumnNamesString(String measureUniqueColumnNamesString) { @@ -936,13 +790,6 @@ public class GraphConfigurationInfo { } /** - * @return the type - */ - public char[] getType() { - return type; - } - - /** * @param type the type to set */ public void setType(char[] type) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java index a2f0c59..559f363 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java +++ b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java @@ -295,10 +295,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface this.tableName = tableName; } - public String getAggregateLevels() { - return aggregateLevels; - } - public void setAggregateLevels(String aggregateLevels) { this.aggregateLevels = aggregateLevels; } @@ -307,14 +303,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface return complexTypes; } - public void setComplexTypes(Map<String, GenericDataType> complexTypes) { - this.complexTypes = complexTypes; - } - - public String getNumberOfCores() { - return numberOfCores; - } - public void setNumberOfCores(String numberOfCores) { this.numberOfCores = numberOfCores; } @@ -376,13 +364,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface } /** - * @return the complexTypeString - */ - public int getComplexTypeString() { - return Integer.parseInt(complexTypeString); - } - - /** * @param complexTypeString the complexTypeString to set */ public void setComplexTypeString(String complexTypeString) { @@ -417,10 +398,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface this.noDictionaryCount = noDictionaryCount; } - public String getColumnGroupsString() { - return this.columnGroupsString; - } - public void setColumnGroupsString(String columnGroups) { this.columnGroupsString = columnGroups; @@ -461,10 +438,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface return complexTypesMap; } - public String getMeasureDataType() { - return measureDataType; - } - public void setMeasureDataType(String measureDataType) { this.measureDataType = measureDataType; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileManager.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileManager.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileManager.java index 7182701..8cece09 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileManager.java +++ b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/file/FileManager.java @@ -46,13 +46,6 @@ public class FileManager implements IFileManagerComposite { return listOfFileData.get(i); } - /** - * Renames the File/Folders - */ - public boolean rename(IFileManagerComposite composite) { - return false; - } - @Override public void setName(String name) { this.fileName = name; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java b/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java index d98d40a..68e7d2a 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java +++ b/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java @@ -22,8 +22,6 @@ package org.apache.carbondata.processing.merger.step; import java.util.List; import java.util.Map; -import org.apache.carbondata.core.constants.CarbonCommonConstants; - import org.pentaho.di.core.CheckResult; import org.pentaho.di.core.CheckResultInterface; import org.pentaho.di.core.Counter; @@ -357,15 +355,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta } /** - * This method will return mdkey size - * - * @return mdkey size - */ - public String getMdkeySize() { - return mdkeySize; - } - - /** * This method will be used to set the mdkey * * @param mdkeySize @@ -375,15 +364,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta } /** - * This method will be used to get the measure count - * - * @return measure count - */ - public String getMeasureCount() { - return measureCount; - } - - /** * This method will be used to set the measure count * * @param measureCount @@ -393,15 +373,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta } /** - * This method will be used to get the heir and its key suze string - * - * @return heirAndKeySize - */ - public String getHeirAndKeySize() { - return heirAndKeySize; - } - - /** * This method will be used to set the heir and key size string * * @param heirAndKeySize @@ -439,13 +410,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta } /** - * @return the isGroupByEnabled - */ - public boolean isGroupByEnabled() { - return Boolean.parseBoolean(groupByEnabled); - } - - /** * @param isGroupByEnabled the isGroupByEnabled to set */ public void setGroupByEnabled(String isGroupByEnabled) { @@ -453,27 +417,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta } /** - * @return the aggregators - */ - public String[] getAggregators() { - return aggregatorString.split(CarbonCommonConstants.HASH_SPC_CHARACTER); - } - - /** - * @return the aggregatorClass - */ - public String[] getAggregatorClass() { - return aggregatorClassString.split(CarbonCommonConstants.HASH_SPC_CHARACTER); - } - - /** - * @return the aggregatorString - */ - public String getAggregatorString() { - return aggregatorString; - } - - /** * @param aggregatorString the aggregatorString to set */ public void setAggregatorString(String aggregatorString) { @@ -481,13 +424,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta } /** - * @return the aggregatorClassString - */ - public String getAggregatorClassString() { - return aggregatorClassString; - } - - /** * @param aggregatorClassString the aggregatorClassString to set */ public void setAggregatorClassString(String aggregatorClassString) { @@ -495,35 +431,17 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta } /** - * @return the factDimLensString - */ - public String getFactDimLensString() { - return factDimLensString; - } - - /** * @param factDimLensString1 the factDimLensString to set */ public void setFactDimLensString(String factDimLensString1) { this.factDimLensString = factDimLensString1; } - public String getLevelAnddataTypeString() { - return levelAnddataTypeString; - } - public void setLevelAnddataTypeString(String levelAnddataTypeString) { this.levelAnddataTypeString = levelAnddataTypeString; } /** - * @return partitionId - */ - public String getPartitionID() { - return partitionID; - } - - /** * @param partitionID */ public void setPartitionID(String partitionID) { @@ -531,14 +449,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta } /** - * return segmentId - * @return - */ - public int getSegmentId() { - return Integer.parseInt(segmentId); - } - - /** * set segment Id * @param segmentId */ http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java index 692e5d7..9e8f5b0 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java +++ b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java @@ -211,10 +211,6 @@ public class CarbonLoadModel implements Serializable { return factFilesToProcess; } - public void setFactFilesToProcess(List<String> factFilesToProcess) { - this.factFilesToProcess = factFilesToProcess; - } - public String getCsvHeader() { return csvHeader; } @@ -402,13 +398,6 @@ public class CarbonLoadModel implements Serializable { } /** - * @return the aggTables - */ - public String[] getAggTables() { - return aggTables; - } - - /** * @param aggTables the aggTables to set */ public void setAggTables(String[] aggTables) { @@ -467,13 +456,6 @@ public class CarbonLoadModel implements Serializable { } /** - * @param isRetentionRequest - */ - public void setRetentionRequest(boolean isRetentionRequest) { - this.isRetentionRequest = isRetentionRequest; - } - - /** * getLoadMetadataDetails. * * @return http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java index 26300d6..79e17e2 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java @@ -133,14 +133,6 @@ public class CarbonDataLoadConfiguration { dataLoadProperties.put(key, value); } - public Object getDataLoadProperty(String key, Object defaultValue) { - Object value = dataLoadProperties.get(key); - if (value == null) { - value = defaultValue; - } - return value; - } - public Object getDataLoadProperty(String key) { return dataLoadProperties.get(key); } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/newflow/DataField.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataField.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataField.java index c5dc5d1..07b4fa1 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataField.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataField.java @@ -21,7 +21,6 @@ package org.apache.carbondata.processing.newflow; import java.io.Serializable; -import org.apache.carbondata.core.carbon.metadata.blocklet.compressor.CompressionCodec; import org.apache.carbondata.core.carbon.metadata.encoder.Encoding; import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonColumn; @@ -36,8 +35,6 @@ public class DataField implements Serializable { private CarbonColumn column; - private CompressionCodec compressionCodec; - private String dateFormat; public boolean hasDictionaryEncoding() { @@ -48,14 +45,6 @@ public class DataField implements Serializable { return column; } - public CompressionCodec getCompressionCodec() { - return compressionCodec; - } - - public void setCompressionCodec(CompressionCodec compressionCodec) { - this.compressionCodec = compressionCodec; - } - public String getDateFormat() { return dateFormat; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/newflow/iterator/InputIterator.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/iterator/InputIterator.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/iterator/InputIterator.java deleted file mode 100644 index a87aaa2..0000000 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/iterator/InputIterator.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.processing.newflow.iterator; - -import java.util.Iterator; - -/** - * It iterates the data of record readers - */ -public abstract class InputIterator<E> implements Iterator<E> { - - @Override - public void remove() { - throw new UnsupportedOperationException("Not permitted"); - } - - /** - * Initialize the iterator - */ - public abstract void initialize(); - - /** - * Close the resources - */ - public abstract void close(); -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java index 5d12ec4..036d15a 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java @@ -64,10 +64,7 @@ public class ParallelReadMergeSorterImpl implements Sorter { private SingleThreadFinalSortFilesMerger finalMerger; - private DataField[] inputDataFields; - public ParallelReadMergeSorterImpl(DataField[] inputDataFields) { - this.inputDataFields = inputDataFields; } @Override @@ -187,15 +184,12 @@ public class ParallelReadMergeSorterImpl implements Sorter { private SortDataRows sortDataRows; - private SortParameters parameters; - private Object[][] buffer; public SortIteratorThread(Iterator<CarbonRowBatch> iterator, SortDataRows sortDataRows, SortParameters parameters, int batchSize) { this.iterator = iterator; this.sortDataRows = sortDataRows; - this.parameters = parameters; this.buffer = new Object[batchSize][]; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/UnsafeParallelReadMergeSorterImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/UnsafeParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/UnsafeParallelReadMergeSorterImpl.java index 3a29647..4504f0c 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/UnsafeParallelReadMergeSorterImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/UnsafeParallelReadMergeSorterImpl.java @@ -64,10 +64,7 @@ public class UnsafeParallelReadMergeSorterImpl implements Sorter { private UnsafeSingleThreadFinalSortFilesMerger finalMerger; - private DataField[] inputDataFields; - public UnsafeParallelReadMergeSorterImpl(DataField[] inputDataFields) { - this.inputDataFields = inputDataFields; } @Override public void initialize(SortParameters sortParameters) { @@ -180,15 +177,12 @@ public class UnsafeParallelReadMergeSorterImpl implements Sorter { private UnsafeSortDataRows sortDataRows; - private SortParameters parameters; - private Object[][] buffer; public SortIteratorThread(Iterator<CarbonRowBatch> iterator, UnsafeSortDataRows sortDataRows, SortParameters parameters, int batchSize) { this.iterator = iterator; this.sortDataRows = sortDataRows; - this.parameters = parameters; this.buffer = new Object[batchSize][]; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java index 7296e74..0adce19 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java @@ -20,9 +20,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.math.BigDecimal; import java.util.Arrays; -import java.util.Iterator; -import org.apache.carbondata.common.CarbonIterator; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.memory.MemoryBlock; import org.apache.carbondata.core.unsafe.CarbonUnsafe; @@ -73,10 +71,6 @@ public class UnsafeCarbonRowPage { lastSize = lastSize + size; } - public Iterator<Object[]> getIterator() { - return new UnsafeIterator(); - } - private int addRow(Object[] row, long address) { if (row == null) { throw new RuntimeException("Row is null ??"); @@ -310,30 +304,6 @@ public class UnsafeCarbonRowPage { return dataBlock; } - class UnsafeIterator extends CarbonIterator<Object[]> { - - private int counter; - - private int actualSize; - - public UnsafeIterator() { - this.actualSize = buffer.getActualSize(); - } - - @Override public boolean hasNext() { - if (counter < actualSize) { - return true; - } - return false; - } - - @Override public Object[] next() { - long address = buffer.get(counter); - counter++; - return getRow(address + dataBlock.getBaseOffset()); - } - } - public static void set(long[] words, int index) { int wordOffset = (index >> 6); words[wordOffset] |= (1L << index); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java index 9d73ba2..214e574 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java @@ -43,9 +43,6 @@ import org.apache.carbondata.processing.newflow.sort.unsafe.sort.TimSort; import org.apache.carbondata.processing.newflow.sort.unsafe.sort.UnsafeIntSortDataFormat; import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException; import org.apache.carbondata.processing.sortandgroupby.sortdata.SortParameters; -import org.apache.carbondata.processing.sortandgroupby.sortdata.SortTempFileChunkWriter; -import org.apache.carbondata.processing.sortandgroupby.sortdata.TempSortFileWriter; -import org.apache.carbondata.processing.sortandgroupby.sortdata.TempSortFileWriterFactory; import org.apache.carbondata.processing.util.CarbonDataProcessorUtil; public class UnsafeSortDataRows { @@ -201,24 +198,6 @@ public class UnsafeSortDataRows { startFileBasedMerge(); } - private void writeSortTempFile(Object[][] recordHolderList, int entryCountLocal, File file) - throws CarbonSortKeyAndGroupByException { - TempSortFileWriter writer = null; - - try { - writer = getWriter(); - writer.initiaize(file, entryCountLocal); - writer.writeSortTempFile(recordHolderList); - } catch (CarbonSortKeyAndGroupByException e) { - LOGGER.error(e, "Problem while writing the sort temp file"); - throw e; - } finally { - if (writer != null) { - writer.finish(); - } - } - } - private void writeData(UnsafeCarbonRowPage rowPage, File file) throws CarbonSortKeyAndGroupByException { DataOutputStream stream = null; @@ -242,24 +221,6 @@ public class UnsafeSortDataRows { } } - private TempSortFileWriter getWriter() { - TempSortFileWriter chunkWriter = null; - TempSortFileWriter writer = TempSortFileWriterFactory.getInstance() - .getTempSortFileWriter(parameters.isSortFileCompressionEnabled(), - parameters.getDimColCount(), parameters.getComplexDimColCount(), - parameters.getMeasureColCount(), parameters.getNoDictionaryCount(), - parameters.getFileWriteBufferSize()); - - if (parameters.isPrefetch() && !parameters.isSortFileCompressionEnabled()) { - chunkWriter = new SortTempFileChunkWriter(writer, parameters.getBufferSize()); - } else { - chunkWriter = - new SortTempFileChunkWriter(writer, parameters.getSortTempFileNoOFRecordsInCompression()); - } - - return chunkWriter; - } - /** * This method will be used to delete sort temp location is it is exites * http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java index 9f157a0..2c91f8f 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java @@ -76,10 +76,6 @@ public class UnsafeInmemoryMergeHolder implements Comparable<UnsafeInmemoryMerge return comparator.compare(currentRow, baseObject, o.getRow(), o.getBaseObject()); } - public int numberOfRows() { - return actualSize; - } - public Object getBaseObject() { return baseObject; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeInMemoryIntermediateDataMerger.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeInMemoryIntermediateDataMerger.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeInMemoryIntermediateDataMerger.java index 0d36d90..199a83e 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeInMemoryIntermediateDataMerger.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeInMemoryIntermediateDataMerger.java @@ -73,7 +73,6 @@ public class UnsafeInMemoryIntermediateDataMerger implements Callable<Void> { @Override public Void call() throws Exception { long intermediateMergeStartTime = System.currentTimeMillis(); int holderCounterConst = holderCounter; - boolean isFailed = false; try { startSorting(); while (hasNext()) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java index a142823..0eaccaf 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java @@ -45,11 +45,6 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec LogServiceFactory.getLogService(UnsafeSingleThreadFinalSortFilesMerger.class.getName()); /** - * lockObject - */ - private static final Object LOCKOBJECT = new Object(); - - /** * fileCounter */ private int fileCounter; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java index b979af6..23f100e 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java @@ -10,8 +10,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.apache.carbondata.common.CarbonIterator; -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep; import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration; @@ -27,9 +25,6 @@ import org.apache.carbondata.processing.newflow.row.CarbonRowBatch; */ public class InputProcessorStepImpl extends AbstractDataLoadProcessorStep { - private static final LogService LOGGER = - LogServiceFactory.getLogService(InputProcessorStepImpl.class.getName()); - private RowParser rowParser; private CarbonIterator<Object[]>[] inputIterators; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/schema/metadata/ColumnsInfo.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/schema/metadata/ColumnsInfo.java b/processing/src/main/java/org/apache/carbondata/processing/schema/metadata/ColumnsInfo.java index 43a4b8a..de94724 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/schema/metadata/ColumnsInfo.java +++ b/processing/src/main/java/org/apache/carbondata/processing/schema/metadata/ColumnsInfo.java @@ -216,38 +216,17 @@ public class ColumnsInfo { } /** - * @return Returns the batchSize. - */ - public int getBatchSize() { - return batchSize; - } - - /** * @param batchSize The batchSize to set. */ public void setBatchSize(int batchSize) { this.batchSize = batchSize; } - /** - * @return Returns the isInitialLoad. - */ - public boolean isAggregateLoad() { - return isAggregateLoad; - } - public void setAggregateLoad(boolean isAggregateLoad) { this.isAggregateLoad = isAggregateLoad; } /** - * @return Returns the storeType. - */ - public String getStoreType() { - return storeType; - } - - /** * @param storeType The storeType to set. */ public void setStoreType(String storeType) { @@ -297,13 +276,6 @@ public class ColumnsInfo { } /** - * @return Returns the propColumns. - */ - public List<String>[] getPropColumns() { - return propColumns; - } - - /** * @param propColumns The propColumns to set. */ public void setPropColumns(List<String>[] propColumns) { @@ -332,13 +304,6 @@ public class ColumnsInfo { } /** - * @param timDimIndexEnd The timDimIndexEnd to set. - */ - public void setTimDimIndexEnd(int timDimIndexEnd) { - this.timDimIndexEnd = timDimIndexEnd; - } - - /** * @return Returns the timeOrdinalIndices. */ public int[] getTimeOrdinalIndices() { @@ -353,13 +318,6 @@ public class ColumnsInfo { } /** - * @return Returns the timeOrdinalCols. - */ - public String[] getTimeOrdinalCols() { - return timeOrdinalCols; - } - - /** * @param timeOrdinalCols The timeOrdinalCols to set. */ public void setTimeOrdinalCols(String[] timeOrdinalCols) { @@ -367,13 +325,6 @@ public class ColumnsInfo { } /** - * @return Returns the propTypes. - */ - public List<String>[] getPropTypes() { - return propTypes; - } - - /** * @param propTypes The propTypes to set. */ public void setPropTypes(List<String>[] propTypes) { @@ -395,13 +346,6 @@ public class ColumnsInfo { } /** - * @return Returns the dimHierRel. - */ - public String[] getDimHierRel() { - return dimHierRel; - } - - /** * @param dimHierRel The dimHierRel to set. */ public void setDimHierRel(String[] dimHierRel) { @@ -432,22 +376,6 @@ public class ColumnsInfo { } /** - * setDimsPresent - * - * @param dimsPresent - */ - public void setDimsPresent(boolean[] dimsPresent) { - this.dimsPresent = dimsPresent; - } - - /** - * @return Returns the measureColumns. - */ - public String[] getMeasureColumns() { - return measureColumns; - } - - /** * @param measureColumns The measureColumns to set. */ public void setMeasureColumns(String[] measureColumns) { http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/schema/metadata/HierarchiesInfo.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/schema/metadata/HierarchiesInfo.java b/processing/src/main/java/org/apache/carbondata/processing/schema/metadata/HierarchiesInfo.java index d875cf2..2d51dab 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/schema/metadata/HierarchiesInfo.java +++ b/processing/src/main/java/org/apache/carbondata/processing/schema/metadata/HierarchiesInfo.java @@ -29,48 +29,10 @@ public class HierarchiesInfo { private String hierarichieName; /** - * columnIndex - */ - private int[] columnIndex; - - /** - * columnNames - */ - private String[] columnNames; - - /** * columnPropMap */ private Map<String, String[]> columnPropMap; - /** - * loadToHierarichiTable - */ - private boolean loadToHierarichiTable; - - /** - * query - */ - private String query; - - /** - * Is Time Dimension - */ - private boolean isTimeDimension; - - /** - * levelTypeColumnMap - */ - private Map<String, String> levelTypeColumnMap; - - public boolean isLoadToHierarichiTable() { - return loadToHierarichiTable; - } - - public void setLoadToHierarichiTable(boolean loadToHierarichiTable) { - this.loadToHierarichiTable = loadToHierarichiTable; - } - public String getHierarichieName() { return hierarichieName; } @@ -79,22 +41,6 @@ public class HierarchiesInfo { this.hierarichieName = hierarichieName; } - public int[] getColumnIndex() { - return columnIndex; - } - - public void setColumnIndex(int[] columnIndex) { - this.columnIndex = columnIndex; - } - - public String[] getColumnNames() { - return columnNames; - } - - public void setColumnNames(String[] columnNames) { - this.columnNames = columnNames; - } - public Map<String, String[]> getColumnPropMap() { return columnPropMap; } @@ -103,28 +49,5 @@ public class HierarchiesInfo { this.columnPropMap = columnPropMap; } - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public boolean isTimeDimension() { - return isTimeDimension; - } - - public void setTimeDimension(boolean isTimeDimension) { - this.isTimeDimension = isTimeDimension; - } - - public Map<String, String> getLevelTypeColumnMap() { - return levelTypeColumnMap; - } - - public void setLevelTypeColumnMap(Map<String, String> levelTypeColumnMap) { - this.levelTypeColumnMap = levelTypeColumnMap; - } }