Repository: carbondata Updated Branches: refs/heads/carbonstore-rebase5 7540cc9ca -> c305f309e (forced update)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c305f309/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java index d2faef5..142b2cb 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java @@ -21,7 +21,16 @@ import java.io.File; import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; @@ -32,7 +41,6 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.locks.ICarbonLock; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; -import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.mutate.CarbonUpdateUtil; import org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails; @@ -592,10 +600,6 @@ public final class CarbonDataMergerUtil { List<LoadMetadataDetails> segmentsToBeMerged = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); - CarbonTable carbonTable = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable(); - CarbonTableIdentifier tableIdentifier = carbonTable.getCarbonTableIdentifier(); - - // total length long totalLength = 0; @@ -1013,7 +1017,8 @@ public final class CarbonDataMergerUtil { CarbonFile[] updateDeltaFiles = null; Set<String> uniqueBlocks = new HashSet<String>(); - String segmentPath = CarbonTablePath.getSegmentPath(absoluteTableIdentifier.getTablePath(), seg.getSegmentNo()); + String segmentPath = CarbonTablePath.getSegmentPath( + absoluteTableIdentifier.getTablePath(), seg.getSegmentNo()); CarbonFile segDir = FileFactory.getCarbonFile(segmentPath, FileFactory.getFileType(segmentPath)); CarbonFile[] allSegmentFiles = segDir.listFiles(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/c305f309/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java index ea11e22..ebcf944 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java +++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java @@ -405,8 +405,8 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor { partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR + carbonLoadModel.getFactTimeStamp() + ".tmp"; } else { - carbonStoreLocation = CarbonDataProcessorUtil - .createCarbonStoreLocation(carbonLoadModel.getDatabaseName(), tableName, carbonLoadModel.getSegmentId()); + carbonStoreLocation = CarbonDataProcessorUtil.createCarbonStoreLocation( + carbonLoadModel.getDatabaseName(), tableName, carbonLoadModel.getSegmentId()); } CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel .getCarbonFactDataHandlerModel(carbonLoadModel, carbonTable, segmentProperties, tableName, http://git-wip-us.apache.org/repos/asf/carbondata/blob/c305f309/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java index 278d5bb..2616def 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java +++ b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java @@ -76,8 +76,8 @@ public class RowResultMergerProcessor extends AbstractResultProcessor { partitionSpec.getLocation().toString() + CarbonCommonConstants.FILE_SEPARATOR + loadModel .getFactTimeStamp() + ".tmp"; } else { - carbonStoreLocation = CarbonDataProcessorUtil - .createCarbonStoreLocation(loadModel.getDatabaseName(), tableName, loadModel.getSegmentId()); + carbonStoreLocation = CarbonDataProcessorUtil.createCarbonStoreLocation( + loadModel.getDatabaseName(), tableName, loadModel.getSegmentId()); } CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel .getCarbonFactDataHandlerModel(loadModel, carbonTable, segProp, tableName, http://git-wip-us.apache.org/repos/asf/carbondata/blob/c305f309/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java index df2e2a2..221697f 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java +++ b/processing/src/main/java/org/apache/carbondata/processing/partition/spliter/RowResultProcessor.java @@ -47,8 +47,8 @@ public class RowResultProcessor { CarbonDataProcessorUtil.createLocations(tempStoreLocation); this.segmentProperties = segProp; String tableName = carbonTable.getTableName(); - String carbonStoreLocation = CarbonDataProcessorUtil - .createCarbonStoreLocation(loadModel.getDatabaseName(), tableName, loadModel.getSegmentId()); + String carbonStoreLocation = CarbonDataProcessorUtil.createCarbonStoreLocation( + loadModel.getDatabaseName(), tableName, loadModel.getSegmentId()); CarbonFactDataHandlerModel carbonFactDataHandlerModel = CarbonFactDataHandlerModel.getCarbonFactDataHandlerModel(loadModel, carbonTable, segProp, tableName, tempStoreLocation, carbonStoreLocation); http://git-wip-us.apache.org/repos/asf/carbondata/blob/c305f309/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java index dc8ffd7..19ad47d 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java @@ -370,7 +370,8 @@ public final class CarbonDataProcessorUtil { * * @return data directory path */ - public static String createCarbonStoreLocation(String databaseName, String tableName, String segmentId) { + public static String createCarbonStoreLocation(String databaseName, String tableName, + String segmentId) { CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(databaseName, tableName); return CarbonTablePath.getSegmentPath(carbonTable.getTablePath(), segmentId); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c305f309/store/sdk/pom.xml ---------------------------------------------------------------------- diff --git a/store/sdk/pom.xml b/store/sdk/pom.xml index 54fba55..b3dd464 100644 --- a/store/sdk/pom.xml +++ b/store/sdk/pom.xml @@ -7,7 +7,7 @@ <parent> <groupId>org.apache.carbondata</groupId> <artifactId>carbondata-parent</artifactId> - <version>1.3.0-SNAPSHOT</version> + <version>1.4.0-SNAPSHOT</version> <relativePath>../../pom.xml</relativePath> </parent> http://git-wip-us.apache.org/repos/asf/carbondata/blob/c305f309/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java ---------------------------------------------------------------------- diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java index dc5696a..df6afc6 100644 --- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java +++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CSVCarbonWriter.java @@ -23,7 +23,7 @@ import java.util.UUID; import org.apache.carbondata.common.annotations.InterfaceAudience; import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat; -import org.apache.carbondata.processing.loading.csvinput.StringArrayWritable; +import org.apache.carbondata.hadoop.internal.ObjectArrayWritable; import org.apache.carbondata.processing.loading.model.CarbonLoadModel; import org.apache.hadoop.conf.Configuration; @@ -42,9 +42,9 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; @InterfaceAudience.Internal class CSVCarbonWriter extends CarbonWriter { - private RecordWriter<NullWritable, StringArrayWritable> recordWriter; + private RecordWriter<NullWritable, ObjectArrayWritable> recordWriter; private TaskAttemptContext context; - private StringArrayWritable writable; + private ObjectArrayWritable writable; CSVCarbonWriter(CarbonLoadModel loadModel) throws IOException { Configuration hadoopConf = new Configuration(); @@ -57,7 +57,7 @@ class CSVCarbonWriter extends CarbonWriter { TaskAttemptContextImpl context = new TaskAttemptContextImpl(hadoopConf, attemptID); this.recordWriter = format.getRecordWriter(context); this.context = context; - this.writable = new StringArrayWritable(); + this.writable = new ObjectArrayWritable(); } /**
