Corrected file headers and removed unnecessary open csv files
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/be1675f4 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/be1675f4 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/be1675f4 Branch: refs/heads/branch-0.1 Commit: be1675f4948f02eb362830657bea67a5143bd154 Parents: 17b00c2 Author: ravipesala <ravi.pes...@gmail.com> Authored: Wed Sep 21 21:30:55 2016 +0530 Committer: ravipesala <ravi.pes...@gmail.com> Committed: Thu Sep 22 10:35:19 2016 +0530 ---------------------------------------------------------------------- .../carbondata/scan/QueryExecutor_UT.java | 35 -- .../spark/partition/api/DataPartitioner.java | 9 - .../spark/partition/api/Partition.java | 9 - .../partition/api/impl/CSVFilePartitioner.java | 365 ------------ .../api/impl/DataPartitionerProperties.java | 3 - .../partition/api/impl/DefaultLoadBalancer.java | 9 - .../spark/partition/api/impl/PartitionImpl.java | 9 - .../api/impl/QueryPartitionHelper.java | 9 - .../spark/partition/reader/CSVIterator.java | 74 --- .../spark/partition/reader/CSVParser.java | 559 ------------------- .../spark/partition/reader/CSVReader.java | 496 ---------------- .../spark/partition/reader/CSVWriter.java | 396 ------------- .../spark/partition/reader/LineReader.java | 68 --- .../spark/partition/reader/ResultSetHelper.java | 87 --- .../reader/ResultSetHelperService.java | 327 ----------- .../spark/rdd/CarbonDataPartitionRDD.scala | 112 ---- .../spark/rdd/CarbonDataRDDFactory.scala | 31 - .../spark/rdd/CarbonGlobalDictionaryRDD.scala | 7 +- .../spark/util/GlobalDictionaryUtil.scala | 23 +- .../org/apache/spark/sql/CarbonContext.scala | 42 +- .../execution/command/carbonTableSchema.scala | 50 -- 21 files changed, 20 insertions(+), 2700 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/core/src/test/java/org/apache/carbondata/scan/QueryExecutor_UT.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/carbondata/scan/QueryExecutor_UT.java b/core/src/test/java/org/apache/carbondata/scan/QueryExecutor_UT.java deleted file mode 100644 index 88fa999..0000000 --- a/core/src/test/java/org/apache/carbondata/scan/QueryExecutor_UT.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * Copyright Notice - * ===================================== - * This file contains proprietary information of - * Huawei Technologies India Pvt Ltd. - * Copying or reproduction without prior written approval is prohibited. - * Copyright (c) 2012 - * ===================================== - */ -package org.apache.carbondata.scan; - -import junit.framework.TestCase; - -public class QueryExecutor_UT extends TestCase { - -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/DataPartitioner.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/DataPartitioner.java b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/DataPartitioner.java index 603e01e..58f3a2d 100644 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/DataPartitioner.java +++ b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/DataPartitioner.java @@ -17,15 +17,6 @@ * under the License. */ -/** - * Copyright Notice - * ===================================== - * This file contains proprietary information of - * Huawei Technologies India Pvt Ltd. - * Copying or reproduction without prior written approval is prohibited. - * Copyright (c) 1997 - * ===================================== - */ package org.apache.carbondata.spark.partition.api; import java.util.List; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/Partition.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/Partition.java b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/Partition.java index 5d94fa0..61639d3 100644 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/Partition.java +++ b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/Partition.java @@ -17,15 +17,6 @@ * under the License. */ -/** - * Copyright Notice - * ===================================== - * This file contains proprietary information of - * Huawei Technologies India Pvt Ltd. - * Copying or reproduction without prior written approval is prohibited. - * Copyright (c) 1997 - * ===================================== - */ package org.apache.carbondata.spark.partition.api; import java.io.Serializable; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/CSVFilePartitioner.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/CSVFilePartitioner.java b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/CSVFilePartitioner.java deleted file mode 100644 index c61aafa..0000000 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/CSVFilePartitioner.java +++ /dev/null @@ -1,365 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * Copyright Notice - * ===================================== - * This file contains proprietary information of - * Huawei Technologies India Pvt Ltd. - * Copying or reproduction without prior written approval is prohibited. - * Copyright (c) 1997 - * ===================================== - */ -package org.apache.carbondata.spark.partition.api.impl; - -import java.io.DataInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.zip.GZIPInputStream; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile; -import org.apache.carbondata.core.datastorage.store.impl.FileFactory; -import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType; -import org.apache.carbondata.core.util.CarbonProperties; -import org.apache.carbondata.core.util.CarbonUtil; -import org.apache.carbondata.processing.surrogatekeysgenerator.csvbased.BadRecordslogger; -import org.apache.carbondata.spark.partition.api.DataPartitioner; -import org.apache.carbondata.spark.partition.api.Partition; -import org.apache.carbondata.spark.partition.reader.CSVParser; -import org.apache.carbondata.spark.partition.reader.CSVReader; -import org.apache.carbondata.spark.partition.reader.CSVWriter; - -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; -import org.apache.commons.lang.StringUtils; -import org.apache.spark.sql.execution.command.Partitioner; - -/** - * Split the CSV file into the number of partitions using the given partition information - */ -public class CSVFilePartitioner { - - private static final LogService LOGGER = - LogServiceFactory.getLogService(CSVFilePartitioner.class.getName()); - private String partitionerClass; - private String sourceFilesBasePath; - private boolean partialSuccess; - /** - * badRecordslogger - */ - private BadRecordslogger badRecordslogger; - - /** - * @param partitionerClass - */ - public CSVFilePartitioner(String partitionerClass, String sourceFilesBasePath) { - this.partitionerClass = partitionerClass; - this.sourceFilesBasePath = sourceFilesBasePath; - } - - public boolean isPartialSuccess() { - return partialSuccess; - } - - @Deprecated public void splitFile(String databaseName, String tableName, - List<String> sourceFilePath, String targetFolder, List<String> nodes, int partitionCount, - String[] partitionColumn, String[] requiredColumns, String delimiter, String quoteChar, - String fileHeader, String escapeChar, boolean multiLine) throws Exception { - LOGGER - .info("Processing file split: " + sourceFilePath); - - // Create the target folder - FileFactory.mkdirs(targetFolder, FileFactory.getFileType(targetFolder)); - - String[] headerColumns = null; - - HashMap<Partition, CSVWriter> outputStreamsMap = - new HashMap<Partition, CSVWriter>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); - - String key = databaseName + '_' + tableName; - badRecordslogger = new BadRecordslogger(key, "Partition_" + System.currentTimeMillis() + ".log", - getBadLogStoreLocation("partition/" + databaseName + '/' + tableName)); - - CSVReader dataInputStream = null; - - long recordCounter = 0; - - CSVParser customParser = getCustomParser(delimiter, quoteChar, escapeChar); - - for (int i = 0; i < sourceFilePath.size(); i++) { - try { - CarbonFile file = FileFactory - .getCarbonFile(sourceFilePath.get(i), FileFactory.getFileType(sourceFilePath.get(i))); - // File file = new File(sourceFilePath); - String fileAbsolutePath = file.getAbsolutePath(); - String fileName = null; - if (!sourceFilesBasePath.endsWith(".csv") && fileAbsolutePath - .startsWith(sourceFilesBasePath)) { - if (sourceFilesBasePath.endsWith(File.separator)) { - fileName = fileAbsolutePath.substring(sourceFilesBasePath.length()) - .replace(File.separator, "_"); - } else { - fileName = fileAbsolutePath.substring(sourceFilesBasePath.length() + 1) - .replace(File.separator, "_"); - } - } else { - fileName = file.getName(); - } - - // Read and prepare columns from first row in file - DataInputStream inputStream = FileFactory.getDataInputStream(sourceFilePath.get(i), - FileFactory.getFileType(sourceFilePath.get(i))); - if (fileName.endsWith(".gz")) { - GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream); - dataInputStream = - new CSVReader(new InputStreamReader(gzipInputStream, Charset.defaultCharset()), - CSVReader.DEFAULT_SKIP_LINES, customParser); - fileName = fileName.substring(0, fileName.indexOf(".gz")); - } else if (fileName.endsWith(".bz2")) { - BZip2CompressorInputStream stream = new BZip2CompressorInputStream(inputStream); - dataInputStream = new CSVReader(new InputStreamReader(stream, Charset.defaultCharset()), - CSVReader.DEFAULT_SKIP_LINES, customParser); - fileName = fileName.substring(0, fileName.indexOf(".bz2")); - } else if (fileName.endsWith(".csv")) { - dataInputStream = - new CSVReader(new InputStreamReader(inputStream, Charset.defaultCharset()), - CSVReader.DEFAULT_SKIP_LINES, customParser); - fileName = fileName.substring(0, fileName.indexOf(".csv")); - } else { - LOGGER.info("Processing file split: Unsupported File Extension: Skipping File : " - + file.getAbsolutePath()); - partialSuccess = true; - return; - } - dataInputStream.setBadRecordsLogger(badRecordslogger); - if (fileHeader == null || fileHeader.length() == 0) { - headerColumns = dataInputStream.readNext(); - } else { - headerColumns = fileHeader.split(","); - } - if (null == headerColumns) { - LOGGER.info("Csv file does not contain the header column neither the headers are " - + "passed in DDL or API. Skipping file :: " + sourceFilePath); - partialSuccess = true; - return; - } - int[] indexes = pruneColumnsAndGetIndexes(headerColumns, requiredColumns); - - // In case there is a dummy measure required columns length and - // header columns length will not be equal - if ((null == fileHeader || 0 == fileHeader.length()) && (0 == indexes.length) && ( - fileHeader.length() != indexes.length)) { - LOGGER.info("Column headers are invalid. They do not match with the schema headers." - + "Skipping file :: " + sourceFilePath); - partialSuccess = true; - return; - } - - partitionData(targetFolder, nodes, partitionCount, partitionColumn, headerColumns, - outputStreamsMap, dataInputStream, recordCounter, fileName, indexes, fileAbsolutePath); - } catch (IOException e) { - LOGGER.error(e, e.getMessage()); - } finally { - CarbonUtil.closeStreams(dataInputStream); - - for (CSVWriter dataOutStream : outputStreamsMap.values()) { - CarbonUtil.closeStreams(dataOutStream); - } - badRecordslogger.closeStreams(); - } - } - } - - private void partitionData(String targetFolder, List<String> nodes, int partitionCount, - String[] partitionColumn, String[] headerColumns, - HashMap<Partition, CSVWriter> outputStreamsMap, CSVReader dataInputStream, long recordCounter, - String fileName, int[] indexes, String fileAbsolutePath) - throws InstantiationException, IllegalAccessException, ClassNotFoundException, IOException { - DataPartitioner dataPartitioner = - getDataPartitioner(targetFolder, nodes, partitionCount, partitionColumn, headerColumns); - - //Get partitions and create output streams - List<Partition> allPartitions = dataPartitioner.getAllPartitions(); - - loopPartitionsAndPopulateOutStreamMap(outputStreamsMap, fileName, allPartitions); - - //Write header in all the target files - for (CSVWriter dataOutStream : outputStreamsMap.values()) { - dataOutStream.writeNext(pruneColumns(headerColumns, indexes)); - } - - recordCounter = writeTargetStream(outputStreamsMap, dataInputStream, recordCounter, indexes, - dataPartitioner, headerColumns, fileAbsolutePath); - - LOGGER - .info("Processed Record count: " + recordCounter); - } - - private CSVParser getCustomParser(String delimiter, String quoteChar, String escapeChar) { - CSVParser customParser = null; - boolean ignoreQuote = false; - boolean ignoreEscape = false; - char defaultQuoteChar = CSVParser.DEFAULT_QUOTE_CHARACTER; - char defaultEscapeChar = CSVParser.DEFAULT_ESCAPE_CHARACTER; - if (quoteChar == null || quoteChar.isEmpty() || quoteChar.trim().isEmpty()) { - ignoreQuote = true; - } else { - ignoreQuote = false; - defaultQuoteChar = quoteChar.charAt(0); - } - if (escapeChar == null || escapeChar.isEmpty() || escapeChar.trim().isEmpty()) { - ignoreEscape = true; - } else { - ignoreEscape = false; - defaultEscapeChar = escapeChar.charAt(0); - } - delimiter = CarbonUtil.unescapeChar(delimiter); - customParser = new CSVParser(delimiter.charAt(0), defaultQuoteChar, defaultEscapeChar, - CSVParser.DEFAULT_STRICT_QUOTES, CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE, ignoreQuote, - ignoreEscape); - return customParser; - } - - private DataPartitioner getDataPartitioner(String targetFolder, List<String> nodes, - int partitionCount, String[] partitionColumn, String[] headerColumns) - throws InstantiationException, IllegalAccessException, ClassNotFoundException { - DataPartitioner dataPartitioner = - (DataPartitioner) Class.forName(partitionerClass).newInstance(); - - Partitioner partitioner = new Partitioner(partitionerClass, partitionColumn, partitionCount, - nodes.toArray(new String[nodes.size()])); - //Initialise the partitioner - dataPartitioner.initialize(targetFolder, headerColumns, partitioner); - return dataPartitioner; - } - - private long writeTargetStream(HashMap<Partition, CSVWriter> outputStreamsMap, - CSVReader dataInputStream, long recordCounter, int[] indexes, DataPartitioner dataPartitioner, - String[] headerColumns, String fileAbsolutePath) throws IOException { - String[] record = null; - Partition tartgetPartition = null; - CSVWriter targetStream = null; - record = dataInputStream.readNext(); - int skippedLines = 0; - if (null == record) { - return recordCounter; - } else { - boolean isEqual = compareHeaderColumnWithFirstRecordInCSV(headerColumns, record); - if (isEqual) { - record = dataInputStream.readNext(); - recordCounter++; - } - } - while (null != record) { - tartgetPartition = dataPartitioner.getPartionForTuple(record, recordCounter); - targetStream = outputStreamsMap.get(tartgetPartition); - try { - targetStream.writeNext(pruneColumns(record, indexes)); - } catch (ArrayIndexOutOfBoundsException e) { - partialSuccess = true; - skippedLines++; - badRecordslogger.addBadRecordsToBilder(record, record.length, - "No. of columns not matched with table columns", null); - LOGGER.error("BAD Record Found: No. of columns not matched with table columns, " - + "Skipping line: (" + (recordCounter + 1) + ") in File :" + fileAbsolutePath); - } catch (Exception e) { - partialSuccess = true; - skippedLines++; - badRecordslogger.addBadRecordsToBilder(record, record.length, e.getMessage(), null); - LOGGER.info("Exception while processing the record at line " + (recordCounter + 1) - + " in partiton " + tartgetPartition.getUniqueID()); - } finally { - record = dataInputStream.readNext(); - recordCounter++; - } - } - if (skippedLines != 0) { - LOGGER.info("No. of bad records skipped: (" + skippedLines + ") in file:" + fileAbsolutePath); - } - return recordCounter; - } - - private boolean compareHeaderColumnWithFirstRecordInCSV(String[] headerColumns, - String[] firstRecord) { - String header = StringUtils.join(headerColumns, ','); - String record = StringUtils.join(firstRecord, ','); - if (header != null && header.equals(record)) { - return true; - } - return false; - } - - private void loopPartitionsAndPopulateOutStreamMap(HashMap<Partition, CSVWriter> outputStreamsMap, - String fileName, List<Partition> allPartitions) throws IOException { - for (Partition partition : allPartitions) { - String targetFolderPath = partition.getFilePath(); - FileType fileType = FileFactory.getFileType(targetFolderPath); - FileFactory.mkdirs(targetFolderPath, fileType); - outputStreamsMap.put(partition, new CSVWriter(new OutputStreamWriter(FileFactory - .getDataOutputStream( - targetFolderPath + '/' + fileName + '_' + partition.getUniqueID() + ".csv", fileType, - (short) 1), Charset.defaultCharset()))); - } - } - - private int[] pruneColumnsAndGetIndexes(String[] headerColumns, String[] requiredColumns) { - if (requiredColumns == null) { - requiredColumns = headerColumns; - } - List<Integer> indexesList = new ArrayList<Integer>(); - for (int i = 0; i < headerColumns.length; i++) { - for (int j = 0; j < requiredColumns.length; j++) { - if (headerColumns[i].equalsIgnoreCase(requiredColumns[j])) { - indexesList.add(i); - break; - } - } - } - int[] indexes = new int[indexesList.size()]; - for (int i = 0; i < indexesList.size(); i++) { - indexes[i] = indexesList.get(i); - } - return indexes; - } - - private String[] pruneColumns(String[] tuple, int[] indexes) { - String[] sb = new String[indexes.length]; - int length = indexes.length; - for (int i = 0; i < length; i++) { - sb[i] = tuple[indexes[i]]; - } - return sb; - } - - private String getBadLogStoreLocation(String storeLocation) { - String badLogStoreLocation = - CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC); - badLogStoreLocation = badLogStoreLocation + File.separator + storeLocation; - - return badLogStoreLocation; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/DataPartitionerProperties.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/DataPartitionerProperties.java b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/DataPartitionerProperties.java index 8a80c4e..bc6e54f 100644 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/DataPartitionerProperties.java +++ b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/DataPartitionerProperties.java @@ -17,9 +17,6 @@ * under the License. */ -/** - * - */ package org.apache.carbondata.spark.partition.api.impl; import java.io.File; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/DefaultLoadBalancer.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/DefaultLoadBalancer.java b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/DefaultLoadBalancer.java index 67c257d..9bee8a2 100644 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/DefaultLoadBalancer.java +++ b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/DefaultLoadBalancer.java @@ -17,15 +17,6 @@ * under the License. */ -/** - * Copyright Notice - * ===================================== - * This file contains proprietary information of - * Huawei Technologies India Pvt Ltd. - * Copying or reproduction without prior written approval is prohibited. - * Copyright (c) 1997 - * ===================================== - */ package org.apache.carbondata.spark.partition.api.impl; import java.util.ArrayList; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/PartitionImpl.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/PartitionImpl.java b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/PartitionImpl.java index b49db59..bd7cc42 100644 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/PartitionImpl.java +++ b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/PartitionImpl.java @@ -17,15 +17,6 @@ * under the License. */ -/** - * Copyright Notice - * ===================================== - * This file contains proprietary information of - * Huawei Technologies India Pvt Ltd. - * Copying or reproduction without prior written approval is prohibited. - * Copyright (c) 1997 - * ===================================== - */ package org.apache.carbondata.spark.partition.api.impl; import java.util.List; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/QueryPartitionHelper.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/QueryPartitionHelper.java b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/QueryPartitionHelper.java index 37d772d..72e7b08 100644 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/QueryPartitionHelper.java +++ b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/api/impl/QueryPartitionHelper.java @@ -17,15 +17,6 @@ * under the License. */ -/** - * Copyright Notice - * ===================================== - * This file contains proprietary information of - * Huawei Technologies India Pvt Ltd. - * Copying or reproduction without prior written approval is prohibited. - * Copyright (c) 1997 - * ===================================== - */ package org.apache.carbondata.spark.partition.api.impl; import java.io.File; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVIterator.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVIterator.java b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVIterator.java deleted file mode 100644 index d57f708..0000000 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVIterator.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.partition.reader; - -import java.io.IOException; -import java.util.Iterator; - -/** - * Provides an Iterator over the data found in opencsv. - */ -public class CSVIterator implements Iterator<String[]> { - private CSVReader reader; - private String[] nextLine; - - /** - * @param reader reader for the csv data. - * @throws IOException if unable to read data from the reader. - */ - public CSVIterator(CSVReader reader) throws IOException { - this.reader = reader; - nextLine = reader.readNext(); - } - - /** - * Returns true if the iteration has more elements. - * In other words, returns true if next() would return an element rather - * than throwing an exception. - * - * @return true if the CSVIterator has more elements. - */ - public boolean hasNext() { - return nextLine != null; - } - - /** - * Returns the next elenebt in the iterator. - * - * @return The next element of the iterator. - */ - public String[] next() { - String[] temp = nextLine; - try { - nextLine = reader.readNext(); - } catch (IOException e) { - throw new RuntimeException(e); - } - return temp; - } - - /** - * This method is not supported by openCSV and will throw a UnsupportedOperationException - * if called. - */ - public void remove() { - throw new UnsupportedOperationException("This is a read only iterator."); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVParser.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVParser.java b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVParser.java deleted file mode 100644 index a052f31..0000000 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVParser.java +++ /dev/null @@ -1,559 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.partition.reader; - -/** - * Copyright 2005 Bytecode Pty Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.carbondata.core.constants.CarbonCommonConstants; - -import org.apache.commons.lang3.StringUtils; - -/** - * A very simple CSV parser released under a commercial-friendly license. - * This just implements splitting a single line into fields. - * - * @author Glen Smith - * @author Rainer Pruy - */ -public class CSVParser { - - /** - * The default separator to use if none is supplied to the constructor. - */ - public static final char DEFAULT_SEPARATOR = ','; - /** - * The average size of a line read by openCSV (used for setting the size of StringBuilders). - */ - public static final int INITIAL_READ_SIZE = 128; - /** - * The default quote character to use if none is supplied to the - * constructor. - */ - public static final char DEFAULT_QUOTE_CHARACTER = '"'; - /** - * The default escape character to use if none is supplied to the - * constructor. - */ - public static final char DEFAULT_ESCAPE_CHARACTER = '\\'; - /** - * The default strict quote behavior to use if none is supplied to the - * constructor. - */ - public static final boolean DEFAULT_STRICT_QUOTES = false; - /** - * The default leading whitespace behavior to use if none is supplied to the - * constructor. - */ - public static final boolean DEFAULT_IGNORE_LEADING_WHITESPACE = true; - /** - * If the quote character is set to null then there is no quote character. - */ - public static final boolean DEFAULT_IGNORE_QUOTATIONS = false; - /** - * This is the "null" character - if a value is set to this then it is ignored. - */ - public static final char NULL_CHARACTER = '\0'; - /** - * This is the character that the CSVParser will treat as the separator. - */ - private final char separator; - /** - * This is the character that the CSVParser will treat as the quotation character. - */ - private final char quotechar; - /** - * This is the character that the CSVParser will treat as the escape character. - */ - private final char escape; - /** - * Determines if the field is between quotes (true) or between separators (false). - */ - private final boolean strictQuotes; - /** - * Ignore any leading white space at the start of the field. - */ - private final boolean ignoreLeadingWhiteSpace; - /** - * Skip over quotation characters when parsing. - */ - private final boolean ignoreQuotations; - private boolean ignoreEscapeChar; - private String pending; - private boolean inField; - private int charCountInsideQuote; - - /** - * Constructs CSVParser using a comma for the separator. - */ - public CSVParser() { - this(DEFAULT_SEPARATOR, DEFAULT_QUOTE_CHARACTER, DEFAULT_ESCAPE_CHARACTER); - } - - /** - * Constructs CSVParser with supplied separator. - * - * @param separator the delimiter to use for separating entries. - */ - public CSVParser(char separator) { - this(separator, DEFAULT_QUOTE_CHARACTER, DEFAULT_ESCAPE_CHARACTER); - } - - /** - * Constructs CSVParser with supplied separator and quote char. - * - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - */ - public CSVParser(char separator, char quotechar) { - this(separator, quotechar, DEFAULT_ESCAPE_CHARACTER); - } - - /** - * Constructs CSVReader with supplied separator and quote char. - * - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param escape the character to use for escaping a separator or quote - */ - public CSVParser(char separator, char quotechar, char escape) { - this(separator, quotechar, escape, DEFAULT_STRICT_QUOTES); - } - - /** - * Constructs CSVParser with supplied separator and quote char. - * Allows setting the "strict quotes" flag - * - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param escape the character to use for escaping a separator or quote - * @param strictQuotes if true, characters outside the quotes are ignored - */ - public CSVParser(char separator, char quotechar, char escape, boolean strictQuotes) { - this(separator, quotechar, escape, strictQuotes, DEFAULT_IGNORE_LEADING_WHITESPACE); - } - - /** - * Constructs CSVParser with supplied separator and quote char. - * Allows setting the "strict quotes" and "ignore leading whitespace" flags - * - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param escape the character to use for escaping a separator or quote - * @param strictQuotes if true, characters outside the quotes are ignored - * @param ignoreLeadingWhiteSpace if true, white space in front of a quote in a field is ignored - */ - public CSVParser(char separator, char quotechar, char escape, boolean strictQuotes, - boolean ignoreLeadingWhiteSpace) { - this(separator, quotechar, escape, strictQuotes, ignoreLeadingWhiteSpace, - DEFAULT_IGNORE_QUOTATIONS); - } - - /** - * Constructs CSVParser with supplied separator and quote char. - * Allows setting the "strict quotes" and "ignore leading whitespace" flags - * - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param escape the character to use for escaping a separator or quote - * @param strictQuotes if true, characters outside the quotes are ignored - * @param ignoreLeadingWhiteSpace if true, white space in front of a quote in a field is ignored - * @param ignoreQuotations if true, treat quotations like any other character. - */ - public CSVParser(char separator, char quotechar, char escape, boolean strictQuotes, - boolean ignoreLeadingWhiteSpace, boolean ignoreQuotations) { - if (anyCharactersAreTheSame(separator, quotechar, escape)) { - throw new UnsupportedOperationException( - "The separator, quote, and escape characters must be different!"); - } - if (separator == NULL_CHARACTER) { - throw new UnsupportedOperationException("The separator character must be defined!"); - } - this.separator = separator; - this.quotechar = quotechar; - this.escape = escape; - this.strictQuotes = strictQuotes; - this.ignoreLeadingWhiteSpace = ignoreLeadingWhiteSpace; - this.ignoreQuotations = ignoreQuotations; - } - - /** - * Constructs CSVParser with supplied separator and quote char. - * Allows setting the "strict quotes" and "ignore leading whitespace" flags - * - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param escape the character to use for escaping a separator or quote - * @param strictQuotes if true, characters outside the quotes are ignored - * @param ignoreLeadingWhiteSpace if true, white space in front of a quote in a field is ignored - * @param ignoreQuotations if true, treat quotations like any other character. - */ - public CSVParser(char separator, char quotechar, char escape, boolean strictQuotes, - boolean ignoreLeadingWhiteSpace, boolean ignoreQuotations, boolean ignoreEscape) { - this(separator, quotechar, escape, strictQuotes, ignoreLeadingWhiteSpace, ignoreQuotations); - this.ignoreEscapeChar = ignoreEscape; - } - - /** - * @return The default separator for this parser. - */ - public char getSeparator() { - return separator; - } - - /** - * @return The default quotation character for this parser. - */ - public char getQuotechar() { - return quotechar; - } - - /** - * @return The default escape character for this parser. - */ - public char getEscape() { - return escape; - } - - /** - * @return The default strictQuotes setting for this parser. - */ - public boolean isStrictQuotes() { - return strictQuotes; - } - - /** - * @return The default ignoreLeadingWhiteSpace setting for this parser. - */ - public boolean isIgnoreLeadingWhiteSpace() { - return ignoreLeadingWhiteSpace; - } - - /** - * @return the default ignoreQuotation setting for this parser. - */ - public boolean isIgnoreQuotations() { - return ignoreQuotations; - } - - /** - * checks to see if any two of the three characters are the same. This is because in openCSV - * the separator, quote, and escape characters must the different. - * - * @param separator the defined separator character - * @param quotechar the defined quotation cahracter - * @param escape the defined escape character - * @return true if any two of the three are the same. - */ - private boolean anyCharactersAreTheSame(char separator, char quotechar, char escape) { - return isSameCharacter(separator, quotechar) || isSameCharacter(separator, escape) - || isSameCharacter(quotechar, escape); - } - - /** - * checks that the two characters are the same and are not the defined NULL_CHARACTER. - * - * @param c1 first character - * @param c2 second character - * @return true if both characters are the same and are not the defined NULL_CHARACTER - */ - private boolean isSameCharacter(char c1, char c2) { - return c1 != NULL_CHARACTER && c1 == c2; - } - - /** - * @return true if something was left over from last call(s) - */ - public boolean isPending() { - return pending != null; - } - - public void setPending(String pending) { - this.pending = pending; - } - - /** - * Parses an incoming String and returns an array of elements. This method is used when the - * data spans multiple lines. - * - * @param nextLine current line to be processed - * @return the comma-tokenized list of elements, or null if nextLine is null - * @throws IOException if bad things happen during the read - */ - public String[] parseLineMulti(String nextLine) throws IOException { - return parseLine(nextLine, true); - } - - /** - * Parses an incoming String and returns an array of elements. This method is used when all - * data is contained in a single line. - * - * @param nextLine Line to be parsed. - * @return the comma-tokenized list of elements, or null if nextLine is null - * @throws IOException if bad things happen during the read - */ - public String[] parseLine(String nextLine) throws IOException { - return parseLine(nextLine, false); - } - - /** - * Parses an incoming String and returns an array of elements. - * - * @param nextLine the string to parse - * @param multi Does it take multiple lines to form a single record. - * @return the comma-tokenized list of elements, or null if nextLine is null - * @throws IOException if bad things happen during the read - */ - private String[] parseLine(String nextLine, boolean multi) throws IOException { - - if (!multi && pending != null) { - pending = null; - } - - if (nextLine == null) { - if (pending != null) { - String s = pending; - pending = null; - return new String[] { s }; - } else { - return null; - } - } - - List<String> tokensOnThisLine = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); - //CHECKSTYLE:OFF Approval No:Approval-V1R2C10_010 - StringBuilder sb = new StringBuilder(INITIAL_READ_SIZE); - - boolean inQuotes = false; - // CHECKSTYLE:ON - if (pending != null) { - sb.append(pending); - pending = null; - inQuotes = !this.ignoreQuotations;//true; - } - inQuotes = checkForQuotes(nextLine, tokensOnThisLine, sb, inQuotes); - // line is done - check status - if ((inQuotes && !ignoreQuotations)) { - if (multi) { - // continuing a quoted section, re-append newline - sb.append('\n'); - charCountInsideQuote = sb.length() - (sb.indexOf("\"") + 1); - if (charCountInsideQuote >= 10000) //max char count to wait till the quote terminates - { - throw new IOException("Un-terminated quoted field after 10000 characters"); - } - pending = sb.toString(); - sb = null; // this partial content is not to be added to field list yet - } else { - throw new IOException("Un-terminated quoted field at end of CSV line"); - } - } else { - inField = false; - charCountInsideQuote = 0; - } - - if (sb != null) { - tokensOnThisLine.add(sb.toString()); - } - return tokensOnThisLine.toArray(new String[tokensOnThisLine.size()]); - - } - - /** - * @param nextLine - * @param tokensOnThisLine - * @param sb - * @param inQuotes - * @return - */ - private boolean checkForQuotes(String nextLine, List<String> tokensOnThisLine, StringBuilder sb, - boolean inQuotes) { - for (int i = 0; i < nextLine.length(); i++) { - - char c = nextLine.charAt(i); - //Naresh 00902756 : Modified to skip escape character check - if (!this.ignoreEscapeChar && c == this.escape) { - if (isNextCharacterEscapable(nextLine, inQuotes(inQuotes), i)) { - i = appendNextCharacterAndAdvanceLoop(nextLine, sb, i); - } - } else if (c == quotechar) { - if (isNextCharacterEscapedQuote(nextLine, inQuotes(inQuotes), i)) { - i = appendNextCharacterAndAdvanceLoop(nextLine, sb, i); - } else { - inQuotes = checkForStrictQuotes(nextLine, sb, inQuotes, i, c); - } - inField = !inField; - } else if (c == separator && !(inQuotes && !ignoreQuotations)) { - tokensOnThisLine.add(sb.toString()); - sb.setLength(0); - inField = false; - } else { - if (!strictQuotes || (inQuotes && !ignoreQuotations)) { - sb.append(c); - inField = true; - } - } - } - return inQuotes; - } - - /** - * @param nextLine - * @param sb - * @param inQuotes - * @param i - * @param c - * @return - */ - private boolean checkForStrictQuotes(String nextLine, StringBuilder sb, boolean inQuotes, int i, - char c) { - inQuotes = !inQuotes; - - // the tricky case of an embedded quote in the middle: a,bc"d"ef,g - if (!strictQuotes) { - if (i > 2 //not on the beginning of the line - && nextLine.charAt(i - 1) != this.separator - //not at the beginning of an escape sequence - && nextLine.length() > (i + 1) && - //not at the end of an escape sequence - nextLine.charAt(i + 1) != this.separator) { - if (ignoreLeadingWhiteSpace && sb.length() > 0 && isAllWhiteSpace(sb)) { - sb.setLength(0); - } else { - sb.append(c); - } - - } - } - return inQuotes; - } - - /** - * Appends the next character in the line to the stringbuffer. - * - * @param line - line to process - * @param sb - contains the processed character - * @param i - current position in the line. - * @return new position in the line. - */ - private int appendNextCharacterAndAdvanceLoop(String line, StringBuilder sb, int i) { - sb.append(line.charAt(i + 1)); - i++; - return i; - } - - /** - * Determines if we can process as if we were in quotes. - * - * @param inQuotes - are we currently in quotes. - * @return - true if we should process as if we are inside quotes. - */ - private boolean inQuotes(boolean inQuotes) { - return (inQuotes && !ignoreQuotations) || inField; - } - - /** - * Checks to see if the character after the index is a quotation character. - * precondition: the current character is a quote or an escape - * - * @param nextLine the current line - * @param inQuotes true if the current context is quoted - * @param i current index in line - * @return true if the following character is a quote - */ - private boolean isNextCharacterEscapedQuote(String nextLine, boolean inQuotes, int i) { - return inQuotes // we are in quotes, therefore there can be escaped quotes in here. - && nextLine.length() > (i + 1) // there is indeed another character to check. - && isCharacterQuoteCharacter(nextLine.charAt(i + 1)); - } - - /** - * Checks to see if the passed in character is the defined quotation character. - * - * @param c source character - * @return true if c is the defined quotation character - */ - private boolean isCharacterQuoteCharacter(char c) { - return c == quotechar; - } - - /** - * checks to see if the character is the defined escape character. - * - * @param c source character - * @return true if the character is the defined escape character - */ - private boolean isCharacterEscapeCharacter(char c) { - return c == escape; - } - - /** - * Checks to see if the character passed in could be escapable. Escapable characters for - * openCSV are the quotation character or the escape character. - * - * @param c source character - * @return true if the character could be escapable. - */ - private boolean isCharacterEscapable(char c) { - return isCharacterQuoteCharacter(c) || isCharacterEscapeCharacter(c); - } - - /** - * Checks to see if the character after the current index in a String is an escapable character. - * Meaning the next character is either a quotation character or the escape char and you are - * inside quotes. - * precondition: the current character is an escape - * - * @param nextLine the current line - * @param inQuotes true if the current context is quoted - * @param i current index in line - * @return true if the following character is a quote - */ - protected boolean isNextCharacterEscapable(String nextLine, boolean inQuotes, int i) { - return inQuotes // we are in quotes, therefore there can be escaped quotes in here. - && nextLine.length() > (i + 1) // there is indeed another character to check. - && isCharacterEscapable(nextLine.charAt(i + 1)); - } - - /** - * Checks if every element is the character sequence is whitespace. - * precondition: sb.length() is greater than 0 - * - * @param sb A sequence of characters to examine - * @return true if every character in the sequence is whitespace - */ - protected boolean isAllWhiteSpace(CharSequence sb) { - return StringUtils.isWhitespace(sb); - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVReader.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVReader.java b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVReader.java deleted file mode 100644 index ce3f1b4..0000000 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVReader.java +++ /dev/null @@ -1,496 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.partition.reader; - -/** - * Copyright 2005 Bytecode Pty Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.BufferedReader; -import java.io.Closeable; -import java.io.IOException; -import java.io.Reader; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; -import org.apache.carbondata.core.constants.CarbonCommonConstants; -import org.apache.carbondata.processing.surrogatekeysgenerator.csvbased.BadRecordslogger; - -/** - * A very simple CSV reader released under a commercial-friendly license. - * - * @author Glen Smith - */ -public class CSVReader implements Closeable, Iterable<String[]> { - - public static final boolean DEFAULT_KEEP_CR = false; - public static final boolean DEFAULT_VERIFY_READER = true; - /** - * The default line to start reading. - */ - public static final int DEFAULT_SKIP_LINES = 0; - private static final LogService LOGGER = - LogServiceFactory.getLogService(CSVReader.class.getName()); - private CSVParser parser; - private int skipLines; - private BufferedReader br; - private LineReader lineReader; - private boolean hasNext = true; - private boolean linesSkiped; - private boolean keepCR; - private boolean verifyReader; - private long lineNum; - private long skippedLines; - private boolean multiLine = true; - private BadRecordslogger badRecordslogger; - - /** - * Constructs CSVReader using a comma for the separator. - * - * @param reader the reader to an underlying CSV source. - */ - public CSVReader(Reader reader) { - this(reader, CSVParser.DEFAULT_SEPARATOR, CSVParser.DEFAULT_QUOTE_CHARACTER, - CSVParser.DEFAULT_ESCAPE_CHARACTER); - } - - /** - * Constructs CSVReader with supplied separator. - * - * @param reader the reader to an underlying CSV source. - * @param separator the delimiter to use for separating entries. - */ - public CSVReader(Reader reader, char separator) { - this(reader, separator, CSVParser.DEFAULT_QUOTE_CHARACTER, CSVParser.DEFAULT_ESCAPE_CHARACTER); - } - - /** - * Constructs CSVReader with supplied separator and quote char. - * - * @param reader the reader to an underlying CSV source. - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - */ - public CSVReader(Reader reader, char separator, char quotechar) { - this(reader, separator, quotechar, CSVParser.DEFAULT_ESCAPE_CHARACTER, DEFAULT_SKIP_LINES, - CSVParser.DEFAULT_STRICT_QUOTES); - } - - /** - * Constructs CSVReader with supplied separator, quote char and quote handling - * behavior. - * - * @param reader the reader to an underlying CSV source. - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param strictQuotes sets if characters outside the quotes are ignored - */ - public CSVReader(Reader reader, char separator, char quotechar, boolean strictQuotes) { - this(reader, separator, quotechar, CSVParser.DEFAULT_ESCAPE_CHARACTER, DEFAULT_SKIP_LINES, - strictQuotes); - } - - /** - * Constructs CSVReader. - * - * @param reader the reader to an underlying CSV source. - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param escape the character to use for escaping a separator or quote - */ - - public CSVReader(Reader reader, char separator, char quotechar, char escape) { - this(reader, separator, quotechar, escape, DEFAULT_SKIP_LINES, CSVParser.DEFAULT_STRICT_QUOTES); - } - - /** - * Constructs CSVReader. - * - * @param reader the reader to an underlying CSV source. - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param line the line number to skip for start reading - */ - public CSVReader(Reader reader, char separator, char quotechar, int line) { - this(reader, separator, quotechar, CSVParser.DEFAULT_ESCAPE_CHARACTER, line, - CSVParser.DEFAULT_STRICT_QUOTES); - } - - /** - * Constructs CSVReader. - * - * @param reader the reader to an underlying CSV source. - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param escape the character to use for escaping a separator or quote - * @param line the line number to skip for start reading - */ - public CSVReader(Reader reader, char separator, char quotechar, char escape, int line) { - this(reader, separator, quotechar, escape, line, CSVParser.DEFAULT_STRICT_QUOTES); - } - - /** - * Constructs CSVReader. - * - * @param reader the reader to an underlying CSV source. - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param escape the character to use for escaping a separator or quote - * @param line the line number to skip for start reading - * @param strictQuotes sets if characters outside the quotes are ignored - */ - public CSVReader(Reader reader, char separator, char quotechar, char escape, int line, - boolean strictQuotes) { - this(reader, separator, quotechar, escape, line, strictQuotes, - CSVParser.DEFAULT_IGNORE_LEADING_WHITESPACE); - } - - /** - * Constructs CSVReader with all data entered. - * - * @param reader the reader to an underlying CSV source. - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param escape the character to use for escaping a separator or quote - * @param line the line number to skip for start reading - * @param strictQuotes sets if characters outside the quotes are ignored - * @param ignoreLeadingWhiteSpace it true, parser should ignore white space before a quote - * in a field - */ - public CSVReader(Reader reader, char separator, char quotechar, char escape, int line, - boolean strictQuotes, boolean ignoreLeadingWhiteSpace) { - this(reader, line, - new CSVParser(separator, quotechar, escape, strictQuotes, ignoreLeadingWhiteSpace)); - } - - /** - * Constructs CSVReader with all data entered. - * - * @param reader the reader to an underlying CSV source. - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param escape the character to use for escaping a separator or quote - * @param line the line number to skip for start reading - * @param strictQuotes sets if characters outside the quotes are ignored - * @param ignoreLeadingWhiteSpace if true, parser should ignore white space before a quote - * in a field - * @param keepCR if true the reader will keep carriage returns, - * otherwise it will discard them. - */ - public CSVReader(Reader reader, char separator, char quotechar, char escape, int line, - boolean strictQuotes, boolean ignoreLeadingWhiteSpace, boolean keepCR) { - this(reader, line, - new CSVParser(separator, quotechar, escape, strictQuotes, ignoreLeadingWhiteSpace), keepCR, - DEFAULT_VERIFY_READER); - } - - /** - * Constructs CSVReader with supplied CSVParser. - * - * @param reader the reader to an underlying CSV source. - * @param line the line number to skip for start reading - * @param csvParser the parser to use to parse input - */ - public CSVReader(Reader reader, int line, CSVParser csvParser, boolean multiLine) { - this(reader, line, csvParser); - this.multiLine = multiLine; - } - - public CSVReader(Reader reader, int line, CSVParser csvParser) { - this(reader, line, csvParser, DEFAULT_KEEP_CR, DEFAULT_VERIFY_READER); - } - - /** - * Constructs CSVReader with supplied CSVParser. - * - * @param reader the reader to an underlying CSV source. - * @param line the line number to skip for start reading - * @param csvParser the parser to use to parse input - * @param keepCR true to keep carriage returns in data read, false otherwise - * @param verifyReader true to verify reader before each read, false otherwise - */ - CSVReader(Reader reader, int line, CSVParser csvParser, boolean keepCR, boolean verifyReader) { - this.br = (reader instanceof BufferedReader ? - (BufferedReader) reader : - new BufferedReader(reader, 30720)); - this.lineReader = new LineReader(br, keepCR); - this.skipLines = line; - this.parser = csvParser; - this.keepCR = keepCR; - this.verifyReader = verifyReader; - } - - /** - * @return the CSVParser used by the reader. - */ - public CSVParser getParser() { - return parser; - } - - /** - * Returns the number of lines in the csv file to skip before processing. This is - * useful when there is miscellaneous data at the beginning of a file. - * - * @return the number of lines in the csv file to skip before processing. - */ - public int getSkipLines() { - return skipLines; - } - - /** - * Returns if the reader will keep carriage returns found in data or remove them. - * - * @return true if reader will keep carriage returns, false otherwise. - */ - public boolean keepCarriageReturns() { - return keepCR; - } - - /** - * Reads the entire file into a List with each element being a String[] of - * tokens. - * - * @return a List of String[], with each String[] representing a line of the - * file. - * @throws IOException if bad things happen during the read - */ - public List<String[]> readAll() throws IOException { - - //CHECKSTYLE:OFF Approval No:Approval-V1R2C10_005 - List<String[]> allElements = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); - - while (hasNext) { - String[] nextLineAsTokens = readNext(); - if (nextLineAsTokens != null) { - allElements.add(nextLineAsTokens); - } - } - return allElements; - //CHECKSTYLE:ON - } - - /** - * Reads the next line from the buffer and converts to a string array. - * - * @return a string array with each comma-separated element as a separate - * entry. - * @throws IOException if bad things happen during the read - */ - public String[] readNext() throws IOException { - - if (this.multiLine) { - return readMultiLine(); - } else { - return readSingleLine(); - } - } - - private String[] readSingleLine() throws IOException { - String[] result = null; - String nextLine = getNextLine(); - try { - this.lineNum += 1L; - result = parser.parseLine(nextLine); - } catch (IOException e) { - if ("Un-terminated quoted field at end of CSV line".equals(e.getMessage())) { - badRecordslogger.addBadRecordsToBilder(new String[] { nextLine }, 1, - "Un-terminated quoted field at end of CSV line", null); - LOGGER.info("Found Un-terminated quote @ line [" + this.lineNum + "] : Skipping Line : " - + nextLine); - this.skippedLines += 1L; - result = readNext(); - } else { - throw e; - } - } - if (null == nextLine) { - LOGGER.info("Total Number of Lines : " + --this.lineNum); - LOGGER.info("Number of Lines Skipped: " + this.skippedLines); - // System.out.println("Total Number of Lines : "+ --this.lineNum); - // System.out.println("Number of Lines Skipped: "+ this.skippedLines); - } - return result; - } - - private String[] readMultiLine() throws IOException { - int linesread = 0; - String[] result = null; - String firstLine = null; - do { - this.lineNum += 1L; - linesread++; - if (linesread == 2) { - br.mark(12000); - } - String nextLine = getNextLine(); - if (!hasNext) { - LOGGER.info("Total Number of Lines : " + --this.lineNum); - LOGGER.info("Number of Lines Skipped: " + this.skippedLines); - // System.out.println("Total Number of Lines : "+ --this.lineNum); - // System.out.println("Number of Lines Skipped: "+ this.skippedLines); - return result; // should throw if still pending? - } - try { - String[] r = parser.parseLineMulti(nextLine); - if (r.length > 0) { - if (result == null) { - result = r; - } else { - result = combineResultsFromMultipleReads(result, r); - } - } - } catch (IOException e) { - if ("Un-terminated quoted field after 10000 characters".equals(e.getMessage())) { - LOGGER.info("Un-terminated quoted field found after 10000 characters in MultiLine " - + "(No. Of Line searched : " + linesread + " ) starting from Line :" + ( - this.lineNum - linesread + 1)); - LOGGER.info("Skipped Line Info : " + firstLine); - parser.setPending(null); - this.skippedLines += 1; - this.lineNum += (1 - linesread); - if (linesread > 1) { - br.reset(); - } - int resLength = result != null ? result.length : 0; - badRecordslogger.addBadRecordsToBilder(result, resLength, - "Un-terminated quoted field after 10000 characters", null); - result = readNext(); - } else { - throw e; - } - } - if (linesread == 1) { - firstLine = nextLine; - } - // String[] r = parser.parseLine(nextLine); - } while (parser.isPending()); - return result; - } - - /** - * For multi line records this method combines the current result with the result - * from previous read(s). - * - * @param buffer - previous data read for this record - * @param lastRead - latest data read for this record. - * @return String array with union of the buffer and lastRead arrays. - */ - private String[] combineResultsFromMultipleReads(String[] buffer, String[] lastRead) { - String[] t = new String[buffer.length + lastRead.length]; - System.arraycopy(buffer, 0, t, 0, buffer.length); - System.arraycopy(lastRead, 0, t, buffer.length, lastRead.length); - return t; - } - - /** - * Reads the next line from the file. - * - * @return the next line from the file without trailing newline - * @throws IOException if bad things happen during the read - */ - public String getNextLine() throws IOException { - if (isClosed()) { - hasNext = false; - return null; - } - - if (!this.linesSkiped) { - for (int i = 0; i < skipLines; i++) { - lineReader.readLine(); - } - this.linesSkiped = true; - } - String nextLine = lineReader.readLine(); - if (nextLine == null) { - hasNext = false; - } - return hasNext ? nextLine : null; - } - - /** - * Checks to see if the file is closed. - * - * @return true if the reader can no longer be read from. - */ - private boolean isClosed() { - if (!verifyReader) { - return false; - } - try { - return !br.ready(); - } catch (IOException e) { - return true; - } - } - - /** - * Closes the underlying reader. - * - * @throws IOException if the close fails - */ - public void close() throws IOException { - br.close(); - } - - /** - * Creates an Iterator for processing the csv data. - * - * @return an String[] iterator. - */ - public Iterator<String[]> iterator() { - try { - return new CSVIterator(this); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Returns if the CSVReader will verify the reader before each read. - * By default the value is true which is the functionality for version 3.0. - * If set to false the reader is always assumed ready to read - this is the functionality - * for version 2.4 and before. - * The reason this method was needed was that certain types of Readers would return - * false for its ready() method until a read was done (namely readers created using Channels). - * This caused opencsv not to read from those readers. - * - * @return true if CSVReader will verify the reader before reads. False otherwise. - * @link https://sourceforge.net/p/opencsv/bugs/108/ - */ - public boolean verifyReader() { - return this.verifyReader; - } - - public void setBadRecordsLogger(BadRecordslogger badRecordslogger) { - this.badRecordslogger = badRecordslogger; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVWriter.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVWriter.java b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVWriter.java deleted file mode 100644 index 68dbd59..0000000 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/CSVWriter.java +++ /dev/null @@ -1,396 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.partition.reader; - -/** - * Copyright 2005 Bytecode Pty Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.Closeable; -import java.io.Flushable; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.Writer; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -import org.apache.carbondata.common.logging.LogService; -import org.apache.carbondata.common.logging.LogServiceFactory; - -/** - * A very simple CSV writer released under a commercial-friendly license. - * - * @author Glen Smith - */ -public class CSVWriter implements Closeable, Flushable { - - public static final int INITIAL_STRING_SIZE = 128; - /** - * The character used for escaping quotes. - */ - public static final char DEFAULT_ESCAPE_CHARACTER = '"'; - /** - * The default separator to use if none is supplied to the constructor. - */ - public static final char DEFAULT_SEPARATOR = ','; - /** - * The default quote character to use if none is supplied to the - * constructor. - */ - public static final char DEFAULT_QUOTE_CHARACTER = '"'; - /** - * The quote constant to use when you wish to suppress all quoting. - */ - public static final char NO_QUOTE_CHARACTER = '\u0000'; - /** - * The escape constant to use when you wish to suppress all escaping. - */ - public static final char NO_ESCAPE_CHARACTER = '\u0000'; - /** - * Default line terminator uses platform encoding. - */ - public static final String DEFAULT_LINE_END = "\n"; - - public static final String CARRIAGE_RETURN = "\r"; - - private static final LogService LOGGER = - LogServiceFactory.getLogService(CSVWriter.class.getName()); - - private Writer rawWriter; - private PrintWriter pw; - private char separator; - private char quotechar; - private char escapechar; - private String lineEnd; - private ResultSetHelper resultService = new ResultSetHelperService(); - - /** - * Constructs CSVWriter using a comma for the separator. - * - * @param writer the writer to an underlying CSV source. - */ - public CSVWriter(Writer writer) { - this(writer, DEFAULT_SEPARATOR); - } - - /** - * Constructs CSVWriter with supplied separator. - * - * @param writer the writer to an underlying CSV source. - * @param separator the delimiter to use for separating entries. - */ - public CSVWriter(Writer writer, char separator) { - this(writer, separator, DEFAULT_QUOTE_CHARACTER); - } - - /** - * Constructs CSVWriter with supplied separator and quote char. - * - * @param writer the writer to an underlying CSV source. - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - */ - public CSVWriter(Writer writer, char separator, char quotechar) { - this(writer, separator, quotechar, DEFAULT_ESCAPE_CHARACTER); - } - - /** - * Constructs CSVWriter with supplied separator and quote char. - * - * @param writer the writer to an underlying CSV source. - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param escapechar the character to use for escaping quotechars or escapechars - */ - public CSVWriter(Writer writer, char separator, char quotechar, char escapechar) { - this(writer, separator, quotechar, escapechar, DEFAULT_LINE_END); - } - - /** - * Constructs CSVWriter with supplied separator and quote char. - * - * @param writer the writer to an underlying CSV source. - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param lineEnd the line feed terminator to use - */ - public CSVWriter(Writer writer, char separator, char quotechar, String lineEnd) { - this(writer, separator, quotechar, DEFAULT_ESCAPE_CHARACTER, lineEnd); - } - - /** - * Constructs CSVWriter with supplied separator, quote char, escape char and line ending. - * - * @param writer the writer to an underlying CSV source. - * @param separator the delimiter to use for separating entries - * @param quotechar the character to use for quoted elements - * @param escapechar the character to use for escaping quotechars or escapechars - * @param lineEnd the line feed terminator to use - */ - public CSVWriter(Writer writer, char separator, char quotechar, char escapechar, String lineEnd) { - this.rawWriter = writer; - this.pw = new PrintWriter(writer); - this.separator = separator; - this.quotechar = quotechar; - this.escapechar = escapechar; - this.lineEnd = lineEnd; - } - - /** - * Writes the entire list to a CSV file. The list is assumed to be a - * String[] - * - * @param allLines a List of String[], with each String[] representing a line of - * the file. - * @param applyQuotesToAll true if all values are to be quoted. false if quotes only - * to be applied to values which contain the separator, escape, - * quote or new line characters. - */ - public void writeAll(List<String[]> allLines, boolean applyQuotesToAll) { - for (String[] line : allLines) { - writeNext(line, applyQuotesToAll); - } - } - - /** - * Writes the entire list to a CSV file. The list is assumed to be a - * String[] - * - * @param allLines a List of String[], with each String[] representing a line of - * the file. - */ - public void writeAll(List<String[]> allLines) { - for (String[] line : allLines) { - writeNext(line); - } - } - - /** - * Writes the column names. - * - * @param rs - ResultSet containing column names. - * @throws SQLException - thrown by ResultSet::getColumnNames - */ - protected void writeColumnNames(ResultSet rs) throws SQLException { - - writeNext(resultService.getColumnNames(rs)); - } - - /** - * Writes the entire ResultSet to a CSV file. - * The caller is responsible for closing the ResultSet. - * - * @param rs the result set to write - * @param includeColumnNames true if you want column names in the output, false otherwise - * @throws IOException thrown by getColumnValue - * @throws SQLException thrown by getColumnValue - */ - public void writeAll(ResultSet rs, boolean includeColumnNames) throws SQLException, IOException { - writeAll(rs, includeColumnNames, false); - } - - /** - * Writes the entire ResultSet to a CSV file. - * The caller is responsible for closing the ResultSet. - * - * @param rs the Result set to write. - * @param includeColumnNames include the column names in the output. - * @param trim remove spaces from the data before writing. - * @throws IOException thrown by getColumnValue - * @throws SQLException thrown by getColumnValue - */ - public void writeAll(ResultSet rs, boolean includeColumnNames, boolean trim) - throws SQLException, IOException { - - if (includeColumnNames) { - writeColumnNames(rs); - } - - while (rs.next()) { - writeNext(resultService.getColumnValues(rs, trim)); - } - } - - /** - * Writes the next line to the file. - * - * @param nextLine a string array with each comma-separated element as a separate - * entry. - * @param applyQuotesToAll true if all values are to be quoted. false applies quotes only - * to values which contain the separator, escape, quote or new line - * characters. - */ - public void writeNext(String[] nextLine, boolean applyQuotesToAll) { - - if (nextLine == null) { - return; - } - - StringBuilder sb = new StringBuilder(INITIAL_STRING_SIZE); - for (int i = 0; i < nextLine.length; i++) { - - if (i != 0) { - sb.append(separator); - } - - String nextElement = nextLine[i]; - - if (nextElement == null) { - continue; - } - - Boolean stringContainsSpecialCharacters = stringContainsSpecialCharacters(nextElement); - - if ((applyQuotesToAll || stringContainsSpecialCharacters) - && quotechar != NO_QUOTE_CHARACTER) { - sb.append(quotechar); - } - - if (stringContainsSpecialCharacters) { - sb.append(processLine(nextElement)); - } else { - sb.append(nextElement); - } - - if ((applyQuotesToAll || stringContainsSpecialCharacters) - && quotechar != NO_QUOTE_CHARACTER) { - sb.append(quotechar); - } - } - - sb.append(lineEnd); - pw.write(sb.toString()); - } - - /** - * Writes the next line to the file. - * - * @param nextLine a string array with each comma-separated element as a separate - * entry. - */ - public void writeNext(String[] nextLine) { - writeNext(nextLine, true); - } - - /** - * checks to see if the line contains special characters. - * - * @param line - element of data to check for special characters. - * @return true if the line contains the quote, escape, separator, newline or return. - */ - private boolean stringContainsSpecialCharacters(String line) { - return line.indexOf(quotechar) != -1 || line.indexOf(escapechar) != -1 - || line.indexOf(separator) != -1 || line.contains(DEFAULT_LINE_END) || line - .contains(CARRIAGE_RETURN); - } - - /** - * Processes all the characters in a line. - * - * @param nextElement - element to process. - * @return a StringBuilder with the elements data. - */ - protected StringBuilder processLine(String nextElement) { - StringBuilder sb = new StringBuilder(INITIAL_STRING_SIZE); - for (int j = 0; j < nextElement.length(); j++) { - char nextChar = nextElement.charAt(j); - processCharacter(sb, nextChar); - } - - return sb; - } - - /** - * Appends the character to the StringBuilder adding the escape character if needed. - * - * @param sb - StringBuffer holding the processed character. - * @param nextChar - character to process - */ - private void processCharacter(StringBuilder sb, char nextChar) { - if (escapechar != NO_ESCAPE_CHARACTER && (nextChar == quotechar || nextChar == escapechar)) { - sb.append(escapechar).append(nextChar); - } else { - sb.append(nextChar); - } - } - - /** - * Flush underlying stream to writer. - * - * @throws IOException if bad things happen - */ - public void flush() throws IOException { - - pw.flush(); - - } - - /** - * Close the underlying stream writer flushing any buffered content. - * - * @throws IOException if bad things happen - */ - public void close() throws IOException { - flush(); - pw.close(); - rawWriter.close(); - } - - /** - * Checks to see if the there has been an error in the printstream. - * - * @return <code>true</code> if the print stream has encountered an error, - * either on the underlying output stream or during a format - * conversion. - */ - public boolean checkError() { - return pw.checkError(); - } - - /** - * Sets the result service. - * - * @param resultService - the ResultSetHelper - */ - public void setResultService(ResultSetHelper resultService) { - this.resultService = resultService; - } - - /** - * flushes the writer without throwing any exceptions. - */ - public void flushQuietly() { - try { - flush(); - } catch (IOException e) { - LOGGER.debug("Error while flushing"); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/be1675f4/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/LineReader.java ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/LineReader.java b/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/LineReader.java deleted file mode 100644 index 3ee6228..0000000 --- a/integration/spark/src/main/java/org/apache/carbondata/spark/partition/reader/LineReader.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.carbondata.spark.partition.reader; - -import java.io.BufferedReader; -import java.io.IOException; - -/** - * This class was created for issue #106 (https://sourceforge.net/p/opencsv/bugs/106/) where - * carriage returns were being removed. This class allows the user to determine if they wish to - * keep or remove them from the data being read. - * Created by scott on 2/19/15. - */ - -public class LineReader { - private BufferedReader reader; - private boolean keepCarriageReturns; - - /** - * LineReader constructor. - * - * @param reader - Reader that data will be read from. - * @param keepCarriageReturns - true if carriage returns should remain in the data, false - * to remove them. - */ - public LineReader(BufferedReader reader, boolean keepCarriageReturns) { - this.reader = reader; - this.keepCarriageReturns = keepCarriageReturns; - } - - /** - * Reads the next line from the Reader. - * - * @return - Line read from reader. - * @throws IOException - on error from BufferedReader - */ - public String readLine() throws IOException { - return keepCarriageReturns ? readUntilNewline() : reader.readLine(); - } - - private String readUntilNewline() throws IOException { - StringBuilder sb = new StringBuilder(CSVParser.INITIAL_READ_SIZE); - for (int c = reader.read(); - c > -1 && c != '\n'; - c = reader.read()) { //CHECKSTYLE:OFF Approval No:Approval-V1R2C10_006 - sb.append((char) c); - }//CHECKSTYLE:ON - - return sb.length() > 0 ? sb.toString() : null; - } -}
