This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new d6f907f [Baseline] Apply baseline to iceberg-orc #158 (#211)
d6f907f is described below
commit d6f907fb704f919fc89846fcd7af4b679c2efd6f
Author: Ratandeep Ratti <[email protected]>
AuthorDate: Wed Jun 12 11:39:06 2019 -0700
[Baseline] Apply baseline to iceberg-orc #158 (#211)
---
build.gradle | 5 ++--
.../java/org/apache/iceberg/orc/ColumnIdMap.java | 6 ++--
orc/src/main/java/org/apache/iceberg/orc/ORC.java | 33 +++++++++++-----------
.../org/apache/iceberg/orc/OrcFileAppender.java | 20 ++++++-------
.../java/org/apache/iceberg/orc/OrcIterable.java | 4 +--
.../java/org/apache/iceberg/orc/OrcMetrics.java | 5 ++--
.../org/apache/iceberg/orc/TypeConversion.java | 6 ++--
7 files changed, 39 insertions(+), 40 deletions(-)
diff --git a/build.gradle b/build.gradle
index 68cf864..eb5ae3d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -112,8 +112,9 @@ subprojects {
// We enable baseline-idea everywhere so that everyone can use IntelliJ to
build code against the
// Baseline style guide.
-def baselineProjects = [ project("iceberg-api"), project("iceberg-common"),
project("iceberg-core"),
- project("iceberg-data") ]
+def baselineProjects = [ project("iceberg-api"), project("iceberg-common"),
project("iceberg-core"),
+ project("iceberg-data"), project("iceberg-orc") ]
+
configure(subprojects - baselineProjects) {
// error-prone is brought in with baseline-idea, but we're not prepared to
handle error-prone
diff --git a/orc/src/main/java/org/apache/iceberg/orc/ColumnIdMap.java
b/orc/src/main/java/org/apache/iceberg/orc/ColumnIdMap.java
index 16dc3b0..6b83afc 100644
--- a/orc/src/main/java/org/apache/iceberg/orc/ColumnIdMap.java
+++ b/orc/src/main/java/org/apache/iceberg/orc/ColumnIdMap.java
@@ -29,7 +29,7 @@ import org.apache.orc.TypeDescription;
/**
* The mapping from ORC's TypeDescription to the Iceberg column ids.
- *
+ * <p>
* Keep the API limited to Map rather than a concrete type so that we can
* change it later.
*/
@@ -101,7 +101,7 @@ public class ColumnIdMap implements Map<TypeDescription,
Integer> {
public ByteBuffer serialize() {
StringBuilder buffer = new StringBuilder();
boolean needComma = false;
- for(TypeDescription key: idMap.keySet()) {
+ for (TypeDescription key : idMap.keySet()) {
if (needComma) {
buffer.append(',');
} else {
@@ -118,7 +118,7 @@ public class ColumnIdMap implements Map<TypeDescription,
Integer> {
ByteBuffer serial) {
ColumnIdMap result = new ColumnIdMap();
String[] parts =
StandardCharsets.UTF_8.decode(serial).toString().split(",");
- for(int i = 0; i < parts.length; ++i) {
+ for (int i = 0; i < parts.length; ++i) {
String[] subparts = parts[i].split(":");
result.put(schema.findSubtype(Integer.parseInt(subparts[0])),
Integer.parseInt(subparts[1]));
diff --git a/orc/src/main/java/org/apache/iceberg/orc/ORC.java
b/orc/src/main/java/org/apache/iceberg/orc/ORC.java
index f275594..a014d2a 100644
--- a/orc/src/main/java/org/apache/iceberg/orc/ORC.java
+++ b/orc/src/main/java/org/apache/iceberg/orc/ORC.java
@@ -33,14 +33,13 @@ import org.apache.iceberg.io.FileAppender;
import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.io.OutputFile;
import org.apache.orc.OrcConf;
-import org.apache.orc.OrcFile;
import org.apache.orc.TypeDescription;
+import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch;
-import static
org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch.DEFAULT_SIZE;
-
+@SuppressWarnings("checkstyle:AbbreviationAsWordInName")
public class ORC {
- static final String VECTOR_ROW_BATCH_SIZE = "iceberg.orc.vectorbatch.size";
+ private static final String VECTOR_ROW_BATCH_SIZE =
"iceberg.orc.vectorbatch.size";
private ORC() {
}
@@ -85,8 +84,8 @@ public class ORC {
return this;
}
- public WriteBuilder schema(Schema schema) {
- this.schema = schema;
+ public WriteBuilder schema(Schema newSchema) {
+ this.schema = newSchema;
return this;
}
@@ -94,7 +93,7 @@ public class ORC {
Preconditions.checkNotNull(schema, "Schema is required");
return new OrcFileAppender<>(TypeConversion.toOrc(schema, new
ColumnIdMap()),
this.file, createWriterFunc, conf, metadata,
- conf.getInt(VECTOR_ROW_BATCH_SIZE, DEFAULT_SIZE));
+ conf.getInt(VECTOR_ROW_BATCH_SIZE, VectorizedRowBatch.DEFAULT_SIZE));
}
}
@@ -109,7 +108,7 @@ public class ORC {
private Long start = null;
private Long length = null;
- private Function<Schema, OrcValueReader<?>> readerFunction;
+ private Function<Schema, OrcValueReader<?>> readerFunc;
private ReadBuilder(InputFile file) {
Preconditions.checkNotNull(file, "Input file cannot be null");
@@ -124,18 +123,18 @@ public class ORC {
/**
* Restricts the read to the given range: [start, start + length).
*
- * @param start the start position for this read
- * @param length the length of the range this read should scan
+ * @param newStart the start position for this read
+ * @param newLength the length of the range this read should scan
* @return this builder for method chaining
*/
- public ReadBuilder split(long start, long length) {
- this.start = start;
- this.length = length;
+ public ReadBuilder split(long newStart, long newLength) {
+ this.start = newStart;
+ this.length = newLength;
return this;
}
- public ReadBuilder schema(org.apache.iceberg.Schema schema) {
- this.schema = schema;
+ public ReadBuilder schema(org.apache.iceberg.Schema projectSchema) {
+ this.schema = projectSchema;
return this;
}
@@ -150,13 +149,13 @@ public class ORC {
}
public ReadBuilder createReaderFunc(Function<Schema, OrcValueReader<?>>
readerFunction) {
- this.readerFunction = readerFunction;
+ this.readerFunc = readerFunction;
return this;
}
public <D> CloseableIterable<D> build() {
Preconditions.checkNotNull(schema, "Schema is required");
- return new OrcIterable<>(file, conf, schema, start, length,
readerFunction);
+ return new OrcIterable<>(file, conf, schema, start, length, readerFunc);
}
}
}
diff --git a/orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java
b/orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java
index 381fcf7..007045a 100644
--- a/orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java
+++ b/orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java
@@ -20,6 +20,7 @@
package org.apache.iceberg.orc;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
@@ -27,7 +28,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
-import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.iceberg.Metrics;
@@ -96,18 +96,18 @@ class OrcFileAppender<D> implements FileAppender<D> {
Map<Integer, Long> valueCounts = new HashMap<>();
Map<Integer, Long> nullCounts = new HashMap<>();
Integer[] icebergIds = new Integer[orcSchema.getMaximumId() + 1];
- for(TypeDescription type: columnIds.keySet()) {
+ for (TypeDescription type : columnIds.keySet()) {
icebergIds[type.getId()] = columnIds.get(type);
}
- for(int c=1; c < stats.length; ++c) {
+ for (int c = 1; c < stats.length; ++c) {
if (icebergIds[c] != null) {
valueCounts.put(icebergIds[c], stats[c].getNumberOfValues());
}
}
- for(TypeDescription child: orcSchema.getChildren()) {
- int c = child.getId();
- if (icebergIds[c] != null) {
- nullCounts.put(icebergIds[c], rows - stats[c].getNumberOfValues());
+ for (TypeDescription child : orcSchema.getChildren()) {
+ int childId = child.getId();
+ if (icebergIds[childId] != null) {
+ nullCounts.put(icebergIds[childId], rows -
stats[childId].getNumberOfValues());
}
}
return new Metrics(rows, null, valueCounts, nullCounts);
@@ -164,14 +164,14 @@ class OrcFileAppender<D> implements FileAppender<D> {
}
writer.addUserMetadata(COLUMN_NUMBERS_ATTRIBUTE, columnIds.serialize());
- metadata.forEach((key,value) -> writer.addUserMetadata(key,
ByteBuffer.wrap(value)));
+ metadata.forEach((key, value) -> writer.addUserMetadata(key,
ByteBuffer.wrap(value)));
return writer;
}
@SuppressWarnings("unchecked")
- private static <D> OrcValueWriter<D> newOrcValueWriter(TypeDescription
schema,
-
Function<TypeDescription, OrcValueWriter<?>> createWriterFunc) {
+ private static <D> OrcValueWriter<D> newOrcValueWriter(
+ TypeDescription schema, Function<TypeDescription, OrcValueWriter<?>>
createWriterFunc) {
return (OrcValueWriter<D>) createWriterFunc.apply(schema);
}
}
diff --git a/orc/src/main/java/org/apache/iceberg/orc/OrcIterable.java
b/orc/src/main/java/org/apache/iceberg/orc/OrcIterable.java
index b4bed83..60094e7 100644
--- a/orc/src/main/java/org/apache/iceberg/orc/OrcIterable.java
+++ b/orc/src/main/java/org/apache/iceberg/orc/OrcIterable.java
@@ -96,8 +96,8 @@ class OrcIterable<T> extends CloseableGroup implements
CloseableIterable<T> {
private int nextRow;
private VectorizedRowBatch current;
- final VectorizedRowBatchIterator batchIter;
- final OrcValueReader<T> reader;
+ private final VectorizedRowBatchIterator batchIter;
+ private final OrcValueReader<T> reader;
OrcIterator(VectorizedRowBatchIterator batchIter, OrcValueReader<T>
reader) {
this.batchIter = batchIter;
diff --git a/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java
b/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java
index 2defc7d..5dc6421 100644
--- a/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java
+++ b/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java
@@ -35,9 +35,8 @@ public class OrcMetrics {
private OrcMetrics() {}
public static Metrics fromInputFile(InputFile file) {
- final Configuration config = (file instanceof HadoopInputFile)
- ? ((HadoopInputFile)file).getConf()
- : new Configuration();
+ final Configuration config = (file instanceof HadoopInputFile) ?
+ ((HadoopInputFile) file).getConf() : new Configuration();
return fromInputFile(file, config);
}
diff --git a/orc/src/main/java/org/apache/iceberg/orc/TypeConversion.java
b/orc/src/main/java/org/apache/iceberg/orc/TypeConversion.java
index f9839f6..29448f9 100644
--- a/orc/src/main/java/org/apache/iceberg/orc/TypeConversion.java
+++ b/orc/src/main/java/org/apache/iceberg/orc/TypeConversion.java
@@ -89,7 +89,7 @@ public class TypeConversion {
}
case STRUCT: {
result = TypeDescription.createStruct();
- for(Types.NestedField field: type.asStructType().fields()) {
+ for (Types.NestedField field : type.asStructType().fields()) {
result.addField(field.name(), toOrc(field.fieldId(), field.type(),
columnIds));
}
break;
@@ -102,7 +102,7 @@ public class TypeConversion {
}
case MAP: {
Types.MapType map = (Types.MapType) type;
- TypeDescription key = toOrc(map.keyId(),map.keyType(), columnIds);
+ TypeDescription key = toOrc(map.keyId(), map.keyType(), columnIds);
result = TypeDescription.createMap(key,
toOrc(map.valueId(), map.valueType(), columnIds));
break;
@@ -156,7 +156,7 @@ public class TypeConversion {
List<String> fieldNames = schema.getFieldNames();
List<TypeDescription> fieldTypes = schema.getChildren();
List<Types.NestedField> fields = new ArrayList<>(fieldNames.size());
- for (int c=0; c < fieldNames.size(); ++c) {
+ for (int c = 0; c < fieldNames.size(); ++c) {
String name = fieldNames.get(c);
TypeDescription type = fieldTypes.get(c);
fields.add(Types.NestedField.optional(columnIds.get(type), name,
