DRILL 1221: Update for changes in Parquet-12 for new converted types. Also converted existing binary files in the test-data directory.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c2e32ac0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c2e32ac0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c2e32ac0 Branch: refs/heads/master Commit: c2e32ac00e850e1ab6bee81e81070dfde4a48d9b Parents: 5bae9c8 Author: Jason Altekruse <[email protected]> Authored: Tue Jul 29 17:07:34 2014 -0500 Committer: Jacques Nadeau <[email protected]> Committed: Tue Jul 29 16:49:47 2014 -0700 ---------------------------------------------------------------------- .../codegen/templates/ParquetTypeHelper.java | 6 ++--- .../ParquetToDrillTypeConverter.java | 25 ++++++++++--------- sample-data/nation.parquet | Bin 3109 -> 1210 bytes sample-data/nationsMF/nation.parquet | Bin 3109 -> 0 bytes sample-data/nationsMF/nation2.parquet | Bin 3109 -> 0 bytes sample-data/nationsMF/nationsMF.parquet | Bin 0 -> 1302 bytes sample-data/nationsSF/nation.parquet | Bin 3109 -> 0 bytes sample-data/nationsSF/nationsSF.parquet | Bin 0 -> 1210 bytes sample-data/region.parquet | Bin 819 -> 455 bytes sample-data/regionsMF/region.parquet | Bin 819 -> 0 bytes sample-data/regionsMF/region2.parquet | Bin 819 -> 0 bytes sample-data/regionsMF/regionsMF_Typed.parquet | Bin 0 -> 468 bytes sample-data/regionsSF/region.parquet | Bin 819 -> 0 bytes sample-data/regionsSF/regionsSF.parquet | Bin 0 -> 455 bytes 14 files changed, 16 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java b/exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java index 15830f6..5dbc860 100644 --- a/exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java +++ b/exec/java-exec/src/main/codegen/templates/ParquetTypeHelper.java @@ -110,9 +110,9 @@ public class ParquetTypeHelper { </#list> originalTypeMap.put(MinorType.VARCHAR, OriginalType.UTF8); originalTypeMap.put(MinorType.DATE, OriginalType.DATE); - originalTypeMap.put(MinorType.TIME, OriginalType.TIME); - originalTypeMap.put(MinorType.TIMESTAMP, OriginalType.TIMESTAMP); - originalTypeMap.put(MinorType.TIMESTAMPTZ, OriginalType.TIMESTAMPTZ); + originalTypeMap.put(MinorType.TIME, OriginalType.TIME_MILLIS); + originalTypeMap.put(MinorType.TIMESTAMP, OriginalType.TIMESTAMP_MILLIS); +// originalTypeMap.put(MinorType.TIMESTAMPTZ, OriginalType.TIMESTAMPTZ); } public static PrimitiveTypeName getPrimitiveTypeNameForMinorType(MinorType minorType) { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java index 7eeeeaa..5bba6be 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/ParquetToDrillTypeConverter.java @@ -59,9 +59,10 @@ public class ParquetToDrillTypeConverter { switch(convertedType) { case DECIMAL: return Types.withScaleAndPrecision(TypeProtos.MinorType.DECIMAL18, DataMode.OPTIONAL, schemaElement.getScale(), schemaElement.getPrecision()); - case FINETIME: - throw new UnsupportedOperationException(); - case TIMESTAMP: + // TODO - add this back if it is decided to be added upstream, was removed form our pull request July 2014 +// case TIME_MICROS: +// throw new UnsupportedOperationException(); + case TIMESTAMP_MILLIS: return Types.optional(MinorType.TIMESTAMP); default: throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType)); @@ -75,7 +76,7 @@ public class ParquetToDrillTypeConverter { return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.OPTIONAL, schemaElement.getScale(), schemaElement.getPrecision()); case DATE: return Types.optional(MinorType.DATE); - case TIME: + case TIME_MILLIS: return Types.optional(MinorType.TIME); default: throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType)); @@ -123,9 +124,9 @@ public class ParquetToDrillTypeConverter { switch(convertedType) { case DECIMAL: return Types.withScaleAndPrecision(MinorType.DECIMAL18, DataMode.REQUIRED, schemaElement.getScale(), schemaElement.getPrecision()); - case FINETIME: - throw new UnsupportedOperationException(); - case TIMESTAMP: +// case FINETIME: +// throw new UnsupportedOperationException(); + case TIMESTAMP_MILLIS: return Types.required(MinorType.TIMESTAMP); default: throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType)); @@ -139,7 +140,7 @@ public class ParquetToDrillTypeConverter { return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.REQUIRED, schemaElement.getScale(), schemaElement.getPrecision()); case DATE: return Types.required(MinorType.DATE); - case TIME: + case TIME_MILLIS: return Types.required(MinorType.TIME); default: throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType)); @@ -187,9 +188,9 @@ public class ParquetToDrillTypeConverter { switch(convertedType) { case DECIMAL: return Types.withScaleAndPrecision(MinorType.DECIMAL18, DataMode.REPEATED, schemaElement.getScale(), schemaElement.getPrecision()); - case FINETIME: - throw new UnsupportedOperationException(); - case TIMESTAMP: +// case FINETIME: +// throw new UnsupportedOperationException(); + case TIMESTAMP_MILLIS: return Types.repeated(MinorType.TIMESTAMP); default: throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType)); @@ -203,7 +204,7 @@ public class ParquetToDrillTypeConverter { return Types.withScaleAndPrecision(MinorType.DECIMAL9, DataMode.REPEATED, schemaElement.getScale(), schemaElement.getPrecision()); case DATE: return Types.repeated(MinorType.DATE); - case TIME: + case TIME_MILLIS: return Types.repeated(MinorType.TIME); default: throw new UnsupportedOperationException(String.format("unsupported type: %s %s", primitiveTypeName, convertedType)); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/nation.parquet ---------------------------------------------------------------------- diff --git a/sample-data/nation.parquet b/sample-data/nation.parquet old mode 100644 new mode 100755 index db65492..9109d8b Binary files a/sample-data/nation.parquet and b/sample-data/nation.parquet differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/nationsMF/nation.parquet ---------------------------------------------------------------------- diff --git a/sample-data/nationsMF/nation.parquet b/sample-data/nationsMF/nation.parquet deleted file mode 100644 index db65492..0000000 Binary files a/sample-data/nationsMF/nation.parquet and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/nationsMF/nation2.parquet ---------------------------------------------------------------------- diff --git a/sample-data/nationsMF/nation2.parquet b/sample-data/nationsMF/nation2.parquet deleted file mode 100644 index db65492..0000000 Binary files a/sample-data/nationsMF/nation2.parquet and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/nationsMF/nationsMF.parquet ---------------------------------------------------------------------- diff --git a/sample-data/nationsMF/nationsMF.parquet b/sample-data/nationsMF/nationsMF.parquet new file mode 100755 index 0000000..614487b Binary files /dev/null and b/sample-data/nationsMF/nationsMF.parquet differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/nationsSF/nation.parquet ---------------------------------------------------------------------- diff --git a/sample-data/nationsSF/nation.parquet b/sample-data/nationsSF/nation.parquet deleted file mode 100644 index db65492..0000000 Binary files a/sample-data/nationsSF/nation.parquet and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/nationsSF/nationsSF.parquet ---------------------------------------------------------------------- diff --git a/sample-data/nationsSF/nationsSF.parquet b/sample-data/nationsSF/nationsSF.parquet new file mode 100755 index 0000000..9109d8b Binary files /dev/null and b/sample-data/nationsSF/nationsSF.parquet differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/region.parquet ---------------------------------------------------------------------- diff --git a/sample-data/region.parquet b/sample-data/region.parquet old mode 100644 new mode 100755 index ab8122c..fc72fcf Binary files a/sample-data/region.parquet and b/sample-data/region.parquet differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/regionsMF/region.parquet ---------------------------------------------------------------------- diff --git a/sample-data/regionsMF/region.parquet b/sample-data/regionsMF/region.parquet deleted file mode 100644 index ab8122c..0000000 Binary files a/sample-data/regionsMF/region.parquet and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/regionsMF/region2.parquet ---------------------------------------------------------------------- diff --git a/sample-data/regionsMF/region2.parquet b/sample-data/regionsMF/region2.parquet deleted file mode 100644 index ab8122c..0000000 Binary files a/sample-data/regionsMF/region2.parquet and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/regionsMF/regionsMF_Typed.parquet ---------------------------------------------------------------------- diff --git a/sample-data/regionsMF/regionsMF_Typed.parquet b/sample-data/regionsMF/regionsMF_Typed.parquet new file mode 100755 index 0000000..eda97a2 Binary files /dev/null and b/sample-data/regionsMF/regionsMF_Typed.parquet differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/regionsSF/region.parquet ---------------------------------------------------------------------- diff --git a/sample-data/regionsSF/region.parquet b/sample-data/regionsSF/region.parquet deleted file mode 100644 index ab8122c..0000000 Binary files a/sample-data/regionsSF/region.parquet and /dev/null differ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/c2e32ac0/sample-data/regionsSF/regionsSF.parquet ---------------------------------------------------------------------- diff --git a/sample-data/regionsSF/regionsSF.parquet b/sample-data/regionsSF/regionsSF.parquet new file mode 100755 index 0000000..fc72fcf Binary files /dev/null and b/sample-data/regionsSF/regionsSF.parquet differ
