This is an automated email from the ASF dual-hosted git repository.
jacques pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new 065e323 [CALCITE-4841] Support decimal column type in CSV file format
065e323 is described below
commit 065e323da89ca73a46bec195efc7bd4019dc4f2f
Author: xwkuang5 <[email protected]>
AuthorDate: Mon Nov 1 09:09:49 2021 -0400
[CALCITE-4841] Support decimal column type in CSV file format
- Introduce a new `decimal` type in the CSV file format for
more accurate representation of decimal data.
- Implement the `decimal` SQL type using the BigDecimal JAVA type.
---
.../calcite/adapter/csv/CsvFilterableTable.java | 4 +-
.../calcite/adapter/csv/CsvScannableTable.java | 4 +-
.../adapter/csv/CsvStreamScannableTable.java | 4 +-
.../org/apache/calcite/adapter/csv/CsvTable.java | 5 +-
.../apache/calcite/adapter/file/CsvEnumerator.java | 166 ++++++++++++++++-----
.../apache/calcite/adapter/file/CsvFieldType.java | 78 ----------
.../org/apache/calcite/adapter/file/CsvTable.java | 4 +-
.../apache/calcite/adapter/file/FileFieldType.java | 2 -
.../calcite/adapter/file/CsvEnumeratorTest.java | 58 +++++++
.../calcite/adapter/file/FileAdapterTest.java | 68 +++++++++
file/src/test/resources/sales-csv/DECIMAL.csv | 4 +
11 files changed, 269 insertions(+), 128 deletions(-)
diff --git
a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java
b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java
index c61a04c..aa96bfa 100644
---
a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java
+++
b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvFilterableTable.java
@@ -18,11 +18,11 @@ package org.apache.calcite.adapter.csv;
import org.apache.calcite.DataContext;
import org.apache.calcite.adapter.file.CsvEnumerator;
-import org.apache.calcite.adapter.file.CsvFieldType;
import org.apache.calcite.adapter.java.JavaTypeFactory;
import org.apache.calcite.linq4j.AbstractEnumerable;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelProtoDataType;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
@@ -57,7 +57,7 @@ public class CsvFilterableTable extends CsvTable
@Override public Enumerable<@Nullable Object[]> scan(DataContext root,
List<RexNode> filters) {
JavaTypeFactory typeFactory = root.getTypeFactory();
- final List<CsvFieldType> fieldTypes = getFieldTypes(typeFactory);
+ final List<RelDataType> fieldTypes = getFieldTypes(typeFactory);
final @Nullable String[] filterValues = new String[fieldTypes.size()];
filters.removeIf(filter -> addFilter(filter, filterValues));
final List<Integer> fields = ImmutableIntList.identity(fieldTypes.size());
diff --git
a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java
b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java
index dd24e21..6d456fb 100644
---
a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java
+++
b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvScannableTable.java
@@ -18,11 +18,11 @@ package org.apache.calcite.adapter.csv;
import org.apache.calcite.DataContext;
import org.apache.calcite.adapter.file.CsvEnumerator;
-import org.apache.calcite.adapter.file.CsvFieldType;
import org.apache.calcite.adapter.java.JavaTypeFactory;
import org.apache.calcite.linq4j.AbstractEnumerable;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelProtoDataType;
import org.apache.calcite.schema.ScannableTable;
import org.apache.calcite.util.ImmutableIntList;
@@ -52,7 +52,7 @@ public class CsvScannableTable extends CsvTable
@Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
JavaTypeFactory typeFactory = root.getTypeFactory();
- final List<CsvFieldType> fieldTypes = getFieldTypes(typeFactory);
+ final List<RelDataType> fieldTypes = getFieldTypes(typeFactory);
final List<Integer> fields = ImmutableIntList.identity(fieldTypes.size());
final AtomicBoolean cancelFlag =
DataContext.Variable.CANCEL_FLAG.get(root);
return new AbstractEnumerable<@Nullable Object[]>() {
diff --git
a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java
b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java
index 2c01cd4..683c4c4 100644
---
a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java
+++
b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvStreamScannableTable.java
@@ -18,11 +18,11 @@ package org.apache.calcite.adapter.csv;
import org.apache.calcite.DataContext;
import org.apache.calcite.adapter.file.CsvEnumerator;
-import org.apache.calcite.adapter.file.CsvFieldType;
import org.apache.calcite.adapter.java.JavaTypeFactory;
import org.apache.calcite.linq4j.AbstractEnumerable;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
+import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelProtoDataType;
import org.apache.calcite.schema.ScannableTable;
import org.apache.calcite.schema.StreamableTable;
@@ -58,7 +58,7 @@ public class CsvStreamScannableTable extends CsvScannableTable
@Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
JavaTypeFactory typeFactory = root.getTypeFactory();
- final List<CsvFieldType> fieldTypes = getFieldTypes(typeFactory);
+ final List<RelDataType> fieldTypes = getFieldTypes(typeFactory);
final List<Integer> fields = ImmutableIntList.identity(fieldTypes.size());
final AtomicBoolean cancelFlag =
DataContext.Variable.CANCEL_FLAG.get(root);
return new AbstractEnumerable<@Nullable Object[]>() {
diff --git
a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTable.java
b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTable.java
index 9af670f..17ad898 100644
--- a/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTable.java
+++ b/example/csv/src/main/java/org/apache/calcite/adapter/csv/CsvTable.java
@@ -17,7 +17,6 @@
package org.apache.calcite.adapter.csv;
import org.apache.calcite.adapter.file.CsvEnumerator;
-import org.apache.calcite.adapter.file.CsvFieldType;
import org.apache.calcite.adapter.java.JavaTypeFactory;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
@@ -37,7 +36,7 @@ public abstract class CsvTable extends AbstractTable {
protected final Source source;
protected final @Nullable RelProtoDataType protoRowType;
private @Nullable RelDataType rowType;
- private @Nullable List<CsvFieldType> fieldTypes;
+ private @Nullable List<RelDataType> fieldTypes;
/** Creates a CsvTable. */
CsvTable(Source source, @Nullable RelProtoDataType protoRowType) {
@@ -57,7 +56,7 @@ public abstract class CsvTable extends AbstractTable {
}
/** Returns the field types of this CSV table. */
- public List<CsvFieldType> getFieldTypes(RelDataTypeFactory typeFactory) {
+ public List<RelDataType> getFieldTypes(RelDataTypeFactory typeFactory) {
if (fieldTypes == null) {
fieldTypes = new ArrayList<>();
CsvEnumerator.deduceRowType((JavaTypeFactory) typeFactory, source,
diff --git
a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java
b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java
index a2f97af..600ec89 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvEnumerator.java
@@ -25,21 +25,32 @@ import org.apache.calcite.util.ImmutableIntList;
import org.apache.calcite.util.ImmutableNullableList;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Source;
+import org.apache.calcite.util.trace.CalciteLogger;
import org.apache.commons.lang3.time.FastDateFormat;
import au.com.bytecode.opencsv.CSVReader;
+import com.google.common.annotations.VisibleForTesting;
+
import org.checkerframework.checker.nullness.qual.Nullable;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.Locale;
import java.util.Objects;
import java.util.TimeZone;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.calcite.linq4j.Nullness.castNonNull;
@@ -48,6 +59,9 @@ import static org.apache.calcite.linq4j.Nullness.castNonNull;
* @param <E> Row type
*/
public class CsvEnumerator<E> implements Enumerator<E> {
+
+ private static final CalciteLogger LOGGER = new CalciteLogger(
+ LoggerFactory.getLogger(CsvEnumerator.class));
private final CSVReader reader;
private final @Nullable List<@Nullable String> filterValues;
private final AtomicBoolean cancelFlag;
@@ -57,6 +71,8 @@ public class CsvEnumerator<E> implements Enumerator<E> {
private static final FastDateFormat TIME_FORMAT_DATE;
private static final FastDateFormat TIME_FORMAT_TIME;
private static final FastDateFormat TIME_FORMAT_TIMESTAMP;
+ private static final Pattern DECIMAL_TYPE_PATTERN = Pattern
+ .compile("\"decimal\\(([0-9]+),([0-9]+)\\)");
static {
final TimeZone gmt = TimeZone.getTimeZone("GMT");
@@ -67,7 +83,7 @@ public class CsvEnumerator<E> implements Enumerator<E> {
}
public CsvEnumerator(Source source, AtomicBoolean cancelFlag,
- List<CsvFieldType> fieldTypes, List<Integer> fields) {
+ List<RelDataType> fieldTypes, List<Integer> fields) {
//noinspection unchecked
this(source, cancelFlag, false, null,
(RowConverter<E>) converter(fieldTypes, fields));
@@ -91,7 +107,7 @@ public class CsvEnumerator<E> implements Enumerator<E> {
}
}
- private static RowConverter<?> converter(List<CsvFieldType> fieldTypes,
+ private static RowConverter<?> converter(List<RelDataType> fieldTypes,
List<Integer> fields) {
if (fields.size() == 1) {
final int field = fields.get(0);
@@ -102,21 +118,14 @@ public class CsvEnumerator<E> implements Enumerator<E> {
}
public static RowConverter<@Nullable Object[]> arrayConverter(
- List<CsvFieldType> fieldTypes, List<Integer> fields, boolean stream) {
+ List<RelDataType> fieldTypes, List<Integer> fields, boolean stream) {
return new ArrayRowConverter(fieldTypes, fields, stream);
}
/** Deduces the names and types of a table's columns by reading the first
line
* of a CSV file. */
- static RelDataType deduceRowType(JavaTypeFactory typeFactory, Source source,
- List<CsvFieldType> fieldTypes) {
- return deduceRowType(typeFactory, source, fieldTypes, false);
- }
-
- /** Deduces the names and types of a table's columns by reading the first
line
- * of a CSV file. */
public static RelDataType deduceRowType(JavaTypeFactory typeFactory,
- Source source, @Nullable List<CsvFieldType> fieldTypes, Boolean stream) {
+ Source source, @Nullable List<RelDataType> fieldTypes, Boolean stream) {
final List<RelDataType> types = new ArrayList<>();
final List<String> names = new ArrayList<>();
if (stream) {
@@ -130,30 +139,69 @@ public class CsvEnumerator<E> implements Enumerator<E> {
}
for (String string : strings) {
final String name;
- final CsvFieldType fieldType;
+ final RelDataType fieldType;
final int colon = string.indexOf(':');
if (colon >= 0) {
name = string.substring(0, colon);
String typeString = string.substring(colon + 1);
- fieldType = CsvFieldType.of(typeString);
- if (fieldType == null) {
- System.out.println("WARNING: Found unknown type: "
- + typeString + " in file: " + source.path()
- + " for column: " + name
- + ". Will assume the type of column is string");
+ Matcher decimalMatcher = DECIMAL_TYPE_PATTERN.matcher(typeString);
+ if (decimalMatcher.matches()) {
+ int precision = Integer.parseInt(decimalMatcher.group(1));
+ int scale = Integer.parseInt(decimalMatcher.group(2));
+ fieldType = parseDecimalSqlType(typeFactory, precision, scale);
+ } else {
+ switch (typeString) {
+ case "string":
+ fieldType = toNullableRelDataType(typeFactory,
SqlTypeName.VARCHAR);
+ break;
+ case "boolean":
+ fieldType = toNullableRelDataType(typeFactory,
SqlTypeName.BOOLEAN);
+ break;
+ case "byte":
+ fieldType = toNullableRelDataType(typeFactory,
SqlTypeName.TINYINT);
+ break;
+ case "char":
+ fieldType = toNullableRelDataType(typeFactory, SqlTypeName.CHAR);
+ break;
+ case "short":
+ fieldType = toNullableRelDataType(typeFactory,
SqlTypeName.SMALLINT);
+ break;
+ case "int":
+ fieldType = toNullableRelDataType(typeFactory,
SqlTypeName.INTEGER);
+ break;
+ case "long":
+ fieldType = toNullableRelDataType(typeFactory,
SqlTypeName.BIGINT);
+ break;
+ case "float":
+ fieldType = toNullableRelDataType(typeFactory, SqlTypeName.REAL);
+ break;
+ case "double":
+ fieldType = toNullableRelDataType(typeFactory,
SqlTypeName.DOUBLE);
+ break;
+ case "date":
+ fieldType = toNullableRelDataType(typeFactory, SqlTypeName.DATE);
+ break;
+ case "timestamp":
+ fieldType = toNullableRelDataType(typeFactory,
SqlTypeName.TIMESTAMP);
+ break;
+ case "time":
+ fieldType = toNullableRelDataType(typeFactory, SqlTypeName.TIME);
+ break;
+ default:
+ LOGGER.warn(
+ "Found unknown type: {} in file: {} for column: {}. Will
assume the type of "
+ + "column is string.",
+ typeString, source.path(), name);
+ fieldType = toNullableRelDataType(typeFactory,
SqlTypeName.VARCHAR);
+ break;
+ }
}
} else {
name = string;
- fieldType = null;
- }
- final RelDataType type;
- if (fieldType == null) {
- type = typeFactory.createSqlType(SqlTypeName.VARCHAR);
- } else {
- type = fieldType.toType(typeFactory);
+ fieldType = typeFactory.createSqlType(SqlTypeName.VARCHAR);
}
names.add(name);
- types.add(type);
+ types.add(fieldType);
if (fieldTypes != null) {
fieldTypes.add(fieldType);
}
@@ -237,6 +285,11 @@ public class CsvEnumerator<E> implements Enumerator<E> {
return integers;
}
+ private static RelDataType toNullableRelDataType(JavaTypeFactory typeFactory,
+ SqlTypeName sqlTypeName) {
+ return
typeFactory.createTypeWithNullability(typeFactory.createSqlType(sqlTypeName),
true);
+ }
+
/** Row converter.
*
* @param <E> element type */
@@ -244,32 +297,32 @@ public class CsvEnumerator<E> implements Enumerator<E> {
abstract E convertRow(@Nullable String[] rows);
@SuppressWarnings("JavaUtilDate")
- protected @Nullable Object convert(@Nullable CsvFieldType fieldType,
@Nullable String string) {
+ protected @Nullable Object convert(@Nullable RelDataType fieldType,
@Nullable String string) {
if (fieldType == null || string == null) {
return string;
}
- switch (fieldType) {
+ switch (fieldType.getSqlTypeName()) {
case BOOLEAN:
if (string.length() == 0) {
return null;
}
return Boolean.parseBoolean(string);
- case BYTE:
+ case TINYINT:
if (string.length() == 0) {
return null;
}
return Byte.parseByte(string);
- case SHORT:
+ case SMALLINT:
if (string.length() == 0) {
return null;
}
return Short.parseShort(string);
- case INT:
+ case INTEGER:
if (string.length() == 0) {
return null;
}
return Integer.parseInt(string);
- case LONG:
+ case BIGINT:
if (string.length() == 0) {
return null;
}
@@ -284,6 +337,11 @@ public class CsvEnumerator<E> implements Enumerator<E> {
return null;
}
return Double.parseDouble(string);
+ case DECIMAL:
+ if (string.length() == 0) {
+ return null;
+ }
+ return parseDecimal(fieldType.getPrecision(), fieldType.getScale(),
string);
case DATE:
if (string.length() == 0) {
return null;
@@ -314,22 +372,56 @@ public class CsvEnumerator<E> implements Enumerator<E> {
} catch (ParseException e) {
return null;
}
- case STRING:
+ case VARCHAR:
default:
return string;
}
}
}
+ private static RelDataType parseDecimalSqlType(JavaTypeFactory typeFactory,
int precision,
+ int scale) {
+ checkArgument(precision > 0, "DECIMAL type must have precision > 0. Found
%s", precision);
+ checkArgument(scale >= 0, "DECIMAL type must have scale >= 0. Found %s",
scale);
+ checkArgument(precision >= scale,
+ "DECIMAL type must have precision >= scale. Found precision (%s) and
scale (%s).",
+ precision, scale);
+ return typeFactory.createTypeWithNullability(
+ typeFactory.createSqlType(SqlTypeName.DECIMAL, precision, scale),
true);
+ }
+
+ @VisibleForTesting
+ protected static BigDecimal parseDecimal(int precision, int scale, String
string) {
+ BigDecimal result = new BigDecimal(string);
+ // If the parsed value has more fractional digits than the specified
scale, round ties away
+ // from 0.
+ if (result.scale() > scale) {
+ LOGGER.warn(
+ "Decimal value {} exceeds declared scale ({}). Performing rounding
to keep the "
+ + "first {} fractional digits.",
+ result, scale, scale);
+ result = result.setScale(scale, RoundingMode.HALF_UP);
+ }
+ // Throws an exception if the parsed value has more digits to the left of
the decimal point
+ // than the specified value.
+ if (result.precision() - result.scale() > precision - scale) {
+ throw new IllegalArgumentException(String
+ .format(Locale.ROOT, "Decimal value %s exceeds declared precision
(%d) and scale (%d).",
+ result, precision, scale));
+ }
+ return result;
+ }
+
/** Array row converter. */
static class ArrayRowConverter extends RowConverter<@Nullable Object[]> {
+
/** Field types. List must not be null, but any element may be null. */
- private final List<CsvFieldType> fieldTypes;
+ private final List<RelDataType> fieldTypes;
private final ImmutableIntList fields;
/** Whether the row to convert is from a stream. */
private final boolean stream;
- ArrayRowConverter(List<CsvFieldType> fieldTypes, List<Integer> fields,
+ ArrayRowConverter(List<RelDataType> fieldTypes, List<Integer> fields,
boolean stream) {
this.fieldTypes = ImmutableNullableList.copyOf(fieldTypes);
this.fields = ImmutableIntList.copyOf(fields);
@@ -366,10 +458,10 @@ public class CsvEnumerator<E> implements Enumerator<E> {
/** Single column row converter. */
private static class SingleColumnRowConverter extends RowConverter<Object> {
- private final CsvFieldType fieldType;
+ private final RelDataType fieldType;
private final int fieldIndex;
- private SingleColumnRowConverter(CsvFieldType fieldType, int fieldIndex) {
+ private SingleColumnRowConverter(RelDataType fieldType, int fieldIndex) {
this.fieldType = fieldType;
this.fieldIndex = fieldIndex;
}
diff --git
a/file/src/main/java/org/apache/calcite/adapter/file/CsvFieldType.java
b/file/src/main/java/org/apache/calcite/adapter/file/CsvFieldType.java
deleted file mode 100644
index ff11225..0000000
--- a/file/src/main/java/org/apache/calcite/adapter/file/CsvFieldType.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.calcite.adapter.file;
-
-import org.apache.calcite.adapter.java.JavaTypeFactory;
-import org.apache.calcite.linq4j.tree.Primitive;
-import org.apache.calcite.rel.type.RelDataType;
-
-import org.checkerframework.checker.nullness.qual.Nullable;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Type of a field in a CSV file.
- *
- * <p>Usually, and unless specified explicitly in the header row, a field is
- * of type {@link #STRING}. But specifying the field type in the header row
- * makes it easier to write SQL.</p>
- */
-public enum CsvFieldType {
- STRING(String.class, "string"),
- BOOLEAN(Primitive.BOOLEAN),
- BYTE(Primitive.BYTE),
- CHAR(Primitive.CHAR),
- SHORT(Primitive.SHORT),
- INT(Primitive.INT),
- LONG(Primitive.LONG),
- FLOAT(Primitive.FLOAT),
- DOUBLE(Primitive.DOUBLE),
- DATE(java.sql.Date.class, "date"),
- TIME(java.sql.Time.class, "time"),
- TIMESTAMP(java.sql.Timestamp.class, "timestamp");
-
- private final Class clazz;
- private final String simpleName;
-
- private static final Map<String, CsvFieldType> MAP = new HashMap<>();
-
- static {
- for (CsvFieldType value : values()) {
- MAP.put(value.simpleName, value);
- }
- }
-
- CsvFieldType(Primitive primitive) {
- this(primitive.getBoxClass(), primitive.getPrimitiveName());
- }
-
- CsvFieldType(Class clazz, String simpleName) {
- this.clazz = clazz;
- this.simpleName = simpleName;
- }
-
- public RelDataType toType(JavaTypeFactory typeFactory) {
- RelDataType javaType = typeFactory.createJavaType(clazz);
- RelDataType sqlType = typeFactory.createSqlType(javaType.getSqlTypeName());
- return typeFactory.createTypeWithNullability(sqlType, true);
- }
-
- public static @Nullable CsvFieldType of(String typeString) {
- return MAP.get(typeString);
- }
-}
diff --git a/file/src/main/java/org/apache/calcite/adapter/file/CsvTable.java
b/file/src/main/java/org/apache/calcite/adapter/file/CsvTable.java
index 1ab4e70..eb704ae 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/CsvTable.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/CsvTable.java
@@ -36,7 +36,7 @@ public abstract class CsvTable extends AbstractTable {
protected final Source source;
protected final RelProtoDataType protoRowType;
private RelDataType rowType;
- private List<CsvFieldType> fieldTypes;
+ private List<RelDataType> fieldTypes;
/** Creates a CsvTable. */
CsvTable(Source source, RelProtoDataType protoRowType) {
@@ -56,7 +56,7 @@ public abstract class CsvTable extends AbstractTable {
}
/** Returns the field types of this CSV table. */
- public List<CsvFieldType> getFieldTypes(RelDataTypeFactory typeFactory) {
+ public List<RelDataType> getFieldTypes(RelDataTypeFactory typeFactory) {
if (fieldTypes == null) {
fieldTypes = new ArrayList<>();
CsvEnumerator.deduceRowType((JavaTypeFactory) typeFactory, source,
diff --git
a/file/src/main/java/org/apache/calcite/adapter/file/FileFieldType.java
b/file/src/main/java/org/apache/calcite/adapter/file/FileFieldType.java
index 85ed9bd..822966a 100644
--- a/file/src/main/java/org/apache/calcite/adapter/file/FileFieldType.java
+++ b/file/src/main/java/org/apache/calcite/adapter/file/FileFieldType.java
@@ -30,8 +30,6 @@ import java.util.Map;
* <p>Usually, and unless specified explicitly in the header row, a field is
* of type {@link #STRING}. But specifying the field type in the fields
* makes it easier to write SQL.
- *
- * <p>Trivially modified from CsvFieldType.
*/
enum FileFieldType {
STRING(null, String.class),
diff --git
a/file/src/test/java/org/apache/calcite/adapter/file/CsvEnumeratorTest.java
b/file/src/test/java/org/apache/calcite/adapter/file/CsvEnumeratorTest.java
new file mode 100644
index 0000000..0a60329
--- /dev/null
+++ b/file/src/test/java/org/apache/calcite/adapter/file/CsvEnumeratorTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.file;
+
+import org.junit.jupiter.api.Test;
+
+import java.math.BigDecimal;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Test for the {@link CsvEnumerator}.
+ */
+class CsvEnumeratorTest {
+
+ @Test void testParseDecimalScaleRounding() {
+ assertEquals(new BigDecimal("123.45"),
+ CsvEnumerator.parseDecimal(5, 2, "123.45"));
+ assertEquals(new BigDecimal("123.46"),
+ CsvEnumerator.parseDecimal(5, 2, "123.455"));
+ assertEquals(new BigDecimal("-123.46"),
+ CsvEnumerator.parseDecimal(5, 2, "-123.455"));
+ assertEquals(new BigDecimal("123.45"),
+ CsvEnumerator.parseDecimal(5, 2, "123.454"));
+ assertEquals(new BigDecimal("-123.45"),
+ CsvEnumerator.parseDecimal(5, 2, "-123.454"));
+ }
+
+ @Test void testParseDecimalPrecisionExceeded() {
+ assertThrows(IllegalArgumentException.class,
+ () -> CsvEnumerator.parseDecimal(4, 0, "1e+5"));
+ assertThrows(IllegalArgumentException.class,
+ () -> CsvEnumerator.parseDecimal(4, 0, "-1e+5"));
+ assertThrows(IllegalArgumentException.class,
+ () -> CsvEnumerator.parseDecimal(4, 0, "12345"));
+ assertThrows(IllegalArgumentException.class,
+ () -> CsvEnumerator.parseDecimal(4, 0, "-12345"));
+ assertThrows(IllegalArgumentException.class,
+ () -> CsvEnumerator.parseDecimal(4, 2, "123.45"));
+ assertThrows(IllegalArgumentException.class,
+ () -> CsvEnumerator.parseDecimal(4, 2, "-123.45"));
+ }
+}
diff --git
a/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTest.java
b/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTest.java
index 6abb040..bb0a41f 100644
--- a/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTest.java
+++ b/file/src/test/java/org/apache/calcite/adapter/file/FileAdapterTest.java
@@ -29,6 +29,7 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
+import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
@@ -51,6 +52,7 @@ import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.isA;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
/**
* System test of the Calcite file adapter, which can read and parse
@@ -619,6 +621,72 @@ class FileAdapterTest {
return sb.append(')').toString();
}
+ @Test void testDecimalType() {
+ sql("sales-csv", "select BUDGET from sales.\"DECIMAL\"")
+ .checking(resultSet -> {
+ try {
+ ResultSetMetaData metaData = resultSet.getMetaData();
+ assertEquals("DECIMAL", metaData.getColumnTypeName(1));
+ assertEquals(18, metaData.getPrecision(1));
+ assertEquals(2, metaData.getScale(1));
+ } catch (SQLException e) {
+ throw TestUtil.rethrow(e);
+ }
+ })
+ .ok();
+ }
+
+ @Test void testDecimalTypeArithmeticOperations() {
+ sql("sales-csv", "select BUDGET + 100.0 from sales.\"DECIMAL\" where
DEPTNO = 10")
+ .checking(resultSet -> {
+ try {
+ resultSet.next();
+ assertEquals(0,
+ resultSet.getBigDecimal(1).compareTo(new BigDecimal("200")));
+ assertFalse(resultSet.next());
+ } catch (SQLException e) {
+ throw TestUtil.rethrow(e);
+ }
+ })
+ .ok();
+ sql("sales-csv", "select BUDGET - 100.0 from sales.\"DECIMAL\" where
DEPTNO = 10")
+ .checking(resultSet -> {
+ try {
+ resultSet.next();
+ assertEquals(0,
+ resultSet.getBigDecimal(1).compareTo(new BigDecimal("0")));
+ assertFalse(resultSet.next());
+ } catch (SQLException e) {
+ throw TestUtil.rethrow(e);
+ }
+ })
+ .ok();
+ sql("sales-csv", "select BUDGET * 0.01 from sales.\"DECIMAL\" where DEPTNO
= 10")
+ .checking(resultSet -> {
+ try {
+ resultSet.next();
+ assertEquals(0,
+ resultSet.getBigDecimal(1).compareTo(new BigDecimal("1")));
+ assertFalse(resultSet.next());
+ } catch (SQLException e) {
+ throw TestUtil.rethrow(e);
+ }
+ })
+ .ok();
+ sql("sales-csv", "select BUDGET / 100 from sales.\"DECIMAL\" where DEPTNO
= 10")
+ .checking(resultSet -> {
+ try {
+ resultSet.next();
+ assertEquals(0,
+ resultSet.getBigDecimal(1).compareTo(new BigDecimal("1")));
+ assertFalse(resultSet.next());
+ } catch (SQLException e) {
+ throw TestUtil.rethrow(e);
+ }
+ })
+ .ok();
+ }
+
@Test void testDateType() throws SQLException {
Properties info = new Properties();
info.put("model", FileAdapterTests.jsonPath("bug"));
diff --git a/file/src/test/resources/sales-csv/DECIMAL.csv
b/file/src/test/resources/sales-csv/DECIMAL.csv
new file mode 100644
index 0000000..03247cb
--- /dev/null
+++ b/file/src/test/resources/sales-csv/DECIMAL.csv
@@ -0,0 +1,4 @@
+DEPTNO:int,BUDGET:"decimal(18,2)"
+10,100.00
+20,100.01
+30,-100.01
