This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 1dda1ad Add mapping to Iceberg for external name-based schemas (#338)
1dda1ad is described below
commit 1dda1ada9c5b338fd0383b7f528707b8506c49a6
Author: Ryan Blue <[email protected]>
AuthorDate: Wed Jul 31 14:44:05 2019 -0700
Add mapping to Iceberg for external name-based schemas (#338)
---
.../main/java/org/apache/iceberg/SchemaUpdate.java | 32 ++-
.../java/org/apache/iceberg/TableProperties.java | 2 +
.../org/apache/iceberg/mapping/MappedField.java | 96 ++++++++
.../org/apache/iceberg/mapping/MappedFields.java | 110 +++++++++
.../org/apache/iceberg/mapping/MappingUtil.java | 261 +++++++++++++++++++++
.../org/apache/iceberg/mapping/NameMapping.java | 66 ++++++
.../apache/iceberg/mapping/NameMappingParser.java | 144 ++++++++++++
.../java/org/apache/iceberg/util/JsonUtil.java | 10 +
.../java/org/apache/iceberg/TableTestBase.java | 6 +-
.../test/java/org/apache/iceberg/TestTables.java | 8 +-
.../apache/iceberg/mapping/TestMappingUpdates.java | 249 ++++++++++++++++++++
.../apache/iceberg/mapping/TestNameMapping.java | 250 ++++++++++++++++++++
12 files changed, 1226 insertions(+), 8 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java
b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java
index 48cbb04..d94f449 100644
--- a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java
+++ b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java
@@ -28,9 +28,14 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import org.apache.iceberg.mapping.MappingUtil;
+import org.apache.iceberg.mapping.NameMapping;
+import org.apache.iceberg.mapping.NameMappingParser;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.TypeUtil;
import org.apache.iceberg.types.Types;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.iceberg.types.Types.NestedField.optional;
import static org.apache.iceberg.types.Types.NestedField.required;
@@ -39,6 +44,7 @@ import static
org.apache.iceberg.types.Types.NestedField.required;
* Schema evolution API implementation.
*/
class SchemaUpdate implements UpdateSchema {
+ private static final Logger LOG =
LoggerFactory.getLogger(SchemaUpdate.class);
private static final int TABLE_ROOT_ID = -1;
private final TableOperations ops;
@@ -200,7 +206,7 @@ class SchemaUpdate implements UpdateSchema {
@Override
public void commit() {
- TableMetadata update = base.updateSchema(apply(), lastColumnId);
+ TableMetadata update = applyChangesToMapping(base.updateSchema(apply(),
lastColumnId));
ops.commit(base, update);
}
@@ -210,6 +216,30 @@ class SchemaUpdate implements UpdateSchema {
return next;
}
+ private TableMetadata applyChangesToMapping(TableMetadata metadata) {
+ String mappingJson =
metadata.property(TableProperties.DEFAULT_NAME_MAPPING, null);
+ if (mappingJson != null) {
+ try {
+ // parse and update the mapping
+ NameMapping mapping = NameMappingParser.fromJson(mappingJson);
+ NameMapping updated = MappingUtil.update(mapping, updates, adds);
+
+ // replace the table property
+ Map<String, String> updatedProperties = Maps.newHashMap();
+ updatedProperties.putAll(metadata.properties());
+ updatedProperties.put(TableProperties.DEFAULT_NAME_MAPPING,
NameMappingParser.toJson(updated));
+
+ return metadata.replaceProperties(updatedProperties);
+
+ } catch (RuntimeException e) {
+ // log the error, but do not fail the update
+ LOG.warn("Failed to update external schema mapping: {}", mappingJson,
e);
+ }
+ }
+
+ return metadata;
+ }
+
private static Schema applyChanges(Schema schema, List<Integer> deletes,
Map<Integer, Types.NestedField> updates,
Multimap<Integer, Types.NestedField>
adds) {
diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java
b/core/src/main/java/org/apache/iceberg/TableProperties.java
index d426dd3..4dd778d 100644
--- a/core/src/main/java/org/apache/iceberg/TableProperties.java
+++ b/core/src/main/java/org/apache/iceberg/TableProperties.java
@@ -91,4 +91,6 @@ public class TableProperties {
public static final String DEFAULT_WRITE_METRICS_MODE =
"write.metadata.metrics.default";
public static final String DEFAULT_WRITE_METRICS_MODE_DEFAULT =
"truncate(16)";
+
+ public static final String DEFAULT_NAME_MAPPING =
"schema.name-mapping.default";
}
diff --git a/core/src/main/java/org/apache/iceberg/mapping/MappedField.java
b/core/src/main/java/org/apache/iceberg/mapping/MappedField.java
new file mode 100644
index 0000000..2a56afa
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/mapping/MappedField.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mapping;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSet;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * An immutable mapping between a field ID and a set of names.
+ */
+public class MappedField {
+
+ static MappedField of(Integer id, String name) {
+ return new MappedField(id, ImmutableSet.of(name), null);
+ }
+
+ static MappedField of(Integer id, Iterable<String> names) {
+ return new MappedField(id, names, null);
+ }
+
+ static MappedField of(Integer id, String name, MappedFields nestedMapping) {
+ return new MappedField(id, ImmutableSet.of(name), nestedMapping);
+ }
+
+ static MappedField of(Integer id, Iterable<String> names, MappedFields
nestedMapping) {
+ return new MappedField(id, names, nestedMapping);
+ }
+
+ private final Set<String> names;
+ private Integer id;
+ private MappedFields nestedMapping;
+
+ private MappedField(Integer id, Iterable<String> names, MappedFields nested)
{
+ this.id = id;
+ this.names = ImmutableSet.copyOf(names);
+ this.nestedMapping = nested;
+ }
+
+ public Integer id() {
+ return id;
+ }
+
+ public Set<String> names() {
+ return names;
+ }
+
+ public MappedFields nestedMapping() {
+ return nestedMapping;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (other == null || getClass() != other.getClass()) {
+ return false;
+ }
+
+ MappedField that = (MappedField) other;
+ return names.equals(that.names) &&
+ Objects.equals(id, that.id) &&
+ Objects.equals(nestedMapping, that.nestedMapping);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(names, id, nestedMapping);
+ }
+
+ @Override
+ public String toString() {
+ return "([" + Joiner.on(", ").join(names) + "] -> " + (id != null ? id :
"?") +
+ (nestedMapping != null ? ", " + nestedMapping + ")" : ")");
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/mapping/MappedFields.java
b/core/src/main/java/org/apache/iceberg/mapping/MappedFields.java
new file mode 100644
index 0000000..824257b
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/mapping/MappedFields.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mapping;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class MappedFields {
+
+ static MappedFields of(MappedField... fields) {
+ return new MappedFields(ImmutableList.copyOf(fields));
+ }
+
+ static MappedFields of(List<MappedField> fields) {
+ return new MappedFields(fields);
+ }
+
+ private final List<MappedField> fields;
+ private final Map<String, Integer> nameToId;
+ private final Map<Integer, MappedField> idToField;
+
+ private MappedFields(List<MappedField> fields) {
+ this.fields = ImmutableList.copyOf(fields);
+ this.nameToId = indexIds(fields);
+ this.idToField = indexFields(fields);
+ }
+
+ public MappedField field(int id) {
+ return idToField.get(id);
+ }
+
+ public Integer id(String name) {
+ return nameToId.get(name);
+ }
+
+ public int size() {
+ return fields.size();
+ }
+
+ private static Map<String, Integer> indexIds(List<MappedField> fields) {
+ ImmutableMap.Builder<String, Integer> builder = ImmutableMap.builder();
+ fields.forEach(field ->
+ field.names().forEach(name -> {
+ Integer id = field.id();
+ if (id != null) {
+ builder.put(name, id);
+ }
+ }));
+ return builder.build();
+ }
+
+ private static Map<Integer, MappedField> indexFields(List<MappedField>
fields) {
+ ImmutableMap.Builder<Integer, MappedField> builder =
ImmutableMap.builder();
+ fields.forEach(field -> {
+ Integer id = field.id();
+ if (id != null) {
+ builder.put(id, field);
+ }
+ });
+ return builder.build();
+ }
+
+ public List<MappedField> fields() {
+ return fields;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (other == null || getClass() != other.getClass()) {
+ return false;
+ }
+
+ return fields.equals(((MappedFields) other).fields);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(fields);
+ }
+
+ @Override
+ public String toString() {
+ return "[ " + Joiner.on(", ").join(fields) + " ]";
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/mapping/MappingUtil.java
b/core/src/main/java/org/apache/iceberg/mapping/MappingUtil.java
new file mode 100644
index 0000000..95d8b4e
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/mapping/MappingUtil.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mapping;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.types.Types;
+
+public class MappingUtil {
+ private static final Joiner DOT = Joiner.on('.');
+
+ private MappingUtil() {
+ }
+
+ /**
+ * Create a name-based mapping for a schema.
+ * <p>
+ * The mapping returned by this method will use the schema's name for each
field.
+ *
+ * @param schema a {@link Schema}
+ * @return a {@link NameMapping} initialized with the schema's fields and
names
+ */
+ public static NameMapping create(Schema schema) {
+ return new NameMapping(TypeUtil.visit(schema, CreateMapping.INSTANCE));
+ }
+
+ /**
+ * Update a name-based mapping using changes to a schema.
+ *
+ * @param mapping a name-based mapping
+ * @param updates a map from field ID to updated field definitions
+ * @param adds a map from parent field ID to nested fields to be added
+ * @return an updated mapping with names added to renamed fields and the
mapping extended for new fields
+ */
+ public static NameMapping update(NameMapping mapping,
+ Map<Integer, Types.NestedField> updates,
+ Multimap<Integer, Types.NestedField> adds) {
+ return new NameMapping(visit(mapping, new UpdateMapping(updates, adds)));
+ }
+
+ static Map<Integer, MappedField> indexById(MappedFields mapping) {
+ return visit(mapping, new IndexById());
+ }
+
+ static Map<String, MappedField> indexByName(MappedFields mapping) {
+ return visit(mapping, IndexByName.INSTANCE);
+ }
+
+ private static class UpdateMapping implements Visitor<MappedFields,
MappedField> {
+ private final Map<Integer, Types.NestedField> updates;
+ private final Multimap<Integer, Types.NestedField> adds;
+
+ private UpdateMapping(Map<Integer, Types.NestedField> updates,
Multimap<Integer, Types.NestedField> adds) {
+ this.updates = updates;
+ this.adds = adds;
+ }
+
+ @Override
+ public MappedFields mapping(NameMapping mapping, MappedFields result) {
+ return addNewFields(result, -1 /* parent ID used to add top-level fields
*/);
+ }
+
+ @Override
+ public MappedFields fields(MappedFields fields, List<MappedField>
fieldResults) {
+ return MappedFields.of(fieldResults);
+ }
+
+ @Override
+ public MappedField field(MappedField field, MappedFields fieldResult) {
+ // update this field's names
+ Set<String> fieldNames = Sets.newHashSet(field.names());
+ Types.NestedField update = updates.get(field.id());
+ if (update != null) {
+ fieldNames.add(update.name());
+ }
+
+ // add a new mapping for any new nested fields
+ MappedFields nestedMapping = addNewFields(fieldResult, field.id());
+ return MappedField.of(field.id(), fieldNames, nestedMapping);
+ }
+
+ private MappedFields addNewFields(MappedFields mapping, int parentId) {
+ Collection<Types.NestedField> fieldsToAdd = adds.get(parentId);
+ if (fieldsToAdd == null || fieldsToAdd.isEmpty()) {
+ return mapping;
+ }
+
+ List<MappedField> fields = Lists.newArrayList();
+ if (mapping != null) {
+ fields.addAll(mapping.fields());
+ }
+
+ for (Types.NestedField add : fieldsToAdd) {
+ MappedFields nestedMapping = TypeUtil.visit(add.type(),
CreateMapping.INSTANCE);
+ fields.add(MappedField.of(add.fieldId(), add.name(), nestedMapping));
+ }
+
+ return MappedFields.of(fields);
+ }
+ }
+
+ private static class IndexByName implements Visitor<Map<String,
MappedField>, Map<String, MappedField>> {
+ static final IndexByName INSTANCE = new IndexByName();
+
+ @Override
+ public Map<String, MappedField> mapping(NameMapping mapping, Map<String,
MappedField> result) {
+ return result;
+ }
+
+ @Override
+ public Map<String, MappedField> fields(MappedFields fields,
List<Map<String, MappedField>> fieldResults) {
+ // merge the results of each field
+ ImmutableMap.Builder<String, MappedField> builder =
ImmutableMap.builder();
+ for (Map<String, MappedField> results : fieldResults) {
+ builder.putAll(results);
+ }
+ return builder.build();
+ }
+
+ @Override
+ public Map<String, MappedField> field(MappedField field, Map<String,
MappedField> fieldResult) {
+ ImmutableMap.Builder<String, MappedField> builder =
ImmutableMap.builder();
+
+ if (fieldResult != null) {
+ for (String name : field.names()) {
+ for (Map.Entry<String, MappedField> entry : fieldResult.entrySet()) {
+ String fullName = DOT.join(name, entry.getKey());
+ builder.put(fullName, entry.getValue());
+ }
+ }
+ }
+
+ for (String name : field.names()) {
+ builder.put(name, field);
+ }
+
+ return builder.build();
+ }
+ }
+
+ private static class IndexById implements Visitor<Map<Integer, MappedField>,
Map<Integer, MappedField>> {
+ private final Map<Integer, MappedField> result = Maps.newHashMap();
+
+ @Override
+ public Map<Integer, MappedField> mapping(NameMapping mapping, Map<Integer,
MappedField> fieldsResult) {
+ return fieldsResult;
+ }
+
+ @Override
+ public Map<Integer, MappedField> fields(MappedFields fields,
List<Map<Integer, MappedField>> fieldResults) {
+ return result;
+ }
+
+ @Override
+ public Map<Integer, MappedField> field(MappedField field, Map<Integer,
MappedField> fieldResult) {
+ Preconditions.checkState(!result.containsKey(field.id()), "Invalid
mapping: ID %s is not unique", field.id());
+ result.put(field.id(), field);
+ return result;
+ }
+ }
+
+ private interface Visitor<S, T> {
+ S mapping(NameMapping mapping, S result);
+ S fields(MappedFields fields, List<T> fieldResults);
+ T field(MappedField field, S fieldResult);
+ }
+
+ private static <S, T> S visit(NameMapping mapping, Visitor<S, T> visitor) {
+ return visitor.mapping(mapping, visit(mapping.asMappedFields(), visitor));
+ }
+
+ private static <S, T> S visit(MappedFields mapping, Visitor<S, T> visitor) {
+ if (mapping == null) {
+ return null;
+ }
+
+ List<T> fieldResults = Lists.newArrayList();
+ for (MappedField field : mapping.fields()) {
+ fieldResults.add(visitor.field(field, visit(field.nestedMapping(),
visitor)));
+ }
+
+ return visitor.fields(mapping, fieldResults);
+ }
+
+ private static class CreateMapping extends
TypeUtil.SchemaVisitor<MappedFields> {
+ private static final CreateMapping INSTANCE = new CreateMapping();
+
+ private CreateMapping() {
+ }
+
+ @Override
+ public MappedFields schema(Schema schema, MappedFields structResult) {
+ return structResult;
+ }
+
+ @Override
+ public MappedFields struct(Types.StructType struct, List<MappedFields>
fieldResults) {
+ List<MappedField> fields =
Lists.newArrayListWithExpectedSize(fieldResults.size());
+
+ for (int i = 0; i < fieldResults.size(); i += 1) {
+ Types.NestedField field = struct.fields().get(i);
+ MappedFields result = fieldResults.get(i);
+ fields.add(MappedField.of(field.fieldId(), field.name(), result));
+ }
+
+ return MappedFields.of(fields);
+ }
+
+ @Override
+ public MappedFields field(Types.NestedField field, MappedFields
fieldResult) {
+ return fieldResult;
+ }
+
+ @Override
+ public MappedFields list(Types.ListType list, MappedFields elementResult) {
+ return MappedFields.of(MappedField.of(list.elementId(), "element",
elementResult));
+ }
+
+ @Override
+ public MappedFields map(Types.MapType map, MappedFields keyResult,
MappedFields valueResult) {
+ return MappedFields.of(
+ MappedField.of(map.keyId(), "key", keyResult),
+ MappedField.of(map.valueId(), "value", valueResult)
+ );
+ }
+
+ @Override
+ public MappedFields primitive(Type.PrimitiveType primitive) {
+ return null; // no mapping because primitives have no nested fields
+ }
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/mapping/NameMapping.java
b/core/src/main/java/org/apache/iceberg/mapping/NameMapping.java
new file mode 100644
index 0000000..2d8d4d8
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/mapping/NameMapping.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mapping;
+
+import com.google.common.base.Joiner;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represents a mapping from external schema names to Iceberg type IDs.
+ */
+public class NameMapping {
+ private static final Joiner DOT = Joiner.on('.');
+
+ private final MappedFields mapping;
+ private final Map<Integer, MappedField> fieldsById;
+ private final Map<String, MappedField> fieldsByName;
+
+ NameMapping(MappedFields mapping) {
+ this.mapping = mapping;
+ this.fieldsById = MappingUtil.indexById(mapping);
+ this.fieldsByName = MappingUtil.indexByName(mapping);
+ }
+
+ public MappedField find(int id) {
+ return fieldsById.get(id);
+ }
+
+ public MappedField find(String... names) {
+ return fieldsByName.get(DOT.join(names));
+ }
+
+ public MappedField find(List<String> names) {
+ return fieldsByName.get(DOT.join(names));
+ }
+
+ public MappedFields asMappedFields() {
+ return mapping;
+ }
+
+ @Override
+ public String toString() {
+ if (mapping.fields().isEmpty()) {
+ return "[]";
+ } else {
+ return "[\n " + Joiner.on("\n ").join(mapping.fields()) + "\n]";
+ }
+ }
+}
diff --git
a/core/src/main/java/org/apache/iceberg/mapping/NameMappingParser.java
b/core/src/main/java/org/apache/iceberg/mapping/NameMappingParser.java
new file mode 100644
index 0000000..453165c
--- /dev/null
+++ b/core/src/main/java/org/apache/iceberg/mapping/NameMappingParser.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mapping;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.exceptions.RuntimeIOException;
+import org.apache.iceberg.util.JsonUtil;
+
+/**
+ * Parses external name mappings from a JSON representation.
+ * <pre>
+ * [ { "field-id": 1, "names": ["id", "record_id"] },
+ * { "field-id": 2, "names": ["data"] },
+ * { "field-id": 3, "names": ["location"], "fields": [
+ * { "field-id": 4, "names": ["latitude", "lat"] },
+ * { "field-id": 5, "names": ["longitude", "long"] }
+ * ] } ]
+ * </pre>
+ */
+public class NameMappingParser {
+
+ private NameMappingParser() {
+ }
+
+ private static final String FIELD_ID = "field-id";
+ private static final String NAMES = "names";
+ private static final String FIELDS = "fields";
+
+ public static String toJson(NameMapping mapping) {
+ try {
+ StringWriter writer = new StringWriter();
+ JsonGenerator generator = JsonUtil.factory().createGenerator(writer);
+ generator.useDefaultPrettyPrinter();
+ toJson(mapping, generator);
+ generator.flush();
+ return writer.toString();
+ } catch (IOException e) {
+ throw new RuntimeIOException(e, "Failed to write json for: %s", mapping);
+ }
+ }
+
+ static void toJson(NameMapping nameMapping, JsonGenerator generator) throws
IOException {
+ toJson(nameMapping.asMappedFields(), generator);
+ }
+
+ private static void toJson(MappedFields mapping, JsonGenerator generator)
throws IOException {
+ generator.writeStartArray();
+
+ for (MappedField field : mapping.fields()) {
+ toJson(field, generator);
+ }
+
+ generator.writeEndArray();
+ }
+
+ private static void toJson(MappedField field, JsonGenerator generator)
throws IOException {
+ generator.writeStartObject();
+
+ generator.writeNumberField(FIELD_ID, field.id());
+
+ generator.writeArrayFieldStart(NAMES);
+ for (String name : field.names()) {
+ generator.writeString(name);
+ }
+ generator.writeEndArray();
+
+ MappedFields nested = field.nestedMapping();
+ if (nested != null) {
+ generator.writeFieldName(FIELDS);
+ toJson(nested, generator);
+ }
+
+ generator.writeEndObject();
+ }
+
+ public static NameMapping fromJson(String json) {
+ try {
+ return fromJson(JsonUtil.mapper().readValue(json, JsonNode.class));
+ } catch (IOException e) {
+ throw new RuntimeIOException(e, "Failed to convert version from json:
%s", json);
+ }
+ }
+
+ static NameMapping fromJson(JsonNode node) {
+ return new NameMapping(fieldsFromJson(node));
+ }
+
+ private static MappedFields fieldsFromJson(JsonNode node) {
+ Preconditions.checkArgument(node.isArray(), "Cannot parse non-array
mapping fields: %s", node);
+
+ List<MappedField> fields = Lists.newArrayList();
+ node.elements().forEachRemaining(fieldNode ->
fields.add(fieldFromJson(fieldNode)));
+
+ return MappedFields.of(fields);
+ }
+
+ private static MappedField fieldFromJson(JsonNode node) {
+ Preconditions.checkArgument(node != null && !node.isNull() &&
node.isObject(),
+ "Cannot parse non-object mapping field: %s", node);
+
+ Integer id = JsonUtil.getIntOrNull(FIELD_ID, node);
+
+ Set<String> names;
+ if (node.has(NAMES)) {
+ names = ImmutableSet.copyOf(JsonUtil.getStringList(NAMES, node));
+ } else {
+ names = ImmutableSet.of();
+ }
+
+ MappedFields nested;
+ if (node.has(FIELDS)) {
+ nested = fieldsFromJson(node.get(FIELDS));
+ } else {
+ nested = null;
+ }
+
+ return MappedField.of(id, names, nested);
+ }
+}
diff --git a/core/src/main/java/org/apache/iceberg/util/JsonUtil.java
b/core/src/main/java/org/apache/iceberg/util/JsonUtil.java
index 09a5466..976a1cc 100644
--- a/core/src/main/java/org/apache/iceberg/util/JsonUtil.java
+++ b/core/src/main/java/org/apache/iceberg/util/JsonUtil.java
@@ -52,6 +52,16 @@ public class JsonUtil {
return pNode.asInt();
}
+ public static Integer getIntOrNull(String property, JsonNode node) {
+ if (!node.has(property)) {
+ return null;
+ }
+ JsonNode pNode = node.get(property);
+ Preconditions.checkArgument(pNode != null && !pNode.isNull() &&
pNode.isIntegralNumber() && pNode.canConvertToInt(),
+ "Cannot parse %s from non-string value: %s", property, pNode);
+ return pNode.asInt();
+ }
+
public static long getLong(String property, JsonNode node) {
Preconditions.checkArgument(node.has(property), "Cannot parse missing int
%s", property);
JsonNode pNode = node.get(property);
diff --git a/core/src/test/java/org/apache/iceberg/TableTestBase.java
b/core/src/test/java/org/apache/iceberg/TableTestBase.java
index 6c4f682..c55c07d 100644
--- a/core/src/test/java/org/apache/iceberg/TableTestBase.java
+++ b/core/src/test/java/org/apache/iceberg/TableTestBase.java
@@ -40,7 +40,7 @@ import static
org.apache.iceberg.types.Types.NestedField.required;
public class TableTestBase {
// Schema passed to create tables
- static final Schema SCHEMA = new Schema(
+ public static final Schema SCHEMA = new Schema(
required(3, "id", Types.IntegerType.get()),
required(4, "data", Types.StringType.get())
);
@@ -80,7 +80,7 @@ public class TableTestBase {
File tableDir = null;
File metadataDir = null;
- TestTables.TestTable table = null;
+ public TestTables.TestTable table = null;
@Before
public void setupTable() throws Exception {
@@ -117,7 +117,7 @@ public class TableTestBase {
return TestTables.metadataVersion("test");
}
- TableMetadata readMetadata() {
+ public TableMetadata readMetadata() {
return TestTables.readMetadata("test");
}
diff --git a/core/src/test/java/org/apache/iceberg/TestTables.java
b/core/src/test/java/org/apache/iceberg/TestTables.java
index 4275205..13e336e 100644
--- a/core/src/test/java/org/apache/iceberg/TestTables.java
+++ b/core/src/test/java/org/apache/iceberg/TestTables.java
@@ -38,7 +38,7 @@ public class TestTables {
private TestTables() {}
- static TestTable create(File temp, String name, Schema schema, PartitionSpec
spec) {
+ public static TestTable create(File temp, String name, Schema schema,
PartitionSpec spec) {
TestTableOperations ops = new TestTableOperations(name, temp);
if (ops.current() != null) {
throw new AlreadyExistsException("Table %s already exists at location:
%s", name, temp);
@@ -47,7 +47,7 @@ public class TestTables {
return new TestTable(ops, name);
}
- static Transaction beginCreate(File temp, String name, Schema schema,
PartitionSpec spec) {
+ public static Transaction beginCreate(File temp, String name, Schema schema,
PartitionSpec spec) {
TableOperations ops = new TestTableOperations(name, temp);
if (ops.current() != null) {
throw new AlreadyExistsException("Table %s already exists at location:
%s", name, temp);
@@ -77,12 +77,12 @@ public class TestTables {
}
}
- static TestTable load(File temp, String name) {
+ public static TestTable load(File temp, String name) {
TestTableOperations ops = new TestTableOperations(name, temp);
return new TestTable(ops, name);
}
- static class TestTable extends BaseTable {
+ public static class TestTable extends BaseTable {
private final TestTableOperations ops;
private TestTable(TestTableOperations ops, String name) {
diff --git
a/core/src/test/java/org/apache/iceberg/mapping/TestMappingUpdates.java
b/core/src/test/java/org/apache/iceberg/mapping/TestMappingUpdates.java
new file mode 100644
index 0000000..bd39cb0
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/mapping/TestMappingUpdates.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mapping;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.TableTestBase;
+import org.apache.iceberg.types.Types;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+
+public class TestMappingUpdates extends TableTestBase {
+ @Test
+ public void testAddColumnMappingUpdate() {
+ NameMapping mapping = MappingUtil.create(table.schema());
+ table.updateProperties()
+ .set(TableProperties.DEFAULT_NAME_MAPPING,
NameMappingParser.toJson(mapping))
+ .commit();
+
+ Assert.assertEquals(
+ MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data")),
+ mapping.asMappedFields());
+
+ table.updateSchema()
+ .addColumn("ts", Types.TimestampType.withZone())
+ .commit();
+
+ NameMapping updated =
NameMappingParser.fromJson(table.properties().get(TableProperties.DEFAULT_NAME_MAPPING));
+
+ Assert.assertEquals(
+ MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"),
+ MappedField.of(3, "ts")),
+ updated.asMappedFields());
+ }
+
+ @Test
+ public void testAddNestedColumnMappingUpdate() {
+ NameMapping mapping = MappingUtil.create(table.schema());
+ table.updateProperties()
+ .set(TableProperties.DEFAULT_NAME_MAPPING,
NameMappingParser.toJson(mapping))
+ .commit();
+
+ Assert.assertEquals(
+ MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data")),
+ mapping.asMappedFields());
+
+ table.updateSchema()
+ .addColumn("point", Types.StructType.of(
+ required(1, "x", Types.DoubleType.get()),
+ required(2, "y", Types.DoubleType.get())))
+ .commit();
+
+ NameMapping updated =
NameMappingParser.fromJson(table.properties().get(TableProperties.DEFAULT_NAME_MAPPING));
+
+ Assert.assertEquals(
+ MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"),
+ MappedField.of(3, "point", MappedFields.of(
+ MappedField.of(4, "x"),
+ MappedField.of(5, "y")
+ ))),
+ updated.asMappedFields());
+
+ table.updateSchema()
+ .addColumn("point", "z", Types.DoubleType.get())
+ .commit();
+
+ NameMapping pointUpdated =
NameMappingParser.fromJson(table.properties().get(TableProperties.DEFAULT_NAME_MAPPING));
+
+ Assert.assertEquals(
+ MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"),
+ MappedField.of(3, "point", MappedFields.of(
+ MappedField.of(4, "x"),
+ MappedField.of(5, "y"),
+ MappedField.of(6, "z")
+ ))),
+ pointUpdated.asMappedFields());
+ }
+
+ @Test
+ public void testRenameMappingUpdate() {
+ NameMapping mapping = MappingUtil.create(table.schema());
+ table.updateProperties()
+ .set(TableProperties.DEFAULT_NAME_MAPPING,
NameMappingParser.toJson(mapping))
+ .commit();
+
+ Assert.assertEquals(
+ MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data")),
+ mapping.asMappedFields());
+
+ table.updateSchema()
+ .renameColumn("id", "object_id")
+ .commit();
+
+ NameMapping updated =
NameMappingParser.fromJson(table.properties().get(TableProperties.DEFAULT_NAME_MAPPING));
+
+ Assert.assertEquals(
+ MappedFields.of(
+ MappedField.of(1, ImmutableList.of("id", "object_id")),
+ MappedField.of(2, "data")),
+ updated.asMappedFields());
+ }
+
+ @Test
+ public void testRenameNestedFieldMappingUpdate() {
+ NameMapping mapping = MappingUtil.create(table.schema());
+ table.updateProperties()
+ .set(TableProperties.DEFAULT_NAME_MAPPING,
NameMappingParser.toJson(mapping))
+ .commit();
+
+ table.updateSchema()
+ .addColumn("point", Types.StructType.of(
+ required(1, "x", Types.DoubleType.get()),
+ required(2, "y", Types.DoubleType.get())))
+ .commit();
+
+ NameMapping updated =
NameMappingParser.fromJson(table.properties().get(TableProperties.DEFAULT_NAME_MAPPING));
+
+ Assert.assertEquals(
+ MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"),
+ MappedField.of(3, "point", MappedFields.of(
+ MappedField.of(4, "x"),
+ MappedField.of(5, "y")
+ ))),
+ updated.asMappedFields());
+
+ table.updateSchema()
+ .renameColumn("point.x", "X")
+ .renameColumn("point.y", "Y")
+ .commit();
+
+ NameMapping pointUpdated =
NameMappingParser.fromJson(table.properties().get(TableProperties.DEFAULT_NAME_MAPPING));
+
+ Assert.assertEquals(
+ MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"),
+ MappedField.of(3, "point", MappedFields.of(
+ MappedField.of(4, ImmutableList.of("x", "X")),
+ MappedField.of(5, ImmutableList.of("y", "Y"))
+ ))),
+ pointUpdated.asMappedFields());
+ }
+
+
+ @Test
+ public void testRenameComplexFieldMappingUpdate() {
+ NameMapping mapping = MappingUtil.create(table.schema());
+ table.updateProperties()
+ .set(TableProperties.DEFAULT_NAME_MAPPING,
NameMappingParser.toJson(mapping))
+ .commit();
+
+ table.updateSchema()
+ .addColumn("point", Types.StructType.of(
+ required(1, "x", Types.DoubleType.get()),
+ required(2, "y", Types.DoubleType.get())))
+ .commit();
+
+ NameMapping updated =
NameMappingParser.fromJson(table.properties().get(TableProperties.DEFAULT_NAME_MAPPING));
+
+ Assert.assertEquals(
+ MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"),
+ MappedField.of(3, "point", MappedFields.of(
+ MappedField.of(4, "x"),
+ MappedField.of(5, "y")
+ ))),
+ updated.asMappedFields());
+
+ table.updateSchema()
+ .renameColumn("point", "p2")
+ .commit();
+
+ NameMapping pointUpdated =
NameMappingParser.fromJson(table.properties().get(TableProperties.DEFAULT_NAME_MAPPING));
+
+ Assert.assertEquals(
+ MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"),
+ MappedField.of(3, ImmutableList.of("point", "p2"), MappedFields.of(
+ MappedField.of(4, "x"),
+ MappedField.of(5, "y")
+ ))),
+ pointUpdated.asMappedFields());
+ }
+
+ @Test
+ public void testMappingUpdateFailureSkipsMappingUpdate() {
+ NameMapping mapping = MappingUtil.create(table.schema());
+ table.updateProperties()
+ .set(TableProperties.DEFAULT_NAME_MAPPING,
NameMappingParser.toJson(mapping))
+ .commit();
+
+ table.updateSchema()
+ .renameColumn("id", "object_id")
+ .commit();
+
+ String updatedJson =
table.properties().get(TableProperties.DEFAULT_NAME_MAPPING);
+ NameMapping updated = NameMappingParser.fromJson(updatedJson);
+
+ Assert.assertEquals(
+ MappedFields.of(
+ MappedField.of(1, ImmutableList.of("id", "object_id")),
+ MappedField.of(2, "data")),
+ updated.asMappedFields());
+
+ // rename data to id, which conflicts in the mapping above
+ // this update should succeed, even though the mapping update fails
+ table.updateSchema()
+ .renameColumn("data", "id")
+ .commit();
+
+ Assert.assertEquals("Mapping JSON should not change",
+ updatedJson,
table.properties().get(TableProperties.DEFAULT_NAME_MAPPING));
+ }
+}
diff --git a/core/src/test/java/org/apache/iceberg/mapping/TestNameMapping.java
b/core/src/test/java/org/apache/iceberg/mapping/TestNameMapping.java
new file mode 100644
index 0000000..7a49764
--- /dev/null
+++ b/core/src/test/java/org/apache/iceberg/mapping/TestNameMapping.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mapping;
+
+import org.apache.iceberg.AssertHelpers;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.types.Types;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.iceberg.types.Types.NestedField.required;
+
+public class TestNameMapping {
+ @Test
+ public void testFlatSchemaToMapping() {
+ Schema schema = new Schema(
+ required(1, "id", Types.LongType.get()),
+ required(2, "data", Types.StringType.get()));
+
+ MappedFields expected = MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"));
+
+ NameMapping mapping = MappingUtil.create(schema);
+ Assert.assertEquals(expected, mapping.asMappedFields());
+ }
+
+ @Test
+ public void testNestedStructSchemaToMapping() {
+ Schema schema = new Schema(
+ required(1, "id", Types.LongType.get()),
+ required(2, "data", Types.StringType.get()),
+ required(3, "location", Types.StructType.of(
+ required(4, "latitude", Types.FloatType.get()),
+ required(5, "longitude", Types.FloatType.get())
+ )));
+
+ MappedFields expected = MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"),
+ MappedField.of(3, "location", MappedFields.of(
+ MappedField.of(4, "latitude"),
+ MappedField.of(5, "longitude")
+ )));
+
+ NameMapping mapping = MappingUtil.create(schema);
+ Assert.assertEquals(expected, mapping.asMappedFields());
+ }
+
+ @Test
+ public void testMapSchemaToMapping() {
+ Schema schema = new Schema(
+ required(1, "id", Types.LongType.get()),
+ required(2, "data", Types.StringType.get()),
+ required(3, "map", Types.MapType.ofRequired(4, 5,
+ Types.StringType.get(),
+ Types.DoubleType.get())));
+
+ MappedFields expected = MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"),
+ MappedField.of(3, "map", MappedFields.of(
+ MappedField.of(4, "key"),
+ MappedField.of(5, "value")
+ )));
+
+ NameMapping mapping = MappingUtil.create(schema);
+ Assert.assertEquals(expected, mapping.asMappedFields());
+ }
+
+ @Test
+ public void testComplexKeyMapSchemaToMapping() {
+ Schema schema = new Schema(
+ required(1, "id", Types.LongType.get()),
+ required(2, "data", Types.StringType.get()),
+ required(3, "map", Types.MapType.ofRequired(4, 5,
+ Types.StructType.of(
+ required(6, "x", Types.DoubleType.get()),
+ required(7, "y", Types.DoubleType.get())),
+ Types.DoubleType.get())));
+
+ MappedFields expected = MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"),
+ MappedField.of(3, "map", MappedFields.of(
+ MappedField.of(4, "key", MappedFields.of(
+ MappedField.of(6, "x"),
+ MappedField.of(7, "y")
+ )),
+ MappedField.of(5, "value")
+ )));
+
+ NameMapping mapping = MappingUtil.create(schema);
+ Assert.assertEquals(expected, mapping.asMappedFields());
+ }
+
+ @Test
+ public void testComplexValueMapSchemaToMapping() {
+ Schema schema = new Schema(
+ required(1, "id", Types.LongType.get()),
+ required(2, "data", Types.StringType.get()),
+ required(3, "map", Types.MapType.ofRequired(4, 5,
+ Types.DoubleType.get(),
+ Types.StructType.of(
+ required(6, "x", Types.DoubleType.get()),
+ required(7, "y", Types.DoubleType.get()))
+ )));
+
+ MappedFields expected = MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"),
+ MappedField.of(3, "map", MappedFields.of(
+ MappedField.of(4, "key"),
+ MappedField.of(5, "value", MappedFields.of(
+ MappedField.of(6, "x"),
+ MappedField.of(7, "y")
+ ))
+ )));
+
+ NameMapping mapping = MappingUtil.create(schema);
+ Assert.assertEquals(expected, mapping.asMappedFields());
+ }
+
+ @Test
+ public void testListSchemaToMapping() {
+ Schema schema = new Schema(
+ required(1, "id", Types.LongType.get()),
+ required(2, "data", Types.StringType.get()),
+ required(3, "list", Types.ListType.ofRequired(4,
Types.StringType.get())));
+
+ MappedFields expected = MappedFields.of(
+ MappedField.of(1, "id"),
+ MappedField.of(2, "data"),
+ MappedField.of(3, "list", MappedFields.of(
+ MappedField.of(4, "element")
+ )));
+
+ NameMapping mapping = MappingUtil.create(schema);
+ Assert.assertEquals(expected, mapping.asMappedFields());
+ }
+
+ @Test
+ public void testFailsDuplicateId() {
+ // the schema can be created because ID indexing is lazy
+ Schema schema = new Schema(
+ required(1, "id", Types.LongType.get()),
+ required(1, "data", Types.StringType.get()));
+
+ AssertHelpers.assertThrows("Should fail if IDs are reused",
+ IllegalArgumentException.class, "Multiple entries with same key",
+ () -> MappingUtil.create(schema));
+ }
+
+ @Test
+ public void testFailsDuplicateName() {
+ AssertHelpers.assertThrows("Should fail if names are reused",
+ IllegalArgumentException.class, "Multiple entries with same key",
+ () -> new NameMapping(MappedFields.of(MappedField.of(1, "x"),
MappedField.of(2, "x"))));
+ }
+
+ @Test
+ public void testAllowsDuplicateNamesInSeparateContexts() {
+ new NameMapping(MappedFields.of(
+ MappedField.of(1, "x", MappedFields.of(MappedField.of(3, "x"))),
+ MappedField.of(2, "y", MappedFields.of(MappedField.of(4, "x")))
+ ));
+ }
+
+ @Test
+ public void testMappingFindById() {
+ Schema schema = new Schema(
+ required(1, "id", Types.LongType.get()),
+ required(2, "data", Types.StringType.get()),
+ required(3, "map", Types.MapType.ofRequired(4, 5,
+ Types.DoubleType.get(),
+ Types.StructType.of(
+ required(6, "x", Types.DoubleType.get()),
+ required(7, "y", Types.DoubleType.get())))),
+ required(8, "list", Types.ListType.ofRequired(9,
+ Types.StringType.get())),
+ required(10, "location", Types.StructType.of(
+ required(11, "latitude", Types.FloatType.get()),
+ required(12, "longitude", Types.FloatType.get())
+ )));
+
+ NameMapping mapping = MappingUtil.create(schema);
+
+ Assert.assertNull("Should not return a field mapping for a missing ID",
mapping.find(100));
+ Assert.assertEquals(MappedField.of(2, "data"), mapping.find(2));
+ Assert.assertEquals(MappedField.of(6, "x"), mapping.find(6));
+ Assert.assertEquals(MappedField.of(9, "element"), mapping.find(9));
+ Assert.assertEquals(MappedField.of(11, "latitude"), mapping.find(11));
+ Assert.assertEquals(
+ MappedField.of(10, "location", MappedFields.of(
+ MappedField.of(11, "latitude"),
+ MappedField.of(12, "longitude"))),
+ mapping.find(10));
+ }
+
+ @Test
+ public void testMappingFindByName() {
+ Schema schema = new Schema(
+ required(1, "id", Types.LongType.get()),
+ required(2, "data", Types.StringType.get()),
+ required(3, "map", Types.MapType.ofRequired(4, 5,
+ Types.DoubleType.get(),
+ Types.StructType.of(
+ required(6, "x", Types.DoubleType.get()),
+ required(7, "y", Types.DoubleType.get())))),
+ required(8, "list", Types.ListType.ofRequired(9,
+ Types.StringType.get())),
+ required(10, "location", Types.StructType.of(
+ required(11, "latitude", Types.FloatType.get()),
+ required(12, "longitude", Types.FloatType.get())
+ )));
+
+ NameMapping mapping = MappingUtil.create(schema);
+
+ Assert.assertNull("Should not return a field mapping for a nested name",
mapping.find("element"));
+ Assert.assertNull("Should not return a field mapping for a nested name",
mapping.find("x"));
+ Assert.assertNull("Should not return a field mapping for a nested name",
mapping.find("key"));
+ Assert.assertNull("Should not return a field mapping for a nested name",
mapping.find("value"));
+ Assert.assertEquals(MappedField.of(2, "data"), mapping.find("data"));
+ Assert.assertEquals(MappedField.of(6, "x"), mapping.find("map", "value",
"x"));
+ Assert.assertEquals(MappedField.of(9, "element"), mapping.find("list",
"element"));
+ Assert.assertEquals(MappedField.of(11, "latitude"),
mapping.find("location", "latitude"));
+ Assert.assertEquals(
+ MappedField.of(10, "location", MappedFields.of(
+ MappedField.of(11, "latitude"),
+ MappedField.of(12, "longitude"))),
+ mapping.find("location"));
+ }
+}
