Repository: incubator-atlas Updated Branches: refs/heads/master 0e7f8ea46 -> 8101883cc
ATLAS-1734: Import API update to make type import optional Signed-off-by: Madhan Neethiraj <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/8101883c Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/8101883c Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/8101883c Branch: refs/heads/master Commit: 8101883cce0698a59f9f43fb67c94543585127d0 Parents: 0e7f8ea Author: ashutoshm <[email protected]> Authored: Mon Jun 26 22:22:12 2017 -0700 Committer: Madhan Neethiraj <[email protected]> Committed: Tue Jun 27 00:20:47 2017 -0700 ---------------------------------------------------------------------- .../atlas/model/impexp/AtlasImportRequest.java | 22 ++- .../atlas/repository/impexp/ExportService.java | 2 +- .../atlas/repository/impexp/ImportService.java | 42 +----- .../impexp/ImportTypeDefProcessor.java | 79 ++++++++++ .../impexp/TypeAttributeDifference.java | 135 +++++++++++++++++ .../impexp/AtlasImportRequestTest.java | 41 +++++ .../repository/impexp/ImportServiceTest.java | 64 +++++++- .../impexp/ImportServiceTestUtils.java | 116 ++++++++++++++ .../impexp/TypeAttributeDifferenceTest.java | 151 +++++++++++++++++++ .../atlas/repository/impexp/UniqueListTest.java | 69 +++++++++ .../impexp/ZipFileResourceTestUtils.java | 17 ++- .../test/resources/salesNewTypeAttrs-next.zip | Bin 0 -> 2947 bytes .../src/test/resources/salesNewTypeAttrs.zip | Bin 0 -> 2918 bytes 13 files changed, 677 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java ---------------------------------------------------------------------- diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java index b19f709..21b6427 100644 --- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java +++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java @@ -44,6 +44,7 @@ public class AtlasImportRequest implements Serializable { public static final String TRANSFORMS_KEY = "transforms"; private static final String START_POSITION_KEY = "startPosition"; private static final String START_GUID_KEY = "startGuid"; + private static final String UPDATE_TYPE_DEFINITION_KEY = "updateTypeDefinition"; private Map<String, String> options; @@ -76,19 +77,24 @@ public class AtlasImportRequest implements Serializable { @JsonIgnore public String getStartGuid() { - if (this.options == null || !this.options.containsKey(START_GUID_KEY)) { - return null; - } - - return (String) this.options.get(START_GUID_KEY); + return getOptionForKey(START_GUID_KEY); } @JsonIgnore public String getStartPosition() { - if (this.options == null || !this.options.containsKey(START_POSITION_KEY)) { + return getOptionForKey(START_POSITION_KEY); + } + + @JsonIgnore + public String getUpdateTypeDefs() { + return getOptionForKey(UPDATE_TYPE_DEFINITION_KEY); + } + + private String getOptionForKey(String key) { + if (this.options == null || !this.options.containsKey(key)) { return null; } - return (String) this.options.get(START_GUID_KEY); + return (String) this.options.get(key); } - } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java index 7744c78..3538cfd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java @@ -603,7 +603,7 @@ public class ExportService { } } - private class UniqueList<T> { + public static class UniqueList<T> { private final List<T> list = new ArrayList<>(); private final Set<T> set = new HashSet<>(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java index 4ffbb88..3411f8d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java @@ -21,12 +21,7 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.impexp.AtlasImportRequest; import org.apache.atlas.model.impexp.AtlasImportResult; -import org.apache.atlas.model.typedef.AtlasClassificationDef; -import org.apache.atlas.model.typedef.AtlasEntityDef; -import org.apache.atlas.model.typedef.AtlasEnumDef; -import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasTypeRegistry; @@ -51,7 +46,6 @@ public class ImportService { private long startTimestamp; private long endTimestamp; - public ImportService(final AtlasTypeDefStore typeDefStore, final AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) { this.typeDefStore = typeDefStore; this.entityStore = entityStore; @@ -139,40 +133,12 @@ public class ImportService { } private void processTypes(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException { - setGuidToEmpty(typeDefinitionMap); - - AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typeDefinitionMap, this.typeRegistry); - - if (!typesToCreate.isEmpty()) { - typeDefStore.createTypesDef(typesToCreate); - - updateMetricsForTypesDef(typesToCreate, result); - } - } - - private void updateMetricsForTypesDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) { - result.incrementMeticsCounter("typedef:classification", typeDefinitionMap.getClassificationDefs().size()); - result.incrementMeticsCounter("typedef:enum", typeDefinitionMap.getEnumDefs().size()); - result.incrementMeticsCounter("typedef:entitydef", typeDefinitionMap.getEntityDefs().size()); - result.incrementMeticsCounter("typedef:struct", typeDefinitionMap.getStructDefs().size()); - } - - private void setGuidToEmpty(AtlasTypesDef typesDef) { - for (AtlasEntityDef def : typesDef.getEntityDefs()) { - def.setGuid(null); - } - - for (AtlasClassificationDef def : typesDef.getClassificationDefs()) { - def.setGuid(null); + if(result.getRequest().getUpdateTypeDefs() != null && !result.getRequest().getUpdateTypeDefs().equals("true")) { + return; } - for (AtlasEnumDef def : typesDef.getEnumDefs()) { - def.setGuid(null); - } - - for (AtlasStructDef def : typesDef.getStructDefs()) { - def.setGuid(null); - } + ImportTypeDefProcessor importTypeDefProcessor = new ImportTypeDefProcessor(this.typeDefStore, this.typeRegistry); + importTypeDefProcessor.processTypes(typeDefinitionMap, result); } private void processEntities(ZipSource importSource, AtlasImportResult result) throws AtlasBaseException { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java new file mode 100644 index 0000000..ab66460 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportTypeDefProcessor.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.impexp; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.impexp.AtlasImportResult; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; +import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasTypeRegistry; + +public class ImportTypeDefProcessor { + private final AtlasTypeDefStore typeDefStore; + private final AtlasTypeRegistry typeRegistry; + private TypeAttributeDifference typeAttributeDifference; + + public ImportTypeDefProcessor(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) { + this.typeDefStore = typeDefStore; + this.typeRegistry = typeRegistry; + this.typeAttributeDifference = new TypeAttributeDifference(typeDefStore, typeRegistry); + } + + public void processTypes(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException { + setGuidToEmpty(typeDefinitionMap); + + typeAttributeDifference.updateTypes(typeDefinitionMap, result); + + AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typeDefinitionMap, this.typeRegistry); + if (!typesToCreate.isEmpty()) { + typeDefStore.createTypesDef(typesToCreate); + updateMetricsForTypesDef(typesToCreate, result); + + } + } + + private void setGuidToEmpty(AtlasTypesDef typesDef) { + for (AtlasEntityDef def : typesDef.getEntityDefs()) { + def.setGuid(null); + } + + for (AtlasClassificationDef def : typesDef.getClassificationDefs()) { + def.setGuid(null); + } + + for (AtlasEnumDef def : typesDef.getEnumDefs()) { + def.setGuid(null); + } + + for (AtlasStructDef def : typesDef.getStructDefs()) { + def.setGuid(null); + } + } + + private void updateMetricsForTypesDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) { + result.incrementMeticsCounter("typedef:classification", typeDefinitionMap.getClassificationDefs().size()); + result.incrementMeticsCounter("typedef:enum", typeDefinitionMap.getEnumDefs().size()); + result.incrementMeticsCounter("typedef:entitydef", typeDefinitionMap.getEntityDefs().size()); + result.incrementMeticsCounter("typedef:struct", typeDefinitionMap.getStructDefs().size()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java b/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java new file mode 100644 index 0000000..68bc583 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.impexp; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.impexp.AtlasImportResult; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasTypeRegistry; + +import java.util.ArrayList; +import java.util.List; + +public class TypeAttributeDifference { + private final AtlasTypeDefStore typeDefStore; + private final AtlasTypeRegistry typeRegistry; + + public TypeAttributeDifference(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) { + this.typeDefStore = typeDefStore; + this.typeRegistry = typeRegistry; + } + + public void updateTypes(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException { + + updateEntityDef(typeDefinitionMap, result); + updateClassificationDef(typeDefinitionMap, result); + updateEnumDef(typeDefinitionMap, result); + updateStructDef(typeDefinitionMap, result); + } + + private void updateEntityDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException { + for (AtlasEntityDef def: typeDefinitionMap.getEntityDefs()) { + AtlasEntityDef existing = typeRegistry.getEntityDefByName(def.getName()); + if(existing != null && addAttributes(existing, def)) { + typeDefStore.updateEntityDefByName(existing.getName(), existing); + result.incrementMeticsCounter("typedef:entitydef:update"); + } + } + } + + private void updateClassificationDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException { + for (AtlasClassificationDef def: typeDefinitionMap.getClassificationDefs()) { + AtlasClassificationDef existing = typeRegistry.getClassificationDefByName(def.getName()); + if(existing != null && addAttributes(existing, def)) { + typeDefStore.updateClassificationDefByName(existing.getName(), existing); + result.incrementMeticsCounter("typedef:classification:update"); + } + } + } + + private void updateEnumDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException { + for (AtlasEnumDef def: typeDefinitionMap.getEnumDefs()) { + AtlasEnumDef existing = typeRegistry.getEnumDefByName(def.getName()); + if(existing != null && addElements(existing, def)) { + typeDefStore.updateEnumDefByName(existing.getName(), existing); + result.incrementMeticsCounter("typedef:enum:update"); + } + } + } + + private void updateStructDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException { + for (AtlasStructDef def: typeDefinitionMap.getStructDefs()) { + AtlasStructDef existing = typeRegistry.getStructDefByName(def.getName()); + if(existing != null && addAttributes(existing, def)) { + typeDefStore.updateStructDefByName(existing.getName(), existing); + result.incrementMeticsCounter("typedef:struct:update"); + } + } + } + + private boolean addElements(AtlasEnumDef existing, AtlasEnumDef incoming) { + return addElements(existing, getElementsAbsentInExisting(existing, incoming)); + } + + private boolean addAttributes(AtlasStructDef existing, AtlasStructDef incoming) { + return addAttributes(existing, getElementsAbsentInExisting(existing, incoming)); + } + + private List<AtlasStructDef.AtlasAttributeDef> getElementsAbsentInExisting(AtlasStructDef existing, AtlasStructDef incoming) { + List<AtlasStructDef.AtlasAttributeDef> difference = new ArrayList<>(); + for (AtlasStructDef.AtlasAttributeDef attr : incoming.getAttributeDefs()) { + if(existing.getAttribute(attr.getName()) == null) { + difference.add(attr); + } + } + + return difference; + } + + private List<AtlasEnumDef.AtlasEnumElementDef> getElementsAbsentInExisting(AtlasEnumDef existing, AtlasEnumDef incoming) { + List<AtlasEnumDef.AtlasEnumElementDef> difference = new ArrayList<>(); + for (AtlasEnumDef.AtlasEnumElementDef ed : incoming.getElementDefs()) { + if(existing.getElement(ed.getValue()) == null) { + difference.add(ed); + } + } + + return difference; + } + + private boolean addAttributes(AtlasStructDef def, List<AtlasStructDef.AtlasAttributeDef> list) { + for (AtlasStructDef.AtlasAttributeDef ad : list) { + def.addAttribute(ad); + } + + return list.size() > 0; + } + + private boolean addElements(AtlasEnumDef def, List<AtlasEnumDef.AtlasEnumElementDef> list) { + for (AtlasEnumDef.AtlasEnumElementDef ad : list) { + def.addElement(ad); + } + + return list.size() > 0; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java index 0844bf0..b70b181 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java @@ -40,6 +40,47 @@ public class AtlasImportRequestTest { } @Test + public void serializeOptions_VerifyAccessors() { + String guid = "\"abcd\""; + String pos = "\"1\""; + String trueVal = "\"true\""; + + String jsonData = "{ \"options\": " + + " {" + + "\"startGuid\":" + guid + "," + + "\"startPosition\":" + pos + "," + + "\"updateTypeDefinition\":" + trueVal + + "}" + + "}"; + + AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class); + + assertNotNull(request); + assertNotNull(request.getStartGuid()); + assertNotNull(request.getStartPosition()); + assertNotNull(request.getUpdateTypeDefs()); + + assertEquals(request.getStartGuid(), guid.replace("\"", "")); + assertEquals(request.getStartPosition(), pos.replace("\"", "")); + assertEquals(request.getUpdateTypeDefs(), trueVal.replace("\"", "")); + } + + @Test + public void optionsDefaultsTest() { + String jsonData = "{ \"options\": " + + " {" + + "}" + + "}"; + + AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class); + + assertNotNull(request); + assertNull(request.getStartGuid()); + assertNull(request.getStartPosition()); + assertNull(request.getUpdateTypeDefs()); + } + + @Test public void serializeAtlasImportRequstFromJsonWithEmptyTransforms() { String jsonData = "{ \"options\": { \"transforms\": \"{ }\" } }"; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java index a6fed62..de8e7ef 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java @@ -20,6 +20,7 @@ package org.apache.atlas.repository.impexp; import com.google.inject.Inject; import org.apache.atlas.TestModules; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.impexp.AtlasImportRequest; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasTypeRegistry; @@ -31,10 +32,12 @@ import org.testng.annotations.Guice; import org.testng.annotations.Test; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.getZipSource; -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson; -import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runAndVerifyQuickStart_v1_Import; +import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.*; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; @Guice(modules = TestModules.TestOnlyModule.class) public class ImportServiceTest { @@ -55,7 +58,7 @@ public class ImportServiceTest { } @Test(dataProvider = "sales") - public void importDB1_Succeeds(ZipSource zipSource) throws AtlasBaseException, IOException { + public void importDB1(ZipSource zipSource) throws AtlasBaseException, IOException { loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry); runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource); } @@ -66,7 +69,7 @@ public class ImportServiceTest { } @Test(dataProvider = "reporting") - public void importDB2_Succeeds(ZipSource zipSource) throws AtlasBaseException, IOException { + public void importDB2(ZipSource zipSource) throws AtlasBaseException, IOException { loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry); runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource); } @@ -77,8 +80,57 @@ public class ImportServiceTest { } @Test(dataProvider = "logging") - public void importDB3_Succeeds(ZipSource zipSource) throws AtlasBaseException, IOException { + public void importDB3(ZipSource zipSource) throws AtlasBaseException, IOException { loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry); runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource); } + + @DataProvider(name = "salesNewTypeAttrs") + public static Object[][] getDataFromSalesNewTypeAttrs(ITestContext context) throws IOException { + return getZipSource("salesNewTypeAttrs.zip"); + } + + @Test(dataProvider = "salesNewTypeAttrs", dependsOnMethods = "importDB1") + public void importDB4(ZipSource zipSource) throws AtlasBaseException, IOException { + loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry); + runImportWithParameters(new ImportService(typeDefStore, entityStore, typeRegistry), getDefaultImportRequest(), zipSource); + } + + @DataProvider(name = "salesNewTypeAttrs-next") + public static Object[][] getDataFromSalesNewTypeAttrsNext(ITestContext context) throws IOException { + return getZipSource("salesNewTypeAttrs-next.zip"); + } + + + @Test(dataProvider = "salesNewTypeAttrs-next", dependsOnMethods = "importDB4") + public void importDB5(ZipSource zipSource) throws AtlasBaseException, IOException { + final String newEnumDefName = "database_action"; + + assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName)); + + AtlasImportRequest request = getDefaultImportRequest(); + Map<String, String> options = new HashMap<>(); + options.put("updateTypeDefinition", "false"); + request.setOptions(options); + + runImportWithParameters(new ImportService(typeDefStore, entityStore, typeRegistry), request, zipSource); + assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName)); + assertEquals(typeDefStore.getEnumDefByName(newEnumDefName).getElementDefs().size(), 4); + } + + @Test(dataProvider = "salesNewTypeAttrs-next", dependsOnMethods = "importDB4") + public void importDB6(ZipSource zipSource) throws AtlasBaseException, IOException { + final String newEnumDefName = "database_action"; + + assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName)); + + AtlasImportRequest request = getDefaultImportRequest(); + Map<String, String> options = new HashMap<>(); + options.put("updateTypeDefinition", "true"); + request.setOptions(options); + + runImportWithParameters(new ImportService(typeDefStore, entityStore, typeRegistry), request, zipSource); + assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName)); + assertEquals(typeDefStore.getEnumDefByName(newEnumDefName).getElementDefs().size(), 8); + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTestUtils.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTestUtils.java new file mode 100644 index 0000000..7289512 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTestUtils.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.impexp; + +import com.google.common.collect.Sets; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.impexp.AtlasExportResult; +import org.apache.atlas.model.impexp.AtlasImportRequest; +import org.apache.atlas.model.impexp.AtlasImportResult; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; +import org.apache.atlas.store.AtlasTypeDefStore; +import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.testng.Assert; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ImportServiceTestUtils { + + public static void verifyImportedEntities(List<String> creationOrder, List<String> processedEntities) { + Set<String> lhs = com.google.common.collect.Sets.newHashSet(creationOrder); + Set<String> rhs = com.google.common.collect.Sets.newHashSet(processedEntities); + Set<String> difference = Sets.difference(lhs, rhs); + + Assert.assertNotNull(difference); + Assert.assertEquals(difference.size(), 0); + } + + public static void verifyImportedMetrics(AtlasExportResult exportResult, AtlasImportResult importResult) { + Map<String, Integer> metricsForCompare = getImportMetricsForCompare(importResult); + for (Map.Entry<String, Integer> entry : exportResult.getMetrics().entrySet()) { + if(entry.getKey().startsWith("entity") == false || + entry.getKey().contains("withExtInfo") || + entry.getKey().contains("Column") || + entry.getKey().contains("StorageDesc")) continue; + + Assert.assertTrue(metricsForCompare.containsKey(entry.getKey())); + Assert.assertEquals(entry.getValue(), metricsForCompare.get(entry.getKey())); + } + } + + private static Map<String,Integer> getImportMetricsForCompare(AtlasImportResult result) { + Map<String, Integer> r = new HashMap<>(); + for (Map.Entry<String, Integer> entry : result.getMetrics().entrySet()) { + r.put(entry.getKey().replace(":updated", "").replace(":created", ""), entry.getValue()); + } + + return r; + } + + + public static void loadModelFromJson(String fileName, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException { + AtlasTypesDef typesFromJson = getAtlasTypesDefFromFile(fileName); + createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry); + } + + private static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesFromJson, typeRegistry); + + if (!typesToCreate.isEmpty()) { + typeDefStore.createTypesDef(typesToCreate); + } + } + + private static AtlasTypesDef getAtlasTypesDefFromFile(String fileName) throws IOException { + String sampleTypes = ZipFileResourceTestUtils.getModelJson(fileName); + return AtlasType.fromJson(sampleTypes, AtlasTypesDef.class); + } + + public static AtlasImportRequest getDefaultImportRequest() { + return new AtlasImportRequest(); + } + + + public static AtlasImportResult runImportWithParameters(ImportService importService, AtlasImportRequest request, ZipSource source) throws AtlasBaseException, IOException { + final String requestingIP = "1.0.0.0"; + final String hostName = "localhost"; + final String userName = "admin"; + + AtlasImportResult result = importService.run(source, request, userName, hostName, requestingIP); + Assert.assertEquals(result.getOperationStatus(), AtlasImportResult.OperationStatus.SUCCESS); + return result; + } + + public static void runAndVerifyQuickStart_v1_Import(ImportService importService, ZipSource zipSource) throws AtlasBaseException, IOException { + AtlasExportResult exportResult = zipSource.getExportResult(); + List<String> creationOrder = zipSource.getCreationOrder(); + + AtlasImportRequest request = getDefaultImportRequest(); + AtlasImportResult result = runImportWithParameters(importService, request, zipSource); + + Assert.assertNotNull(result); + verifyImportedMetrics(exportResult, result); + verifyImportedEntities(creationOrder, result.getProcessedEntities()); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java new file mode 100644 index 0000000..cf17ef2 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.impexp; + +import org.apache.atlas.model.typedef.AtlasBaseTypeDef; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasStructDef; +import org.powermock.reflect.Whitebox; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.List; + +public class TypeAttributeDifferenceTest { + private TypeAttributeDifference typeAttributeDifference; + + @BeforeClass + public void setup() { + typeAttributeDifference = new TypeAttributeDifference(null, null); + } + + private List<AtlasEnumDef.AtlasEnumElementDef> getEnumElementDefs(int startIndex, String... names) { + int i = startIndex; + List<AtlasEnumDef.AtlasEnumElementDef> list = new ArrayList<>(); + for (String s: names) { + list.add(new AtlasEnumDef.AtlasEnumElementDef(s, s, i++)); + } + + return list; + } + + private List<AtlasStructDef.AtlasAttributeDef> getAtlasAttributeDefs(String... names) { + List<AtlasStructDef.AtlasAttributeDef> list = new ArrayList<>(); + for (String s : names) { + list.add(new AtlasStructDef.AtlasAttributeDef(s, AtlasBaseTypeDef.ATLAS_TYPE_STRING)); + } + + return list; + } + + private AtlasEntityDef getAtlasEntityDefWithAttributes(String... attributeNames) { + AtlasEntityDef e = new AtlasEntityDef(); + for (AtlasStructDef.AtlasAttributeDef a : getAtlasAttributeDefs(attributeNames)) { + e.addAttribute(a); + } + + return e; + } + + @Test + public void entityDefWithNoAttributes() throws Exception { + AtlasEntityDef existing = new AtlasEntityDef(); + AtlasEntityDef incoming = new AtlasEntityDef(); + List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = new ArrayList<>(); + List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming); + + Assert.assertEquals(actualAttributes, expectedAttributes); + } + + private List<AtlasStructDef.AtlasAttributeDef> invokeGetAttributesAbsentInExisting(AtlasStructDef existing, AtlasStructDef incoming) throws Exception { + return Whitebox.invokeMethod(typeAttributeDifference, "getElementsAbsentInExisting", existing, incoming); + } + + private List<AtlasEnumDef.AtlasEnumElementDef> invokeGetAttributesAbsentInExisting(AtlasEnumDef existing, AtlasEnumDef incoming) throws Exception { + return Whitebox.invokeMethod(typeAttributeDifference, "getElementsAbsentInExisting", existing, incoming); + } + + private AtlasEnumDef getAtlasEnumWithAttributes(String... elements) { + AtlasEnumDef enumDef = new AtlasEnumDef(); + for (AtlasEnumDef.AtlasEnumElementDef ed : getEnumElementDefs(0, elements)) { + enumDef.addElement(ed); + } + + return enumDef; + } + + @Test + public void bothSame_DifferenceIsEmptyList() throws Exception { + AtlasEntityDef existing = getAtlasEntityDefWithAttributes("name", "qualifiedName"); + AtlasEntityDef incoming = getAtlasEntityDefWithAttributes("name", "qualifiedName"); + + List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = getAtlasAttributeDefs(); + List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming); + + Assert.assertEquals(actualAttributes, expectedAttributes); + } + + @Test + public void different_ReturnsDifference() throws Exception { + AtlasEntityDef existing = getAtlasEntityDefWithAttributes("name"); + AtlasEntityDef incoming = getAtlasEntityDefWithAttributes("name", "qualifiedName"); + List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = getAtlasAttributeDefs( "qualifiedName"); + + List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming); + Assert.assertEquals(actualAttributes, expectedAttributes); + } + + @Test + public void differentSubset_ReturnsDifference() throws Exception { + AtlasEntityDef existing = getAtlasEntityDefWithAttributes("name", "qualifiedName"); + AtlasEntityDef incoming = getAtlasEntityDefWithAttributes("name"); + List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming); + + List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = getAtlasAttributeDefs(); + Assert.assertEquals(actualAttributes, expectedAttributes); + } + + @Test + public void differentEnumDef_ReturnsDifference () throws Exception { + AtlasEnumDef existing = getAtlasEnumWithAttributes("Alpha", "Bravo"); + AtlasEnumDef incoming = getAtlasEnumWithAttributes("Alpha", "Bravo", "Delta", "Echo"); + List<AtlasEnumDef.AtlasEnumElementDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming); + + List<AtlasEnumDef.AtlasEnumElementDef> expectedAttributes = getEnumElementDefs(2, "Delta", "Echo"); + Assert.assertEquals(actualAttributes, expectedAttributes); + } + + @Test + public void differentEnumDefs_ReturnsDifference () throws Exception { + AtlasEnumDef existing = getAtlasEnumWithAttributes("Alpha", "Bravo"); + AtlasEnumDef incoming = getAtlasEnumWithAttributes("Alpha", "Bravo", "Delta", "Echo"); + boolean ret = invokeUpdate(existing, incoming); + + List<AtlasEnumDef.AtlasEnumElementDef> expectedAttributes = getEnumElementDefs(0, "Alpha", "Bravo", "Delta", "Echo"); + + Assert.assertTrue(ret, "Update took place"); + Assert.assertEquals(existing.getElementDefs(), expectedAttributes); + } + + private boolean invokeUpdate(AtlasEnumDef existing, AtlasEnumDef incoming) throws Exception { + return Whitebox.invokeMethod(typeAttributeDifference, "addElements", existing, incoming); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/repository/src/test/java/org/apache/atlas/repository/impexp/UniqueListTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/UniqueListTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/UniqueListTest.java new file mode 100644 index 0000000..93aa518 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/UniqueListTest.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.impexp; + +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + +public class UniqueListTest { + private final String firstElement = "firstElement"; + private ExportService.UniqueList<String> uniqueList; + + @BeforeClass + public void setup() { + uniqueList = new ExportService.UniqueList(); + uniqueList.add(firstElement); + uniqueList.add("def"); + uniqueList.add("firstElement"); + uniqueList.add("ghi"); + } + + @Test + public void add3Elements_ListHas2() { + assertEquals(3, uniqueList.size()); + } + + @Test + public void addAllList_ListHas2() { + ExportService.UniqueList<String> uniqueList2 = new ExportService.UniqueList<>(); + uniqueList2.addAll(uniqueList); + + assertEquals(3, uniqueList2.size()); + } + + @Test + public void attemptClear_SizeIsZero() { + ExportService.UniqueList<String> uniqueList2 = new ExportService.UniqueList<>(); + uniqueList2.addAll(uniqueList); + uniqueList2.clear(); + + assertEquals(0, uniqueList2.size()); + } + + @Test + public void attemptOneRemove_SizeIsReduced() { + ExportService.UniqueList<String> uniqueList2 = new ExportService.UniqueList<>(); + uniqueList2.addAll(uniqueList); + String removedElement = uniqueList2.remove(0); + + assertEquals(2, uniqueList2.size()); + assertEquals(firstElement, removedElement); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java index 612c445..d901731 100644 --- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java +++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java @@ -31,7 +31,6 @@ import org.apache.commons.io.FileUtils; import org.apache.solr.common.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testng.Assert; import java.io.File; import java.io.FileInputStream; @@ -42,6 +41,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import static org.testng.Assert.*; + public class ZipFileResourceTestUtils { public static final Logger LOG = LoggerFactory.getLogger(ZipFileResourceTestUtils.class); @@ -67,7 +68,7 @@ public class ZipFileResourceTestUtils { String filePath = userDir + "/../addons/models/" + fileName; File f = new File(filePath); String s = FileUtils.readFileToString(f); - Assert.assertFalse(StringUtils.isEmpty(s), "Model file read correctly!"); + assertFalse(StringUtils.isEmpty(s), "Model file read correctly!"); return s; } @@ -84,8 +85,8 @@ public class ZipFileResourceTestUtils { Set<String> rhs = com.google.common.collect.Sets.newHashSet(processedEntities); Set<String> difference = Sets.difference(lhs, rhs); - Assert.assertNotNull(difference); - Assert.assertEquals(difference.size(), 0); + assertNotNull(difference); + assertEquals(difference.size(), 0); } public static void verifyImportedMetrics(AtlasExportResult exportResult, AtlasImportResult importResult) { @@ -96,8 +97,8 @@ public class ZipFileResourceTestUtils { entry.getKey().contains("Column") || entry.getKey().contains("StorageDesc")) continue; - Assert.assertTrue(metricsForCompare.containsKey(entry.getKey())); - Assert.assertEquals(entry.getValue(), metricsForCompare.get(entry.getKey())); + assertTrue(metricsForCompare.containsKey(entry.getKey()), entry.getKey()); + assertEquals(entry.getValue(), metricsForCompare.get(entry.getKey()), entry.getKey()); } } @@ -140,7 +141,7 @@ public class ZipFileResourceTestUtils { final String userName = "admin"; AtlasImportResult result = importService.run(source, request, userName, hostName, requestingIP); - Assert.assertEquals(result.getOperationStatus(), AtlasImportResult.OperationStatus.SUCCESS); + assertEquals(result.getOperationStatus(), AtlasImportResult.OperationStatus.SUCCESS); return result; } @@ -151,7 +152,7 @@ public class ZipFileResourceTestUtils { AtlasImportRequest request = getDefaultImportRequest(); AtlasImportResult result = runImportWithParameters(importService, request, zipSource); - Assert.assertNotNull(result); + assertNotNull(result); verifyImportedMetrics(exportResult, result); verifyImportedEntities(creationOrder, result.getProcessedEntities()); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/repository/src/test/resources/salesNewTypeAttrs-next.zip ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/salesNewTypeAttrs-next.zip b/repository/src/test/resources/salesNewTypeAttrs-next.zip new file mode 100644 index 0000000..e4c8505 Binary files /dev/null and b/repository/src/test/resources/salesNewTypeAttrs-next.zip differ http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/8101883c/repository/src/test/resources/salesNewTypeAttrs.zip ---------------------------------------------------------------------- diff --git a/repository/src/test/resources/salesNewTypeAttrs.zip b/repository/src/test/resources/salesNewTypeAttrs.zip new file mode 100644 index 0000000..a4467e6 Binary files /dev/null and b/repository/src/test/resources/salesNewTypeAttrs.zip differ
