Repository: incubator-atlas Updated Branches: refs/heads/master 161079155 -> 98fdc6d85
ATLAS-512 Decouple currently integrating components from availability of Atlas service for raising metadata events ( yhemanth via sumasai) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/98fdc6d8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/98fdc6d8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/98fdc6d8 Branch: refs/heads/master Commit: 98fdc6d85d8e3da7b2ac434d4a4d1d102a017a53 Parents: 1610791 Author: Suma Shivaprasad <[email protected]> Authored: Thu Mar 10 12:23:48 2016 -0800 Committer: Suma Shivaprasad <[email protected]> Committed: Thu Mar 10 12:23:48 2016 -0800 ---------------------------------------------------------------------- addons/falcon-bridge/pom.xml | 21 ++++ .../apache/atlas/falcon/hook/FalconHook.java | 5 - .../falcon/model/FalconDataModelGenerator.java | 10 +- .../apache/atlas/falcon/hook/FalconHookIT.java | 1 + addons/hive-bridge/pom.xml | 21 ++++ .../org/apache/atlas/hive/hook/HiveHook.java | 5 - .../hive/model/HiveDataModelGenerator.java | 16 ++- .../org/apache/atlas/hive/hook/HiveHookIT.java | 2 + addons/sqoop-bridge/pom.xml | 21 ++++ .../org/apache/atlas/sqoop/hook/SqoopHook.java | 3 +- .../sqoop/model/SqoopDataModelGenerator.java | 10 +- .../apache/atlas/sqoop/hook/SqoopHookIT.java | 5 +- addons/storm-bridge/pom.xml | 22 ++++ .../apache/atlas/storm/hook/StormAtlasHook.java | 4 - .../storm/model/StormDataModelGenerator.java | 41 ++++++++ .../atlas/storm/hook/StormAtlasHookIT.java | 1 + .../atlas/addons/ModelDefinitionDump.java | 51 ++++++++++ .../src/main/assemblies/standalone-package.xml | 20 ++++ release-log.txt | 1 + .../apache/atlas/RepositoryMetadataModule.java | 4 + .../atlas/services/DefaultMetadataService.java | 14 ++- .../services/IBootstrapTypesRegistrar.java | 27 +++++ .../atlas/services/ReservedTypesRegistrar.java | 81 +++++++++++++++ .../ReservedTypesRegistrationException.java | 26 +++++ .../DefaultMetadataServiceMockTest.java | 53 ++++++++++ .../services/ReservedTypesRegistrarTest.java | 101 +++++++++++++++++++ 26 files changed, 544 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/falcon-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/pom.xml b/addons/falcon-bridge/pom.xml index 73ef265..c5fd53a 100644 --- a/addons/falcon-bridge/pom.xml +++ b/addons/falcon-bridge/pom.xml @@ -269,6 +269,27 @@ <generateReports>false</generateReports> </configuration> </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.2.1</version> + <inherited>false</inherited> + <executions> + <execution> + <configuration> + <mainClass>org.apache.atlas.falcon.model.FalconDataModelGenerator</mainClass> + <arguments> + <argument>${project.build.directory}/models/falcon_model.json</argument> + </arguments> + </configuration> + <phase>package</phase> + <goals> + <goal>java</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/hook/FalconHook.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/hook/FalconHook.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/hook/FalconHook.java index 47fa714..d4b0069 100644 --- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/hook/FalconHook.java +++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/hook/FalconHook.java @@ -166,11 +166,6 @@ public class FalconHook extends FalconEventPublisher { private void fireAndForget(FalconEvent event) throws Exception { LOG.info("Entered Atlas hook for Falcon hook operation {}", event.getOperation()); - if (!typesRegistered) { - registerFalconDataModel(); - typesRegistered = true; - } - notifyEntity(createEntities(event)); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java index a68db2f..0dbb5dc 100644 --- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java +++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java @@ -21,6 +21,7 @@ package org.apache.atlas.falcon.model; import com.google.common.collect.ImmutableList; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; +import org.apache.atlas.addons.ModelDefinitionDump; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.typesystem.types.AttributeDefinition; @@ -130,7 +131,14 @@ public class FalconDataModelGenerator { public static void main(String[] args) throws Exception { FalconDataModelGenerator falconDataModelGenerator = new FalconDataModelGenerator(); - System.out.println("falconDataModelAsJSON = " + falconDataModelGenerator.getModelAsJson()); + String modelAsJson = falconDataModelGenerator.getModelAsJson(); + + if (args.length == 1) { + ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson); + return; + } + + System.out.println("falconDataModelAsJSON = " + modelAsJson); TypesDef typesDef = falconDataModelGenerator.getTypesDef(); for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java index 3881bd6..aaffa4a 100644 --- a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java +++ b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java @@ -65,6 +65,7 @@ public class FalconHookIT { AtlasService service = new AtlasService(); service.init(); STORE.registerListener(service); + new FalconHook().registerFalconDataModel(); CurrentUser.authenticate(System.getProperty("user.name")); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/hive-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml index f1cb130..23999fd 100755 --- a/addons/hive-bridge/pom.xml +++ b/addons/hive-bridge/pom.xml @@ -347,6 +347,27 @@ <generateReports>false</generateReports> </configuration> </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.2.1</version> + <inherited>false</inherited> + <executions> + <execution> + <configuration> + <mainClass>org.apache.atlas.hive.model.HiveDataModelGenerator</mainClass> + <arguments> + <argument>${project.build.directory}/models/hive_model.json</argument> + </arguments> + </configuration> + <phase>package</phase> + <goals> + <goal>java</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java index adc28ba..6a1e9bd 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java @@ -201,11 +201,6 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { HiveMetaStoreBridge dgiBridge = new HiveMetaStoreBridge(hiveConf, atlasProperties, event.user, event.ugi); - if (!typesRegistered) { - dgiBridge.registerHiveDataModel(); - typesRegistered = true; - } - switch (event.operation) { case CREATEDATABASE: handleEventOutputs(dgiBridge, event, Type.DATABASE); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java index 0fa92ef..7256ae5 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java @@ -21,6 +21,7 @@ package org.apache.atlas.hive.model; import com.google.common.collect.ImmutableList; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; +import org.apache.atlas.addons.ModelDefinitionDump; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.typesystem.types.AttributeDefinition; @@ -39,6 +40,11 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -370,7 +376,14 @@ public class HiveDataModelGenerator { public static void main(String[] args) throws Exception { HiveDataModelGenerator hiveDataModelGenerator = new HiveDataModelGenerator(); - System.out.println("hiveDataModelAsJSON = " + hiveDataModelGenerator.getModelAsJson()); + String modelAsJson = hiveDataModelGenerator.getModelAsJson(); + + if (args.length==1) { + ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson); + return; + } + + System.out.println("hiveDataModelAsJSON = " + modelAsJson); TypesDef typesDef = hiveDataModelGenerator.getTypesDef(); for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) { @@ -390,4 +403,5 @@ public class HiveDataModelGenerator { Arrays.toString(traitType.attributeDefinitions))); } } + } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index 1611bc2..c924bf8 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -68,6 +68,8 @@ public class HiveHookIT { SessionState.setCurrentSessionState(ss); Configuration configuration = ApplicationProperties.get(); + HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(conf, configuration); + hiveMetaStoreBridge.registerHiveDataModel(); dgiCLient = new AtlasClient(configuration.getString(HiveMetaStoreBridge.ATLAS_ENDPOINT, DGI_URL)); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/sqoop-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/pom.xml b/addons/sqoop-bridge/pom.xml index 53949ef..be32e18 100644 --- a/addons/sqoop-bridge/pom.xml +++ b/addons/sqoop-bridge/pom.xml @@ -352,6 +352,27 @@ <generateReports>false</generateReports> </configuration> </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.2.1</version> + <inherited>false</inherited> + <executions> + <execution> + <configuration> + <mainClass>org.apache.atlas.sqoop.model.SqoopDataModelGenerator</mainClass> + <arguments> + <argument>${project.build.directory}/models/sqoop_model.json</argument> + </arguments> + </configuration> + <phase>package</phase> + <goals> + <goal>java</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> </project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java index af68fcc..b573ac4 100644 --- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java +++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java @@ -70,7 +70,7 @@ public class SqoopHook extends SqoopJobDataPublisher { org.apache.hadoop.conf.Configuration.addDefaultResource("sqoop-site.xml"); } - private synchronized void registerDataModels(AtlasClient client, Configuration atlasConf) throws Exception { + synchronized void registerDataModels(AtlasClient client, Configuration atlasConf) throws Exception { // Make sure hive model exists HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(new HiveConf(), atlasConf, UserGroupInformation.getCurrentUser().getShortUserName(), UserGroupInformation.getCurrentUser()); @@ -190,7 +190,6 @@ public class SqoopHook extends SqoopJobDataPublisher { UserGroupInformation.getCurrentUser(), UserGroupInformation.getCurrentUser().getShortUserName()); org.apache.hadoop.conf.Configuration sqoopConf = new org.apache.hadoop.conf.Configuration(); String clusterName = sqoopConf.get(ATLAS_CLUSTER_NAME, DEFAULT_CLUSTER_NAME); - registerDataModels(atlasClient, atlasProperties); Referenceable dbStoreRef = createDBStoreInstance(data); Referenceable dbRef = createHiveDatabaseInstance(clusterName, data.getHiveDB()); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java index 989469f..7101225 100644 --- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java +++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java @@ -21,6 +21,7 @@ package org.apache.atlas.sqoop.model; import com.google.common.collect.ImmutableList; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; +import org.apache.atlas.addons.ModelDefinitionDump; import org.apache.atlas.typesystem.TypesDef; import org.apache.atlas.typesystem.json.TypesSerialization; import org.apache.atlas.typesystem.types.AttributeDefinition; @@ -161,7 +162,14 @@ public class SqoopDataModelGenerator { public static void main(String[] args) throws Exception { SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator(); - System.out.println("sqoopDataModelAsJSON = " + dataModelGenerator.getModelAsJson()); + String modelAsJson = dataModelGenerator.getModelAsJson(); + + if (args.length == 1) { + ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson); + return; + } + + System.out.println("sqoopDataModelAsJSON = " + modelAsJson); TypesDef typesDef = dataModelGenerator.getTypesDef(); for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java ---------------------------------------------------------------------- diff --git a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java index 5214223..94cd105 100644 --- a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java +++ b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java @@ -22,6 +22,7 @@ import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.hive.model.HiveDataTypes; import org.apache.atlas.sqoop.model.SqoopDataTypes; +import org.apache.commons.configuration.Configuration; import org.apache.sqoop.SqoopJobDataPublisher; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; @@ -41,7 +42,9 @@ public class SqoopHookIT { @BeforeClass public void setUp() throws Exception { //Set-up sqoop session - dgiCLient = new AtlasClient(ApplicationProperties.get().getString("atlas.rest.address")); + Configuration configuration = ApplicationProperties.get(); + dgiCLient = new AtlasClient(configuration.getString("atlas.rest.address")); + new SqoopHook().registerDataModels(dgiCLient, configuration); } @Test http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/storm-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/pom.xml b/addons/storm-bridge/pom.xml index 7b0d366..46528d5 100644 --- a/addons/storm-bridge/pom.xml +++ b/addons/storm-bridge/pom.xml @@ -383,6 +383,28 @@ </configuration> </plugin> + + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.2.1</version> + <inherited>false</inherited> + <executions> + <execution> + <configuration> + <mainClass>org.apache.atlas.storm.model.StormDataModelGenerator</mainClass> + <arguments> + <argument>${project.build.directory}/models/storm_model.json</argument> + </arguments> + </configuration> + <phase>package</phase> + <goals> + <goal>java</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> <groupId>net.alchim31.maven</groupId> <artifactId>scala-maven-plugin</artifactId> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java index 490d95e..4c0004b 100644 --- a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java +++ b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java @@ -98,10 +98,6 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook { LOG.info("Collecting metadata for a new storm topology: {}", topologyInfo.get_name()); try { - if( ! typesRegistered ) { - registerDataModel(new HiveDataModelGenerator()); - } - ArrayList<Referenceable> entities = new ArrayList<>(); Referenceable topologyReferenceable = createTopologyInstance(topologyInfo, stormConf); List<Referenceable> dependentEntities = addTopologyDataSets(stormTopology, topologyReferenceable, http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java new file mode 100644 index 0000000..95eead7 --- /dev/null +++ b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.storm.model; + +import org.apache.atlas.addons.ModelDefinitionDump; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.json.TypesSerialization; + +import java.io.IOException; + +public class StormDataModelGenerator { + + public static void main(String[] args) throws IOException { + StormDataModel.main(new String[]{}); + TypesDef typesDef = StormDataModel.typesDef(); + String stormTypesAsJSON = TypesSerialization.toJson(typesDef); + + if (args.length == 1) { + ModelDefinitionDump.dumpModelToFile(args[0], stormTypesAsJSON); + return; + } + + System.out.println("stormTypesAsJSON = " + stormTypesAsJSON); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java ---------------------------------------------------------------------- diff --git a/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java b/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java index 2463a77..79f1b07 100644 --- a/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java +++ b/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java @@ -57,6 +57,7 @@ public class StormAtlasHookIT { Configuration configuration = ApplicationProperties.get(); atlasClient = new AtlasClient(configuration.getString("atlas.rest.address", ATLAS_URL)); + new StormAtlasHook().registerDataModel(new HiveDataModelGenerator()); } @AfterClass http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/common/src/main/java/org/apache/atlas/addons/ModelDefinitionDump.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/atlas/addons/ModelDefinitionDump.java b/common/src/main/java/org/apache/atlas/addons/ModelDefinitionDump.java new file mode 100644 index 0000000..777bfab --- /dev/null +++ b/common/src/main/java/org/apache/atlas/addons/ModelDefinitionDump.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.addons; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; + +/** + * A class to write a model to a file. + */ +public final class ModelDefinitionDump { + + private ModelDefinitionDump() { + } + + /** + * Write given model as JSON to given file. + * @param outputFileName file name to write model to + * @param modelAsJson model serialized as JSON + * @throws IOException + */ + public static void dumpModelToFile(String outputFileName, String modelAsJson) throws IOException { + File dir = new File(outputFileName).getParentFile(); + if (!dir.exists()) { + dir.mkdirs(); + } + PrintWriter printWriter = new PrintWriter(new BufferedWriter(new FileWriter(outputFileName))); + printWriter.write(modelAsJson); + printWriter.close(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/distro/src/main/assemblies/standalone-package.xml ---------------------------------------------------------------------- diff --git a/distro/src/main/assemblies/standalone-package.xml b/distro/src/main/assemblies/standalone-package.xml index 88d0e60..18dd3cf 100755 --- a/distro/src/main/assemblies/standalone-package.xml +++ b/distro/src/main/assemblies/standalone-package.xml @@ -99,23 +99,43 @@ <outputDirectory>hook</outputDirectory> </fileSet> + <fileSet> + <directory>../addons/hive-bridge/target/models</directory> + <outputDirectory>models</outputDirectory> + </fileSet> + <!-- addons/falcon --> <fileSet> <directory>../addons/falcon-bridge/target/dependency/hook</directory> <outputDirectory>hook</outputDirectory> </fileSet> + <fileSet> + <directory>../addons/falcon-bridge/target/models</directory> + <outputDirectory>models</outputDirectory> + </fileSet> + <!-- addons/sqoop --> <fileSet> <directory>../addons/sqoop-bridge/target/dependency/hook</directory> <outputDirectory>hook</outputDirectory> </fileSet> + <fileSet> + <directory>../addons/sqoop-bridge/target/models</directory> + <outputDirectory>models</outputDirectory> + </fileSet> + <!-- addons/storm --> <fileSet> <directory>../addons/storm-bridge/target/dependency/hook</directory> <outputDirectory>hook</outputDirectory> </fileSet> + + <fileSet> + <directory>../addons/storm-bridge/target/models</directory> + <outputDirectory>models</outputDirectory> + </fileSet> </fileSets> <files> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 9330850..f5ca60b 100644 --- a/release-log.txt +++ b/release-log.txt @@ -11,6 +11,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-512 Decouple currently integrating components from availability of Atlas service for raising metadata events ( yhemanth via sumasai) ATLAS-537 Falcon hook failing when tried to submit a process which creates a hive table ( shwethags via sumasai) ATLAS-476 Update type attribute with Reserved characters updated the original type as unknown (yhemanth via shwethags) ATLAS-463 Disconnect inverse references ( dkantor via sumasai) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java index f77c237..7651bc7 100755 --- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java +++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java @@ -38,7 +38,9 @@ import org.apache.atlas.repository.graph.TitanGraphProvider; import org.apache.atlas.repository.typestore.GraphBackedTypeStore; import org.apache.atlas.repository.typestore.ITypeStore; import org.apache.atlas.services.DefaultMetadataService; +import org.apache.atlas.services.IBootstrapTypesRegistrar; import org.apache.atlas.services.MetadataService; +import org.apache.atlas.services.ReservedTypesRegistrar; import org.apache.atlas.typesystem.types.TypeSystem; /** @@ -71,6 +73,8 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule { // bind the MetadataService interface to an implementation bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton(); + bind(IBootstrapTypesRegistrar.class).to(ReservedTypesRegistrar.class); + // bind the DiscoveryService interface to an implementation bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java index 67e4826..fe132b5 100755 --- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java +++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java @@ -68,6 +68,7 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; import javax.inject.Singleton; +import java.io.File; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -89,18 +90,28 @@ public class DefaultMetadataService implements MetadataService { private final TypeSystem typeSystem; private final MetadataRepository repository; private final ITypeStore typeStore; + private IBootstrapTypesRegistrar typesRegistrar; private final Collection<Provider<TypesChangeListener>> typeChangeListeners; @Inject DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, + final IBootstrapTypesRegistrar typesRegistrar, final Collection<Provider<TypesChangeListener>> typeChangeListeners) throws AtlasException { + this(repository, typeStore, typesRegistrar, typeChangeListeners, TypeSystem.getInstance()); + } + DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore, + final IBootstrapTypesRegistrar typesRegistrar, + final Collection<Provider<TypesChangeListener>> typeChangeListeners, + final TypeSystem typeSystem) throws AtlasException { this.typeStore = typeStore; - this.typeSystem = TypeSystem.getInstance(); + this.typesRegistrar = typesRegistrar; + this.typeSystem = typeSystem; this.repository = repository; this.typeChangeListeners = typeChangeListeners; restoreTypeSystem(); + typesRegistrar.registerTypes(ReservedTypesRegistrar.getTypesDir(), typeSystem, this); } private void restoreTypeSystem() { @@ -111,7 +122,6 @@ public class DefaultMetadataService implements MetadataService { // restore types before creating super types createSuperTypes(); - } catch (AtlasException e) { throw new RuntimeException(e); } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java b/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java new file mode 100644 index 0000000..fce5cb3 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.services; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.types.TypeSystem; + +public interface IBootstrapTypesRegistrar { + void registerTypes(String typesDirName, TypeSystem typeSystem, MetadataService metadataService) + throws AtlasException; +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java b/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java new file mode 100644 index 0000000..430bb6b --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.services; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.json.TypesSerialization; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + +public class ReservedTypesRegistrar implements IBootstrapTypesRegistrar { + + private static final Logger LOG = LoggerFactory.getLogger(ReservedTypesRegistrar.class); + + static String getTypesDir() { + return System.getProperty("atlas.home")+ File.separator+"models"; + } + + @Override + public void registerTypes(String typesDirName, TypeSystem typeSystem, MetadataService metadataService) + throws AtlasException { + File typesDir = new File(typesDirName); + if (!typesDir.exists()) { + LOG.info("No types directory {} found - not registering any reserved types", typesDirName); + return; + } + File[] typeDefFiles = typesDir.listFiles(); + for (File typeDefFile : typeDefFiles) { + try { + String typeDefJSON = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8); + registerType(typeSystem, metadataService, typeDefFile.getAbsolutePath(), typeDefJSON); + } catch (IOException e) { + e.printStackTrace(); + } + } + + } + + void registerType(TypeSystem typeSystem, MetadataService metadataService, String typeDefName, String typeDefJSON) + throws AtlasException { + TypesDef typesDef = null; + try { + typesDef = TypesSerialization.fromJson(typeDefJSON); + } catch (Exception e) { + LOG.error("Error while deserializing JSON in {}", typeDefName); + throw new ReservedTypesRegistrationException("Error while deserializing JSON in " + typeDefName, e); + } + HierarchicalTypeDefinition<ClassType> classDef = typesDef.classTypesAsJavaList().get(0); + if (!typeSystem.isRegistered(classDef.typeName)) { + metadataService.createType(typeDefJSON); + LOG.info("Registered types in {}", typeDefName); + } else { + LOG.warn("class {} already registered, ignoring types in {}", classDef.typeName, + typeDefName); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrationException.java ---------------------------------------------------------------------- diff --git a/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrationException.java b/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrationException.java new file mode 100644 index 0000000..4b3b31d --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrationException.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.atlas.services; + +public class ReservedTypesRegistrationException extends RuntimeException { + public ReservedTypesRegistrationException(String message, Exception e) { + super(message, e); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java b/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java new file mode 100644 index 0000000..84ec761 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.services; + +import com.google.inject.Provider; +import org.apache.atlas.AtlasException; +import org.apache.atlas.listener.TypesChangeListener; +import org.apache.atlas.repository.MetadataRepository; +import org.apache.atlas.repository.typestore.ITypeStore; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.mockito.Matchers; +import org.testng.annotations.Test; + +import java.util.ArrayList; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DefaultMetadataServiceMockTest { + + @Test + public void testShouldInvokeTypesRegistrarOnCreation() throws AtlasException { + IBootstrapTypesRegistrar typesRegistrar = mock(IBootstrapTypesRegistrar.class); + TypeSystem typeSystem = mock(TypeSystem.class); + when(typeSystem.isRegistered(any(String.class))).thenReturn(true); + DefaultMetadataService defaultMetadataService = new DefaultMetadataService(mock(MetadataRepository.class), + mock(ITypeStore.class), + typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(), typeSystem); + + verify(typesRegistrar).registerTypes(ReservedTypesRegistrar.getTypesDir(), + typeSystem, defaultMetadataService); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/98fdc6d8/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java ---------------------------------------------------------------------- diff --git a/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java b/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java new file mode 100644 index 0000000..d602bd8 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.services; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.TestUtils; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.json.TypesSerialization; +import org.apache.atlas.typesystem.types.TypeSystem; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +public class ReservedTypesRegistrarTest { + + @Mock + private TypeSystem typeSystem; + + @Mock + private MetadataService metadataService; + + @BeforeMethod + public void setup() { + MockitoAnnotations.initMocks(this); + } + + @Test + public void testRegistrationWithNoFiles() throws AtlasException { + IBootstrapTypesRegistrar bootstrapTypesRegistrar = new ReservedTypesRegistrar(); + bootstrapTypesRegistrar.registerTypes("/some/dir/", typeSystem, metadataService); + verifyZeroInteractions(typeSystem); + } + + @Test + public void testRegisterFirstChecksClassTypeIsRegistered() throws AtlasException { + ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar(); + TypesDef typesDef = TestUtils.defineHiveTypes(); + String typesJson = TypesSerialization.toJson(typesDef); + reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson); + InOrder inOrder = inOrder(typeSystem, metadataService); + inOrder.verify(typeSystem).isRegistered(typesDef.classTypesAsJavaList().get(0).typeName); + inOrder.verify(metadataService).createType(typesJson); + } + + @Test + public void testRegisterCreatesTypesUsingMetadataService() throws AtlasException { + ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar(); + TypesDef typesDef = TestUtils.defineHiveTypes(); + String typesJson = TypesSerialization.toJson(typesDef); + reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson); + verify(metadataService).createType(typesJson); + } + + @Test(expectedExceptions = ReservedTypesRegistrationException.class) + public void testRegisterFailsIfErrorInJson() throws AtlasException { + ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar(); + reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", "invalid json"); + } + + @Test(expectedExceptions = AtlasException.class) + public void testRegisterFailsOnTypeCreationException() throws AtlasException { + ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar(); + TypesDef typesDef = TestUtils.defineHiveTypes(); + String typesJson = TypesSerialization.toJson(typesDef); + when(metadataService.createType(typesJson)).thenThrow(new AtlasException("some exception")); + reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson); + } + + @Test + public void testShouldNotRegisterIfTypeIsAlreadyRegistered() throws AtlasException { + ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar(); + TypesDef typesDef = TestUtils.defineHiveTypes(); + String typesJson = TypesSerialization.toJson(typesDef); + when(typeSystem.isRegistered(typesDef.classTypesAsJavaList().get(0).typeName)).thenReturn(true); + reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson); + verifyZeroInteractions(metadataService); + } +}
