Remove falcon and metadata-bridge addon modules
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/054dfb76 Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/054dfb76 Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/054dfb76 Branch: refs/heads/apache-local Commit: 054dfb763ef6b945060f0a4e005cd55d1733c62b Parents: fef50ce Author: Venkatesh Seetharam <[email protected]> Authored: Fri Jun 12 11:23:23 2015 -0700 Committer: Venkatesh Seetharam <[email protected]> Committed: Fri Jun 12 11:23:23 2015 -0700 ---------------------------------------------------------------------- addons/falcon-bridge/pom.xml | 82 ----- .../org/apache/atlas/falcon/FalconImporter.java | 151 -------- .../apache/atlas/falcon/FalconTypeSystem.java | 165 --------- .../apache/atlas/falcon/FalconImporterTest.java | 109 ------ .../atlas/falcon/FalconTypeSystemTest.java | 37 -- .../metadata-bridge-core/.gitignore | 17 - .../metadata-bridge-core/pom.xml | 75 ---- .../apache/hadoop/metadata/bridge/ABridge.java | 158 --------- .../hadoop/metadata/bridge/AEntityBean.java | 23 -- .../hadoop/metadata/bridge/BridgeException.java | 34 -- .../hadoop/metadata/bridge/BridgeManager.java | 147 -------- .../metadata/bridge/BridgeTypeBootstrapper.java | 99 ------ .../apache/hadoop/metadata/bridge/IBridge.java | 25 -- .../bridge/hivelineage/HiveLineageBridge.java | 35 -- .../bridge/hivelineage/hook/HiveLineage.java | 349 ------------------- .../bridge/hivestructure/HiveMetaImporter.java | 215 ------------ .../hivestructure/HiveStructureBridge.java | 110 ------ .../metadata/bridge/module/BridgeModule.java | 83 ----- .../web/resources/HiveLineageResource.java | 82 ----- .../main/resources/bridge-manager.properties | 22 -- .../metadata/bridge/BridgeManagerTest.java | 57 --- .../metadata/bridge/TestBridgeModule.java | 34 -- .../metadata/bridge/TestGenericBridges.java | 25 -- .../hivelineage/TestHiveLineageBridge.java | 85 ----- .../test/resources/bridge-manager.properties | 22 -- .../src/test/resources/graph.properties | 25 -- .../src/test/resources/one.json | 48 --- .../resources/test-bridge-manager.properties | 22 -- .../src/test/resources/two.json | 48 --- addons/metadata-bridge-parent/pom.xml | 35 -- 30 files changed, 2419 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/falcon-bridge/pom.xml ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/pom.xml b/addons/falcon-bridge/pom.xml deleted file mode 100755 index fd71258..0000000 --- a/addons/falcon-bridge/pom.xml +++ /dev/null @@ -1,82 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xmlns="http://maven.apache.org/POM/4.0.0"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <artifactId>apache-atlas</artifactId> - <groupId>org.apache.atlas</groupId> - <version>0.1-incubating-SNAPSHOT</version> - <relativePath>../../</relativePath> - </parent> - <artifactId>falcon-bridge</artifactId> - <description>Apache Atlas Falcon Bridge Module</description> - <name>Apache Atlas Falcon Bridge</name> - <packaging>jar</packaging> - - <properties> - <falcon.version>0.6.0.2.2.0.0-2041</falcon.version> - </properties> - - <dependencies> - <dependency> - <groupId>org.apache.falcon</groupId> - <artifactId>falcon-client</artifactId> - <version>${falcon.version}</version> - </dependency> - - <!-- falcon-client depends on jersey-client in provided scope. Hence explicit dependency --> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-client</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.atlas</groupId> - <artifactId>atlas-typesystem</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.atlas</groupId> - <artifactId>atlas-repository</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - </dependency> - - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </dependency> - - <dependency> - <groupId>org.testng</groupId> - <artifactId>testng</artifactId> - </dependency> - - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-all</artifactId> - </dependency> - </dependencies> -</project> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java deleted file mode 100755 index 82710ba..0000000 --- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconImporter.java +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.falcon; - -import com.google.inject.Inject; -import org.apache.atlas.MetadataException; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.typesystem.ITypedInstance; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.Struct; -import org.apache.atlas.typesystem.types.EnumType; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.StructType; -import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.commons.lang.StringUtils; -import org.apache.falcon.client.FalconCLIException; -import org.apache.falcon.client.FalconClient; -import org.apache.falcon.entity.v0.Entity; -import org.apache.falcon.entity.v0.EntityType; -import org.apache.falcon.entity.v0.cluster.Cluster; -import org.apache.falcon.entity.v0.cluster.Interface; -import org.apache.falcon.entity.v0.cluster.Location; -import org.apache.falcon.entity.v0.cluster.Properties; -import org.apache.falcon.entity.v0.cluster.Property; -import org.apache.falcon.resource.EntityList; - -import javax.xml.bind.JAXBException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class FalconImporter { - - private static final TypeSystem typeSystem = TypeSystem.getInstance(); - - private final FalconClient client; - private final MetadataRepository repository; - - @Inject - public FalconImporter(FalconClient client, MetadataRepository repo) { - this.client = client; - this.repository = repo; - } - - private Entity getEntity(FalconClient client, EntityType type, String name) - throws FalconCLIException, JAXBException { - String entityStr = client.getDefinition(type.name(), name); - return (Entity) type.getUnmarshaller().unmarshal(new StringReader(entityStr)); - } - - public void importClusters() throws MetadataException { - try { - EntityList clusters = - client.getEntityList(EntityType.CLUSTER.name(), null, null, null, null, null, null, null); - for (EntityList.EntityElement element : clusters.getElements()) { - Cluster cluster = (Cluster) getEntity(client, EntityType.CLUSTER, element.name); - - Referenceable clusterRef = new Referenceable(FalconTypeSystem.DefinedTypes.CLUSTER.name()); - clusterRef.set("name", cluster.getName()); - - if (cluster.getACL() != null) { - Struct acl = new Struct(FalconTypeSystem.DefinedTypes.ACL.name()); - acl.set("owner", cluster.getACL().getOwner()); - acl.set("group", cluster.getACL().getGroup()); - acl.set("permission", cluster.getACL().getPermission()); - StructType aclType = - typeSystem.getDataType(StructType.class, FalconTypeSystem.DefinedTypes.ACL.name()); - clusterRef.set("acl", aclType.convert(acl, Multiplicity.REQUIRED)); - } - - if (StringUtils.isNotEmpty(cluster.getTags())) { - String[] parts = cluster.getTags().split(","); - List<ITypedInstance> tags = new ArrayList<>(); - for (String part : parts) { - TraitType tagType = - typeSystem.getDataType(TraitType.class, FalconTypeSystem.DefinedTypes.TAG.name()); - String[] kv = part.trim().split("="); - Struct tag = new Struct(FalconTypeSystem.DefinedTypes.TAG.name()); - tag.set("name", kv[0]); - tag.set("value", kv[0]); - tags.add(tagType.convert(tag, Multiplicity.REQUIRED)); - } - clusterRef.set("tags", tags); - } - - if (cluster.getProperties() != null) { - clusterRef.set("properties", getMap(cluster.getProperties())); - } - - if (cluster.getLocations() != null) { - List<ITypedInstance> locations = new ArrayList<>(); - for (Location loc : cluster.getLocations().getLocations()) { - Struct location = new Struct(FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION.name()); - EnumType locationType = typeSystem.getDataType(EnumType.class, - FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION_TYPE.name()); - location.set("type", locationType.fromValue(loc.getName().toUpperCase())); - location.set("path", loc.getPath()); - StructType type = typeSystem - .getDataType(StructType.class, FalconTypeSystem.DefinedTypes.CLUSTER_LOCATION.name()); - locations.add(type.convert(location, Multiplicity.REQUIRED)); - } - clusterRef.set("locations", locations); - } - - if (cluster.getInterfaces() != null) { - List<ITypedInstance> interfaces = new ArrayList<>(); - for (Interface interfaceFld : cluster.getInterfaces().getInterfaces()) { - Struct interfaceStruct = new Struct(FalconTypeSystem.DefinedTypes.CLUSTER_INTERFACE.name()); - interfaceStruct.set("type", interfaceFld.getType().name()); - interfaceStruct.set("endpoint", interfaceFld.getEndpoint()); - interfaceStruct.set("version", interfaceFld.getVersion()); - StructType type = typeSystem - .getDataType(StructType.class, FalconTypeSystem.DefinedTypes.CLUSTER_INTERFACE.name()); - interfaces.add(type.convert(interfaceStruct, Multiplicity.REQUIRED)); - } - clusterRef.set("interfaces", interfaces); - } - repository.createEntity(clusterRef); - } - } catch (Exception e) { - throw new MetadataException(e); - } - } - - private Map<String, String> getMap(Properties properties) { - Map<String, String> map = new HashMap(); - for (Property property : properties.getProperties()) { - map.put(property.getName().trim(), property.getValue().trim()); - } - return map; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java deleted file mode 100755 index 4e06777..0000000 --- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/FalconTypeSystem.java +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.falcon; - -import com.google.common.collect.ImmutableList; -import org.apache.atlas.MetadataException; -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.DataTypes; -import org.apache.atlas.typesystem.types.EnumTypeDefinition; -import org.apache.atlas.typesystem.types.EnumValue; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.StructTypeDefinition; -import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -public class FalconTypeSystem { - public static final Logger LOG = LoggerFactory.getLogger(FalconTypeSystem.class); - public static final TypeSystem TYPE_SYSTEM = TypeSystem.getInstance(); - private static FalconTypeSystem INSTANCE; - - private List<StructTypeDefinition> structTypeDefinitions = new ArrayList<>(); - private List<HierarchicalTypeDefinition<TraitType>> traitTypeDefinitions = new ArrayList<>(); - - private FalconTypeSystem() throws MetadataException { - HierarchicalTypeDefinition<ClassType> cluster = defineCluster(); - //TODO define feed and process - - TYPE_SYSTEM.defineTypes(ImmutableList.copyOf(structTypeDefinitions), ImmutableList.copyOf(traitTypeDefinitions), - ImmutableList.of(cluster)); - } - - public static FalconTypeSystem getInstance() throws MetadataException { - if (INSTANCE == null) { - synchronized (LOG) { - if (INSTANCE == null) { - INSTANCE = new FalconTypeSystem(); - } - } - } - return INSTANCE; - } - - private HierarchicalTypeDefinition<ClassType> defineCluster() throws MetadataException { - defineACL(); - defineClusterInterface(); - defineClusterLocation(); - defineTags(); - - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("acl", DefinedTypes.ACL.name(), Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("tags", DefinedTypes.TAG.name(), Multiplicity.COLLECTION, false, null), - new AttributeDefinition("locations", - TYPE_SYSTEM.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), - Multiplicity.COLLECTION, false, null), - new AttributeDefinition("interfaces", DefinedTypes.CLUSTER_INTERFACE.name(), Multiplicity.COLLECTION, - false, null), new AttributeDefinition("properties", - TYPE_SYSTEM.defineMapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE).getName(), - Multiplicity.OPTIONAL, false, null),}; - HierarchicalTypeDefinition<ClassType> cluster = - new HierarchicalTypeDefinition<>(ClassType.class, DefinedTypes.CLUSTER.name(), - ImmutableList.<String>of(), attributeDefinitions); - LOG.debug("Created definition for " + DefinedTypes.CLUSTER.name()); - return cluster; - } - - private HierarchicalTypeDefinition<TraitType> defineTags() { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("name", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("value", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null)}; - - HierarchicalTypeDefinition<TraitType> traitType = - new HierarchicalTypeDefinition<>(TraitType.class, DefinedTypes.TAG.name(), ImmutableList.<String>of(), - attributeDefinitions); - LOG.debug("Created definition for " + DefinedTypes.TAG.name()); - traitTypeDefinitions.add(traitType); - return traitType; - } - - private StructTypeDefinition defineClusterLocation() throws MetadataException { - EnumValue values[] = {new EnumValue("WORKING", 1), new EnumValue("STAGING", 2), new EnumValue("TEMP", 3),}; - - LOG.debug("Created definition for " + DefinedTypes.CLUSTER_LOCATION_TYPE.name()); - EnumTypeDefinition locationType = new EnumTypeDefinition(DefinedTypes.CLUSTER_LOCATION_TYPE.name(), values); - TYPE_SYSTEM.defineEnumType(locationType); - - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("type", DefinedTypes.CLUSTER_LOCATION_TYPE.name(), Multiplicity.REQUIRED, false, - null), - new AttributeDefinition("path", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),}; - LOG.debug("Created definition for " + DefinedTypes.CLUSTER_LOCATION.name()); - StructTypeDefinition location = - new StructTypeDefinition(DefinedTypes.CLUSTER_LOCATION.name(), attributeDefinitions); - structTypeDefinitions.add(location); - return location; - } - - private StructTypeDefinition defineClusterInterface() throws MetadataException { - EnumValue values[] = {new EnumValue("READONLY", 1), new EnumValue("WRITE", 2), new EnumValue("EXECUTE", 3), - new EnumValue("WORKFLOW", 4), new EnumValue("MESSAGING", 5), new EnumValue("REGISTRY", 6),}; - - LOG.debug("Created definition for " + DefinedTypes.CLUSTER_INTERFACE_TYPE.name()); - EnumTypeDefinition interfaceType = new EnumTypeDefinition(DefinedTypes.CLUSTER_INTERFACE_TYPE.name(), values); - TYPE_SYSTEM.defineEnumType(interfaceType); - - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("type", DefinedTypes.CLUSTER_INTERFACE_TYPE.name(), Multiplicity.REQUIRED, - false, null), - new AttributeDefinition("endpoint", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - null), - new AttributeDefinition("version", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, - null),}; - LOG.debug("Created definition for " + DefinedTypes.CLUSTER_INTERFACE.name()); - StructTypeDefinition interfaceEntity = - new StructTypeDefinition(DefinedTypes.CLUSTER_INTERFACE.name(), attributeDefinitions); - structTypeDefinitions.add(interfaceEntity); - return interfaceEntity; - } - - public static enum DefinedTypes { - ACL, - TAG, - - CLUSTER, - CLUSTER_INTERFACE, - CLUSTER_INTERFACE_TYPE, - CLUSTER_LOCATION, - CLUSTER_LOCATION_TYPE - } - - private StructTypeDefinition defineACL() { - AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{ - new AttributeDefinition("owner", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("group", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null), - new AttributeDefinition("permission", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, - null),}; - LOG.debug("Created definition for " + DefinedTypes.ACL.name()); - StructTypeDefinition acl = new StructTypeDefinition(DefinedTypes.ACL.name(), attributeDefinitions); - structTypeDefinitions.add(acl); - return acl; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java deleted file mode 100755 index 924b2a4..0000000 --- a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconImporterTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.falcon; - -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.typesystem.IReferenceableInstance; -import org.apache.commons.lang.RandomStringUtils; -import org.apache.falcon.client.FalconClient; -import org.apache.falcon.entity.v0.EntityType; -import org.apache.falcon.entity.v0.cluster.Cluster; -import org.apache.falcon.entity.v0.cluster.Interface; -import org.apache.falcon.entity.v0.cluster.Interfaces; -import org.apache.falcon.entity.v0.cluster.Interfacetype; -import org.apache.falcon.entity.v0.cluster.Location; -import org.apache.falcon.entity.v0.cluster.Locations; -import org.apache.falcon.resource.EntityList; -import org.testng.annotations.Test; - -import java.io.StringWriter; -import java.util.UUID; - -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class FalconImporterTest { - @Test - public void testImport() throws Exception { - MetadataRepository repo = mock(MetadataRepository.class); - FalconClient client = mock(FalconClient.class); - FalconTypeSystem.getInstance(); - - FalconImporter importer = new FalconImporter(client, repo); - when(client.getEntityList(EntityType.CLUSTER.name(), null, null, null, null, null, null, null)) - .thenReturn(getEntityList()); - //TODO Set other fields in cluster - when(client.getDefinition(anyString(), anyString())).thenReturn(getCluster()); - when(repo.createEntity(any(IReferenceableInstance.class), anyString())) - .thenReturn(UUID.randomUUID().toString()); - - importer.importClusters(); - } - - public EntityList getEntityList() { - EntityList.EntityElement[] entities = new EntityList.EntityElement[2]; - entities[0] = new EntityList.EntityElement(); - entities[0].name = "c1"; - entities[1] = new EntityList.EntityElement(); - entities[1].name = "c2"; - return new EntityList(entities); - } - - private Interface getInterface(Interfacetype type, String endpoint) { - Interface clusterInterface = new Interface(); - clusterInterface.setEndpoint(endpoint); - clusterInterface.setType(type); - clusterInterface.setVersion("2.2"); - return clusterInterface; - } - - public String getCluster() throws Exception { - Cluster cluster = new Cluster(); - cluster.setName(RandomStringUtils.randomAlphabetic(10)); - cluster.setColo(RandomStringUtils.randomAlphabetic(5)); - - cluster.setTags("owner=xyz,team=abc"); - Interfaces interfaces = new Interfaces(); - Interface clusterInterface = new Interface(); - clusterInterface.setEndpoint("hdfs://localhost:8030"); - clusterInterface.setType(Interfacetype.WRITE); - clusterInterface.setVersion("2.2"); - interfaces.getInterfaces().add(getInterface(Interfacetype.WRITE, "hdfs://localhost:8030")); - interfaces.getInterfaces().add(getInterface(Interfacetype.READONLY, "hdfs://localhost:8030")); - interfaces.getInterfaces().add(getInterface(Interfacetype.EXECUTE, "http://localhost:8040";)); - cluster.setInterfaces(interfaces); - - Locations locations = new Locations(); - locations.getLocations().add(getLocation()); - cluster.setLocations(locations); - - StringWriter writer = new StringWriter(); - EntityType.CLUSTER.getMarshaller().marshal(cluster, writer); - return writer.toString(); - } - - public Location getLocation() { - Location location = new Location(); - location.setName("staging"); - location.setPath("/staging"); - return location; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java ---------------------------------------------------------------------- diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java deleted file mode 100755 index 7667903..0000000 --- a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/FalconTypeSystemTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.falcon; - -import org.apache.atlas.MetadataException; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.TraitType; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.junit.Assert; -import org.testng.annotations.Test; - -public class FalconTypeSystemTest { - @Test - public void testTypeSystem() throws MetadataException { - FalconTypeSystem.getInstance(); - Assert.assertNotNull( - TypeSystem.getInstance().getDataType(ClassType.class, FalconTypeSystem.DefinedTypes.CLUSTER.name())); - Assert.assertNotNull( - TypeSystem.getInstance().getDataType(TraitType.class, FalconTypeSystem.DefinedTypes.TAG.name())); - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/metadata-bridge-parent/metadata-bridge-core/.gitignore ---------------------------------------------------------------------- diff --git a/addons/metadata-bridge-parent/metadata-bridge-core/.gitignore b/addons/metadata-bridge-parent/metadata-bridge-core/.gitignore deleted file mode 100755 index 35dcf51..0000000 --- a/addons/metadata-bridge-parent/metadata-bridge-core/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -/test-output/ http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/metadata-bridge-parent/metadata-bridge-core/pom.xml ---------------------------------------------------------------------- diff --git a/addons/metadata-bridge-parent/metadata-bridge-core/pom.xml b/addons/metadata-bridge-parent/metadata-bridge-core/pom.xml deleted file mode 100755 index eebf95e..0000000 --- a/addons/metadata-bridge-parent/metadata-bridge-core/pom.xml +++ /dev/null @@ -1,75 +0,0 @@ -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xmlns="http://maven.apache.org/POM/4.0.0"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.atlas</groupId> - <artifactId>atlas-bridge-parent</artifactId> - <version>0.1-incubating-SNAPSHOT</version> - </parent> - <artifactId>atlas-bridge-core</artifactId> - <dependencies> - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - <version>0.14.0</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - <version>2.2.2</version> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>4.10</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.atlas</groupId> - <artifactId>atlas-repository</artifactId> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-core</artifactId> - </dependency> - <dependency> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jetty</artifactId> - </dependency> - <dependency> - <groupId>org.testng</groupId> - <artifactId>testng</artifactId> - <version>6.1.1</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>commons-configuration</groupId> - <artifactId>commons-configuration</artifactId> - </dependency> - <dependency> - <groupId>com.google.inject.extensions</groupId> - <artifactId>guice-multibindings</artifactId> - <version>3.0</version> - </dependency> - </dependencies> -</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/ABridge.java ---------------------------------------------------------------------- diff --git a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/ABridge.java b/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/ABridge.java deleted file mode 100755 index 6559b72..0000000 --- a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/ABridge.java +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.bridge; - -import com.google.common.collect.ImmutableList; -import org.apache.atlas.MetadataException; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.repository.RepositoryException; -import org.apache.atlas.typesystem.ITypedReferenceableInstance; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.AttributeInfo; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.slf4j.Logger; - -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map.Entry; - -public abstract class ABridge implements IBridge { - - protected static final Logger LOG = BridgeManager.LOG; - protected ArrayList<Class<? extends AEntityBean>> typeBeanClasses = new ArrayList<Class<? extends AEntityBean>>(); - MetadataRepository repo; - - protected ABridge(MetadataRepository repo) { - this.repo = repo; - } - - protected HierarchicalTypeDefinition<ClassType> createClassTypeDef(String name, ImmutableList<String> superTypes, - AttributeDefinition... attrDefs) { - return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attrDefs); - } - - public ArrayList<Class<? extends AEntityBean>> getTypeBeanClasses() { - return typeBeanClasses; - } - - public AEntityBean get(String id) throws RepositoryException { - // get from the system by id (?) - ITypedReferenceableInstance ref = repo.getEntityDefinition(id); - // turn into a HiveLineageBean - try { - Class<AEntityBean> c = getTypeBeanInListByName(ref.getTypeName()); - return this.convertFromITypedReferenceable(ref, getTypeBeanInListByName(ref.getTypeName())); - } catch (BridgeException | InstantiationException | IllegalAccessException | - IllegalArgumentException | InvocationTargetException | NoSuchMethodException | - SecurityException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return null; - } - - public String create(AEntityBean bean) throws MetadataException { - - ClassType type = TypeSystem.getInstance().getDataType(ClassType.class, bean.getClass().getSimpleName()); - ITypedReferenceableInstance refBean = null; - try { - refBean = type.convert(this.convertToReferencable(bean), Multiplicity.REQUIRED); - String id = repo.createEntity(refBean); - return id; - } catch (IllegalArgumentException | IllegalAccessException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - throw new MetadataException("Cannot create entity"); - - } - - public Iterable<String> list() throws RepositoryException { - List<String> returnList = null; - for (Class c : typeBeanClasses) { - List<String> inBetweenList = repo.getEntityList(c.getSimpleName()); - try { - returnList.addAll(inBetweenList); - } catch (NullPointerException e) { - returnList = inBetweenList; - } - } - return returnList; - } - - protected final boolean containsType(String s) { - for (Class c : typeBeanClasses) { - if (c.getSimpleName().equals(s)) { - return true; - } - } - return false; - } - - protected final Class<AEntityBean> getTypeBeanInListByName(String s) throws BridgeException { - if (containsType(s)) { - for (Class c : typeBeanClasses) { - if (c.getSimpleName().equals(s)) { - return c; - } - } - } else { - throw new BridgeException("No EntityBean Definition Found"); - } - throw new BridgeException("No EntityBean Definition Found"); - } - - protected final <T extends AEntityBean> Referenceable convertToReferencable(T o) - throws IllegalArgumentException, IllegalAccessException { - Referenceable selfAware = new Referenceable(o.getClass().getSimpleName()); - // TODO - support non-primitive types and deep inspection - for (Field f : o.getClass().getFields()) { - selfAware.set(f.getName(), f.get(o)); - } - return selfAware; - } - - protected final <T extends AEntityBean> T convertFromITypedReferenceable(ITypedReferenceableInstance instance, - Class<? extends AEntityBean> c) - throws InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, - NoSuchMethodException, SecurityException, BridgeException { - if (!instance.getTypeName().equals(c.getSimpleName())) { - throw new BridgeException("ReferenceableInstance type not the same as bean"); - } - Object retObj = this.getClass().newInstance(); - for (Entry<String, AttributeInfo> e : instance.fieldMapping().fields.entrySet()) { - try { - - String convertedName = e.getKey().substring(0, 1).toUpperCase() + e.getKey().substring(1); - this.getClass().getMethod("set" + convertedName, Class.forName(e.getValue().dataType().getName())) - .invoke(this, instance.get(e.getKey())); - } catch (MetadataException | ClassNotFoundException e1) { - // TODO Auto-generated catch block - e1.printStackTrace(); - } - } - return (T) retObj; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/AEntityBean.java ---------------------------------------------------------------------- diff --git a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/AEntityBean.java b/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/AEntityBean.java deleted file mode 100755 index 81bdecb..0000000 --- a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/AEntityBean.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.bridge; - -public abstract class AEntityBean { - -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeException.java ---------------------------------------------------------------------- diff --git a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeException.java b/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeException.java deleted file mode 100755 index e9c5ca0..0000000 --- a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeException.java +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.bridge; - -import org.apache.hadoop.hive.metastore.api.MetaException; - -public class BridgeException extends MetaException { - - /** - * - */ - private static final long serialVersionUID = -384401342591560473L; - - public BridgeException(String msg) { - super(msg); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeManager.java ---------------------------------------------------------------------- diff --git a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeManager.java b/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeManager.java deleted file mode 100755 index 036c24d..0000000 --- a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeManager.java +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.bridge; - -//TODO - Create Index Annotation Framework for BeanConverter -//TODO - Enhance Bean Conversion to handled nested objects -//TODO - Enhance Bean COnversion to handle Collections - -import org.apache.atlas.MetadataException; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; - - -public class BridgeManager { - - public static final Logger LOG = LoggerFactory.getLogger("BridgeLogger"); - private final static String bridgeFileDefault = "bridge-manager.properties"; - TypeSystem ts; - MetadataRepository rs; - ArrayList<ABridge> activeBridges; - - @Inject - BridgeManager(MetadataRepository rs) - throws ConfigurationException, ClassNotFoundException, InstantiationException, IllegalAccessException, - IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException { - this.ts = TypeSystem.getInstance(); - this.rs = rs; - if (System.getProperty("bridgeManager.propsFile") != null - && System.getProperty("bridgeManager.propsFile").length() != 0) { - setActiveBridges(System.getProperty("bridgeManager.propsFile")); - } else { - setActiveBridges(bridgeFileDefault); - } - - for (ABridge bridge : activeBridges) { - try { - this.loadTypes(bridge, ts); - } catch (MetadataException e) { - BridgeManager.LOG.error(e.getMessage(), e); - e.printStackTrace(); - } - } - - } - - public final static HierarchicalTypeDefinition<ClassType> convertEntityBeanToClassTypeDefinition( - Class<? extends AEntityBean> class1) { - ArrayList<AttributeDefinition> attDefAL = new ArrayList<AttributeDefinition>(); - for (Field f : class1.getFields()) { - try { - attDefAL.add(BridgeManager.convertFieldtoAttributeDefiniton(f)); - } catch (MetadataException e) { - BridgeManager.LOG.error("Class " + class1.getName() + - " cannot be converted to TypeDefinition"); - e.printStackTrace(); - } - } - - HierarchicalTypeDefinition<ClassType> typeDef = - new HierarchicalTypeDefinition<>(ClassType.class, class1.getSimpleName(), null, - (AttributeDefinition[]) attDefAL.toArray(new AttributeDefinition[0])); - - return typeDef; - } - - public final static AttributeDefinition convertFieldtoAttributeDefiniton(Field f) throws MetadataException { - - return new AttributeDefinition(f.getName(), f.getType().getSimpleName(), Multiplicity.REQUIRED, false, null); - } - - public ArrayList<ABridge> getActiveBridges() { - return this.activeBridges; - } - - private void setActiveBridges(String bridgePropFileName) { - if (bridgePropFileName == null || bridgePropFileName.isEmpty()) { - bridgePropFileName = BridgeManager.bridgeFileDefault; - } - ArrayList<ABridge> aBList = new ArrayList<ABridge>(); - - PropertiesConfiguration config = new PropertiesConfiguration(); - - try { - BridgeManager.LOG.info("Loading : Active Bridge List"); - config.load(bridgePropFileName); - String[] activeBridgeList = ((String) config.getProperty("BridgeManager.activeBridges")).split(","); - BridgeManager.LOG.info("Loaded : Active Bridge List"); - BridgeManager.LOG.info("First Loaded :" + activeBridgeList[0]); - - for (String s : activeBridgeList) { - Class<?> bridgeCls = (Class<?>) Class.forName(s); - if (ABridge.class.isAssignableFrom(bridgeCls)) { - System.out.println(s + " is able to be instaciated"); - aBList.add((ABridge) bridgeCls.getConstructor(MetadataRepository.class).newInstance(rs)); - } - } - - } catch (InstantiationException | ConfigurationException | IllegalAccessException | - IllegalArgumentException | InvocationTargetException | NoSuchMethodException | - SecurityException | ClassNotFoundException e) { - BridgeManager.LOG.error(e.getMessage(), e); - e.printStackTrace(); - } - this.activeBridges = aBList; - - - } - - private final boolean loadTypes(ABridge bridge, TypeSystem ts) throws MetadataException { - for (Class<? extends AEntityBean> clazz : bridge.getTypeBeanClasses()) { - ts.defineClassType(BridgeManager.convertEntityBeanToClassTypeDefinition(clazz)); - } - return false; - - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeTypeBootstrapper.java ---------------------------------------------------------------------- diff --git a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeTypeBootstrapper.java b/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeTypeBootstrapper.java deleted file mode 100755 index 3cdd2d2..0000000 --- a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/BridgeTypeBootstrapper.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.bridge; - -import org.apache.atlas.MetadataException; -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.inject.Inject; -import javax.inject.Singleton; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Map; - -@Singleton -public class BridgeTypeBootstrapper { - private static final Logger LOG = LoggerFactory.getLogger(BridgeTypeBootstrapper.class); - private final Map<Class, IBridge> bridges; - private boolean isSetup = false; - - @Inject - BridgeTypeBootstrapper(Map<Class, IBridge> bridges) throws MetadataException { - this.bridges = bridges; - } - - public final static HierarchicalTypeDefinition<ClassType> convertEntityBeanToClassTypeDefinition( - Class<? extends AEntityBean> class1) { - ArrayList<AttributeDefinition> attDefAL = new ArrayList<AttributeDefinition>(); - for (Field f : class1.getFields()) { - try { - attDefAL.add(BridgeTypeBootstrapper.convertFieldtoAttributeDefiniton(f)); - } catch (MetadataException e) { - BridgeManager.LOG.error("Class " + class1.getName() + " cannot be converted to TypeDefinition"); - e.printStackTrace(); - } - } - - HierarchicalTypeDefinition<ClassType> typeDef = - new HierarchicalTypeDefinition<>(ClassType.class, class1.getSimpleName(), null, - (AttributeDefinition[]) attDefAL.toArray(new AttributeDefinition[0])); - - return typeDef; - } - - public final static AttributeDefinition convertFieldtoAttributeDefiniton(Field f) throws MetadataException { - - return new AttributeDefinition(f.getName(), f.getType().getSimpleName().toLowerCase(), Multiplicity.REQUIRED, - false, null); - } - - public synchronized boolean bootstrap() throws MetadataException { - if (isSetup) { - return false; - } else { - LOG.info("Bootstrapping types"); - _bootstrap(); - isSetup = true; - LOG.info("Bootstrapping complete"); - return true; - } - } - - private void _bootstrap() throws MetadataException { - TypeSystem ts = TypeSystem.getInstance(); - for (IBridge bridge : bridges.values()) { - LOG.info("Registering bridge, %s", bridge.getClass().getSimpleName()); - loadTypes(bridge, ts); - } - } - - private final boolean loadTypes(IBridge bridge, TypeSystem ts) throws MetadataException { - for (Class<? extends AEntityBean> clazz : bridge.getTypeBeanClasses()) { - LOG.info("Registering %s", clazz.getSimpleName()); - ts.defineClassType(BridgeTypeBootstrapper.convertEntityBeanToClassTypeDefinition(clazz)); - } - return false; - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/IBridge.java ---------------------------------------------------------------------- diff --git a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/IBridge.java b/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/IBridge.java deleted file mode 100755 index b2ec333..0000000 --- a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/IBridge.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.bridge; - -import java.util.ArrayList; - -public interface IBridge { - ArrayList<Class<? extends AEntityBean>> getTypeBeanClasses(); -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/HiveLineageBridge.java ---------------------------------------------------------------------- diff --git a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/HiveLineageBridge.java b/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/HiveLineageBridge.java deleted file mode 100755 index 4a37a79..0000000 --- a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/HiveLineageBridge.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.bridge.hivelineage; - -import org.apache.atlas.bridge.ABridge; -import org.apache.atlas.bridge.hivelineage.hook.HiveLineage; -import org.apache.atlas.repository.MetadataRepository; - -import javax.inject.Inject; - -public class HiveLineageBridge extends ABridge { - - @Inject - HiveLineageBridge(MetadataRepository mr) { - super(mr); - this.typeBeanClasses.add(HiveLineage.class); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/hook/HiveLineage.java ---------------------------------------------------------------------- diff --git a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/hook/HiveLineage.java b/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/hook/HiveLineage.java deleted file mode 100755 index 06c010d..0000000 --- a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivelineage/hook/HiveLineage.java +++ /dev/null @@ -1,349 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.bridge.hivelineage.hook; - - -import org.apache.atlas.bridge.AEntityBean; - -import java.io.Serializable; -import java.util.ArrayList; - -public class HiveLineage extends AEntityBean implements Serializable { - - /** - * - */ - private static final long serialVersionUID = 1L; - - public String queryId; - public String hiveId; - public String user; - public String queryStartTime; - public String queryEndTime; - public String query; - public String tableName; - public String tableLocation; - public boolean success; - public boolean failed; - public String executionEngine; - ArrayList<SourceTables> sourceTables; - ArrayList<QueryColumns> queryColumns; - ArrayList<WhereClause> whereClause; - ArrayList<CreateColumns> createColumns; - ArrayList<GroupBy> groupBy; - ArrayList<GroupBy> orderBy; - - - public String getQueryId() { - return this.queryId; - } - - public void setQueryId(String queryId) { - this.queryId = queryId; - } - - public String getExecutionEngine() { - return this.executionEngine; - } - - public void setExecutionEngine(String executionEngine) { - this.executionEngine = executionEngine; - } - - public String getHiveId() { - return this.hiveId; - } - - public void setHiveId(String hiveId) { - this.hiveId = hiveId; - } - - public boolean getSuccess() { - return this.success; - } - - public void setSuccess(boolean success) { - this.success = success; - } - - public boolean getFailed() { - return this.failed; - } - - public void setFailed(boolean failed) { - this.failed = failed; - } - - - public String getTableName() { - return this.tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public String getTableLocation() { - return this.tableLocation; - } - - public void setTableLocation(String tableLocation) { - this.tableLocation = tableLocation; - } - - public String getUser() { - return this.user; - } - - public void setUser(String user) { - this.user = user; - } - - public String getQueryStartTime() { - return this.queryStartTime; - } - - public void setQueryStartTime(String queryStartTime) { - this.queryStartTime = queryStartTime; - } - - public String getQueryEndTime() { - return this.queryEndTime; - } - - public void setQueryEndTime(String queryEndTime) { - this.queryEndTime = queryEndTime; - } - - public String getQuery() { - return this.query; - } - - public void setQuery(String query) { - this.query = query; - } - - public ArrayList<SourceTables> getSourceTables() { - return this.sourceTables; - } - - public void setSourceTables(ArrayList<SourceTables> sourceTables) { - this.sourceTables = sourceTables; - } - - public ArrayList<QueryColumns> getQueryColumns() { - return this.queryColumns; - } - - public void setQueryColumns(ArrayList<QueryColumns> queryColumns) { - this.queryColumns = queryColumns; - } - - - public ArrayList<WhereClause> getWhereClause() { - return this.whereClause; - } - - public void setWhereClause(ArrayList<WhereClause> whereClause) { - this.whereClause = whereClause; - } - - - public ArrayList<GroupBy> getGroupBy() { - return this.groupBy; - } - - public void setGroupBy(ArrayList<GroupBy> groupBy) { - this.groupBy = groupBy; - } - - public ArrayList<CreateColumns> getCreateColumns() { - return this.createColumns; - } - - public void setCreateColumns(ArrayList<CreateColumns> createColumns) { - this.createColumns = createColumns; - } - - public class SourceTables { - public String tableName; - public String tableAlias; - public String databaseName; - - public String getTableName() { - return this.tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public String getTableAlias() { - return this.tableAlias; - } - - public void setTableAlias(String tableAlias) { - this.tableAlias = tableAlias; - } - - - public String getDatabaseName() { - return this.databaseName; - } - - public void setDatabaseName(String databaseName) { - this.databaseName = databaseName; - } - } - - public class QueryColumns { - public String tbAliasOrName; - public String columnName; - public String columnAlias; - public String columnFunction; - - - public String getTbAliasOrName() { - return this.tbAliasOrName; - } - - public void setTbAliasOrName(String tbAliasOrName) { - this.tbAliasOrName = tbAliasOrName; - } - - public String getColumnName() { - return this.columnName; - } - - public void setColumnName(String columnName) { - this.columnName = columnName; - } - - public String getColumnAlias() { - return this.columnAlias; - } - - public void setColumnAlias(String columnAlias) { - this.columnAlias = columnAlias; - } - - - public String getColumnFunction() { - return this.columnFunction; - } - - public void setColumnFunction(String columnFunction) { - this.columnFunction = columnFunction; - } - } - - public class GroupBy { - public String tbAliasOrName; - public String columnName; - - public String getTbAliasOrName() { - return this.tbAliasOrName; - } - - public void setTbAliasOrName(String tbAliasOrName) { - this.tbAliasOrName = tbAliasOrName; - } - - public String getColumnName() { - return this.columnName; - } - - public void setColumnName(String columnName) { - this.columnName = columnName; - } - } - - public class WhereClause { - public String tbAliasOrName; - public String columnCondition; - public String columnName; - public String columnOperator; - public String columnValue; - - - public String getColumnCondition() { - return this.columnCondition; - } - - public void setColumnCondition(String columnCondition) { - this.columnCondition = columnCondition; - } - - public String getTbAliasOrName() { - return this.tbAliasOrName; - } - - public void setTbAliasOrName(String tbAliasOrName) { - this.tbAliasOrName = tbAliasOrName; - } - - public String getColumnName() { - return this.columnName; - } - - public void setColumnName(String columnName) { - this.columnName = columnName; - } - - public String getColumnOperator() { - return this.columnOperator; - } - - public void setColumnOperator(String columnOperator) { - this.columnOperator = columnOperator; - } - - - public String getColumnValue() { - return this.columnValue; - } - - public void setColumnValue(String columnValue) { - this.columnValue = columnValue; - } - } - - public class CreateColumns { - public String columnName; - public String columnType; - - public String getColumnName() { - return this.columnName; - } - - public void setColumnName(String columnName) { - this.columnName = columnName; - } - - public String getColumnType() { - return this.columnType; - } - - public void setColumnType(String columnType) { - this.columnType = columnType; - } - } - - -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveMetaImporter.java ---------------------------------------------------------------------- diff --git a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveMetaImporter.java b/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveMetaImporter.java deleted file mode 100755 index e85014d..0000000 --- a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveMetaImporter.java +++ /dev/null @@ -1,215 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.bridge.hivestructure; - -import org.apache.atlas.MetadataException; -import org.apache.atlas.repository.IRepository; -import org.apache.atlas.repository.RepositoryException; -import org.apache.atlas.typesystem.Referenceable; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.TypeSystem; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.UnknownDBException; -import org.apache.hadoop.hive.metastore.api.UnknownTableException; -import org.apache.thrift.TException; -/* - * Initial pass at one time importer TODO - needs re-write - */ - - -public class HiveMetaImporter { - - private static HiveMetaStoreClient msc; - private static IRepository repo; - - public HiveMetaImporter(IRepository repo) { - - try { - this.repo = repo; - msc = new HiveMetaStoreClient(new HiveConf()); - // TODO Get hive-site.conf from class path first - } catch (MetaException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - } - - public static boolean fullImport() { - try { - databasesImport(); - for (String dbName : msc.getAllDatabases()) { - tablesImport(dbName); - for (String tbName : msc.getAllTables(dbName)) { - fieldsImport(dbName, tbName); - } - return true; - } - } catch (MetaException me) { - me.printStackTrace(); - } catch (RepositoryException re) { - re.printStackTrace(); - } - - return false; - } - - public static boolean databasesImport() throws MetaException, RepositoryException { - ClassType classType = null; - try { - classType = TypeSystem.getInstance().getDataType(ClassType.class, HiveStructureBridge.DB_CLASS_TYPE); - } catch (MetadataException e1) { - e1.printStackTrace(); - } - for (String dbName : msc.getAllDatabases()) { - databaseImport(dbName); - } - return true; - } - - public static boolean databaseImport(String dbName) throws MetaException, RepositoryException { - try { - Database db = msc.getDatabase(dbName); - Referenceable dbRef = new Referenceable(HiveStructureBridge.DB_CLASS_TYPE); - dbRef.set("DESC", db.getDescription()); - dbRef.set("DB_LOCATION_URI", db.getLocationUri()); - dbRef.set("NAME", db.getName()); - if (db.isSetOwnerType()) { - dbRef.set("OWNER_TYPE", db.getOwnerType()); - } - if (db.isSetOwnerName()) { - dbRef.set("OWNER_NAME", db.getOwnerName()); - } - - repo.create(dbRef); - } catch (NoSuchObjectException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (TException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return true; - } - - public static boolean tablesImport(String dbName) throws MetaException, RepositoryException { - ClassType classType = null; - try { - classType = TypeSystem.getInstance().getDataType(ClassType.class, HiveStructureBridge.TB_CLASS_TYPE); - } catch (MetadataException e1) { - e1.printStackTrace(); - } - for (String tbName : msc.getAllTables(dbName)) { - tableImport(dbName, tbName); - } - return true; - } - - public static boolean tableImport(String dbName, String tbName) throws MetaException, RepositoryException { - try { - Table tb = msc.getTable(dbName, tbName); - Referenceable tbRef = new Referenceable(HiveStructureBridge.TB_CLASS_TYPE); - tbRef.set("CREATE_TIME", tb.getCreateTime()); - tbRef.set("LAST_ACCESS_TIME", tb.getLastAccessTime()); - tbRef.set("OWNER", tb.getOwner()); - tbRef.set("TBL_NAME", tb.getTableName()); - tbRef.set("TBL_TYPE", tb.getTableType()); - if (tb.isSetViewExpandedText()) { - tbRef.set("VIEW_EXPANDED_TEXT", tb.getViewExpandedText()); - } - if (tb.isSetViewOriginalText()) { - tbRef.set("VIEW_ORIGINAL_TEXT", tb.getViewOriginalText()); - } - - repo.create(tbRef); - } catch (NoSuchObjectException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (TException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return true; - } - - public static boolean fieldsImport(String dbName, String tbName) throws MetaException, RepositoryException { - ClassType classType = null; - try { - classType = TypeSystem.getInstance().getDataType(ClassType.class, HiveStructureBridge.FD_CLASS_TYPE); - } catch (MetadataException e1) { - e1.printStackTrace(); - } - try { - for (FieldSchema fs : msc.getFields(dbName, tbName)) { - Referenceable fdRef = new Referenceable(HiveStructureBridge.FD_CLASS_TYPE); - if (fs.isSetComment()) { - fdRef.set("COMMENT", fs.getName()); - } - fdRef.set("COLUMN_NAME", fs.getName()); - fdRef.set("TYPE_NAME", fs.getType()); - - repo.create(fdRef); - } - } catch (UnknownTableException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (UnknownDBException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (TException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return true; - } - - public static boolean fieldImport(String dbName, String tbName, String fdName) throws MetaException { - try { - for (FieldSchema fs : msc.getFields(dbName, tbName)) { - if (fs.getName().equals(fs)) { - Referenceable fdRef = new Referenceable(HiveStructureBridge.TB_CLASS_TYPE); - if (fs.isSetComment()) { - fdRef.set("COMMENT", fs.getName()); - } - fdRef.set("COLUMN_NAME", fs.getName()); - fdRef.set("TYPE_NAME", fs.getType()); - //SaveObject to MS Backend - return true; - } - } - } catch (UnknownTableException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (UnknownDBException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (TException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return true; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/054dfb76/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveStructureBridge.java ---------------------------------------------------------------------- diff --git a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveStructureBridge.java b/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveStructureBridge.java deleted file mode 100755 index e9fb143..0000000 --- a/addons/metadata-bridge-parent/metadata-bridge-core/src/main/java/org/apache/hadoop/metadata/bridge/hivestructure/HiveStructureBridge.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.bridge.hivestructure; - -import org.apache.atlas.MetadataException; -import org.apache.atlas.bridge.ABridge; -import org.apache.atlas.repository.MetadataRepository; -import org.apache.atlas.typesystem.types.AttributeDefinition; -import org.apache.atlas.typesystem.types.ClassType; -import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; -import org.apache.atlas.typesystem.types.Multiplicity; -import org.apache.atlas.typesystem.types.TypeSystem; - -import javax.inject.Inject; -import java.util.ArrayList; - - -public class HiveStructureBridge extends ABridge { - - - static final String DB_CLASS_TYPE = "HiveDatabase"; - static final String TB_CLASS_TYPE = "HiveTable"; - static final String FD_CLASS_TYPE = "HiveField"; - - @Inject - protected HiveStructureBridge(MetadataRepository repo) { - super(repo); - // TODO Auto-generated constructor stub - } - - public boolean defineBridgeTypes(TypeSystem ts) { - ArrayList<HierarchicalTypeDefinition<?>> al = new ArrayList<HierarchicalTypeDefinition<?>>(); - // TODO - //convert to helper methods - // Add to arrayList - - try { - HierarchicalTypeDefinition<ClassType> databaseClassTypeDef = - new HierarchicalTypeDefinition<ClassType>("ClassType", DB_CLASS_TYPE, null, - new AttributeDefinition[]{ - new AttributeDefinition("DESC", "STRING_TYPE", Multiplicity.OPTIONAL, false, null), - new AttributeDefinition("DB_LOCATION_URI", "STRING_TYPE", Multiplicity.REQUIRED, - false, null), - new AttributeDefinition("NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, null), - new AttributeDefinition("OWNER_TYPE", "STRING_TYPE", Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition("OWNER_NAME", "STRING_TYPE", Multiplicity.OPTIONAL, false, - null)}); - - HierarchicalTypeDefinition<ClassType> tableClassTypeDef = - new HierarchicalTypeDefinition<ClassType>("ClassType", TB_CLASS_TYPE, null, - new AttributeDefinition[]{ - new AttributeDefinition("CREATE_TIME", "LONG_TYPE", Multiplicity.REQUIRED, false, - null), - new AttributeDefinition("LAST_ACCESS_TIME", "LONG_TYPE", Multiplicity.REQUIRED, - false, null), - new AttributeDefinition("OWNER", "STRING_TYPE", Multiplicity.REQUIRED, false, null), - new AttributeDefinition("TBL_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, - null), - new AttributeDefinition("TBL_TYPE", "STRING_TYPE", Multiplicity.REQUIRED, false, - null), - new AttributeDefinition("VIEW_EXPANDED_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL, - false, null), - new AttributeDefinition("VIEW_ORIGINAL_TEXT", "STRING_TYPE", Multiplicity.OPTIONAL, - false, null)}); - - HierarchicalTypeDefinition<ClassType> columnClassTypeDef = - new HierarchicalTypeDefinition<ClassType>("ClassType", FD_CLASS_TYPE, null, - new AttributeDefinition[]{ - new AttributeDefinition("COMMENT", "STRING_TYPE", Multiplicity.OPTIONAL, false, - null), - new AttributeDefinition("COLUMN_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, - null), - new AttributeDefinition("TYPE_NAME", "STRING_TYPE", Multiplicity.REQUIRED, false, - null)}); - - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - - for (HierarchicalTypeDefinition htd : al) { - try { - ts.defineClassType(htd); - } catch (MetadataException e) { - System.out.println(htd.hierarchicalMetaTypeName + "could not be added to the type system"); - e.printStackTrace(); - } - } - - - return false; - } - -}
