http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/main/java/org/apache/atlas/web/service/ServiceState.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/service/ServiceState.java b/webapp/src/main/java/org/apache/atlas/web/service/ServiceState.java index 2d9e00a..3fe8d18 100644 --- a/webapp/src/main/java/org/apache/atlas/web/service/ServiceState.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/ServiceState.java @@ -19,13 +19,15 @@ package org.apache.atlas.web.service; import com.google.common.base.Preconditions; -import com.google.inject.Singleton; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; import org.apache.atlas.ha.HAConfiguration; import org.apache.commons.configuration.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.inject.Singleton; /** * A class that maintains the state of this instance. @@ -34,6 +36,7 @@ import org.slf4j.LoggerFactory; * directed by {@link ActiveInstanceElectorService}. */ @Singleton +@Component public class ServiceState { private static final Logger LOG = LoggerFactory.getLogger(ServiceState.class);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/main/java/org/apache/atlas/web/service/UserService.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/service/UserService.java b/webapp/src/main/java/org/apache/atlas/web/service/UserService.java index 6e5c210..24fd7cf 100644 --- a/webapp/src/main/java/org/apache/atlas/web/service/UserService.java +++ b/webapp/src/main/java/org/apache/atlas/web/service/UserService.java @@ -17,18 +17,23 @@ package org.apache.atlas.web.service; -import org.springframework.beans.factory.annotation.Autowired; +import org.apache.atlas.web.dao.UserDao; +import org.apache.atlas.web.model.User; import org.springframework.security.core.userdetails.UserDetailsService; import org.springframework.security.core.userdetails.UsernameNotFoundException; import org.springframework.stereotype.Service; -import org.apache.atlas.web.dao.UserDao; -import org.apache.atlas.web.model.User; + +import javax.inject.Inject; @Service public class UserService implements UserDetailsService { - @Autowired - private UserDao userDao; + private final UserDao userDao; + + @Inject + public UserService(UserDao userDao) { + this.userDao = userDao; + } @Override public User loadUserByUsername(final String username) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/main/java/org/apache/atlas/web/setup/AtlasSetup.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/setup/AtlasSetup.java b/webapp/src/main/java/org/apache/atlas/web/setup/AtlasSetup.java deleted file mode 100644 index 41eccd1..0000000 --- a/webapp/src/main/java/org/apache/atlas/web/setup/AtlasSetup.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.setup; - -import com.google.inject.Guice; -import com.google.inject.Injector; -import org.apache.atlas.ApplicationProperties; -import org.apache.atlas.AtlasException; -import org.apache.atlas.setup.SetupException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * An application that is used to setup dependencies for the Atlas web service. - * - * This should be executed immediately after installation with the same configuration - * as the Atlas web service itself. The application runs all steps registered with {@link SetupSteps}. - */ -public class AtlasSetup { - - private static final Logger LOG = LoggerFactory.getLogger(AtlasSetup.class); - - private final Injector injector; - - public AtlasSetup() { - injector = Guice.createInjector(new AtlasSetupModule()); - LOG.info("Got injector: {}", injector); - } - - public static void main(String[] args) { - try { - AtlasSetup atlasSetup = new AtlasSetup(); - atlasSetup.run(); - LOG.info("Finished running all setup steps."); - } catch (SetupException e) { - LOG.error("Could not run setup step.", e); - } - } - - public void run() throws SetupException { - SetupSteps setupSteps = injector.getInstance(SetupSteps.class); - LOG.info("Got setup steps."); - try { - setupSteps.runSetup(ApplicationProperties.get()); - } catch (AtlasException e) { - throw new SetupException("Cannot get application properties.", e); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/main/java/org/apache/atlas/web/setup/AtlasSetupModule.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/setup/AtlasSetupModule.java b/webapp/src/main/java/org/apache/atlas/web/setup/AtlasSetupModule.java deleted file mode 100644 index 6edfb12..0000000 --- a/webapp/src/main/java/org/apache/atlas/web/setup/AtlasSetupModule.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.atlas.web.setup; - -import com.google.inject.AbstractModule; -import com.google.inject.multibindings.Multibinder; -import org.apache.atlas.repository.graph.GraphSchemaInitializer; -import org.apache.atlas.setup.SetupStep; - -public class AtlasSetupModule extends AbstractModule { - @Override - protected void configure() { - Multibinder<SetupStep> setupStepMultibinder = Multibinder.newSetBinder(binder(), SetupStep.class); - setupStepMultibinder.addBinding().to(GraphSchemaInitializer.class); - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/main/java/org/apache/atlas/web/setup/SetupSteps.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/setup/SetupSteps.java b/webapp/src/main/java/org/apache/atlas/web/setup/SetupSteps.java index eadd5ce..cfb49b6 100644 --- a/webapp/src/main/java/org/apache/atlas/web/setup/SetupSteps.java +++ b/webapp/src/main/java/org/apache/atlas/web/setup/SetupSteps.java @@ -19,8 +19,7 @@ package org.apache.atlas.web.setup; import com.google.common.base.Charsets; -import com.google.inject.Inject; -import com.google.inject.Singleton; +import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasConstants; import org.apache.atlas.AtlasException; import org.apache.atlas.ha.AtlasServerIdSelector; @@ -37,32 +36,44 @@ import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.context.annotation.Conditional; +import org.springframework.core.type.AnnotatedTypeMetadata; +import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import javax.inject.Singleton; import java.util.Arrays; import java.util.List; import java.util.Set; @Singleton +@Component +@Conditional(SetupSteps.SetupRequired.class) public class SetupSteps { private static final Logger LOG = LoggerFactory.getLogger(SetupSteps.class); public static final String SETUP_IN_PROGRESS_NODE = "/setup_in_progress"; private final Set<SetupStep> setupSteps; + private final Configuration configuration; private CuratorFactory curatorFactory; @Inject - public SetupSteps(Set<SetupStep> steps, CuratorFactory curatorFactory) { + public SetupSteps(Set<SetupStep> steps, CuratorFactory curatorFactory, Configuration configuration) { setupSteps = steps; this.curatorFactory = curatorFactory; + this.configuration = configuration; } /** * Call each registered {@link SetupStep} one after the other. * @throws SetupException Thrown with any error during running setup, including Zookeeper interactions, and * individual failures in the {@link SetupStep}. - * @param configuration Configuration for Atlas server. */ - public void runSetup(Configuration configuration) throws SetupException { + @PostConstruct + public void runSetup() throws SetupException { HAConfiguration.ZookeeperProperties zookeeperProperties = HAConfiguration.getZookeeperProperties(configuration); InterProcessMutex lock = curatorFactory.lockInstance(zookeeperProperties.getZkRoot()); try { @@ -162,4 +173,25 @@ public class SetupSteps { throw new SetupException("Could not create lock node before running setup.", e); } } + + static class SetupRequired implements Condition { + private static final String ATLAS_SERVER_RUN_SETUP_KEY = "atlas.server.run.setup.on.start"; + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + try { + Configuration configuration = ApplicationProperties.get(); + boolean shouldRunSetup = configuration.getBoolean(ATLAS_SERVER_RUN_SETUP_KEY, false); + if (shouldRunSetup) { + LOG.warn("Running setup per configuration {}.", ATLAS_SERVER_RUN_SETUP_KEY); + return true; + } else { + LOG.info("Not running setup per configuration {}.", ATLAS_SERVER_RUN_SETUP_KEY); + } + } catch (AtlasException e) { + LOG.error("Unable to read config to determine if setup is needed. Not running setup."); + } + return false; + } + } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/main/resources/spring-security.xml ---------------------------------------------------------------------- diff --git a/webapp/src/main/resources/spring-security.xml b/webapp/src/main/resources/spring-security.xml index 208c325..1743218 100644 --- a/webapp/src/main/resources/spring-security.xml +++ b/webapp/src/main/resources/spring-security.xml @@ -1,30 +1,23 @@ <?xml version="1.0" encoding="UTF-8"?> -<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor - license agreements. See the NOTICE file distributed with this work for additional - information regarding copyright ownership. The ASF licenses this file to - You under the Apache License, Version 2.0 (the "License"); you may not use - this file except in compliance with the License. You may obtain a copy of - the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required - by applicable law or agreed to in writing, software distributed under the - License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS - OF ANY KIND, either express or implied. See the License for the specific +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> <beans:beans xmlns="http://www.springframework.org/schema/security"; xmlns:beans="http://www.springframework.org/schema/beans"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xmlns:security="http://www.springframework.org/schema/security"; - xmlns:context="http://www.springframework.org/schema/context"; - xsi:schemaLocation="http://www.springframework.org/schema/beans - http://www.springframework.org/schema/beans/spring-beans-3.1.xsd - http://www.springframework.org/schema/security - http://www.springframework.org/schema/security/spring-security-3.1.xsd - - - + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd + http://www.springframework.org/schema/security http://www.springframework.org/schema/security/spring-security.xsd";> - http://www.springframework.org/schema/context - http://www.springframework.org/schema/context/spring-context-3.1.xsd";> + <!-- This XML is no longer being used, @see AtlasSecurityConfig for the equivalent java config --> <security:http pattern="/login.jsp" security="none" /> <security:http pattern="/css/**" security="none" /> @@ -35,8 +28,7 @@ <security:http pattern="/api/atlas/admin/status" security="none" /> <security:http pattern="/api/atlas/admin/metrics" security="none" /> - <security:http disable-url-rewriting="true" - use-expressions="true" create-session="always" + <security:http create-session="always" entry-point-ref="entryPoint"> <security:session-management session-fixation-protection="newSession" /> @@ -48,6 +40,7 @@ <form-login login-page="/login.jsp" + login-processing-url="/j_spring_security_check" authentication-success-handler-ref="atlasAuthenticationSuccessHandler" authentication-failure-handler-ref="atlasAuthenticationFailureHandler" username-parameter="j_username" @@ -56,6 +49,8 @@ <security:logout logout-success-url="/login.jsp" delete-cookies="ATLASSESSIONID" logout-url="/logout.html" /> <http-basic /> + <headers disabled="true"/> + <csrf disabled="true"/> <security:custom-filter position="LAST" ref="atlasAuthorizationFilter"/> </security:http> @@ -76,7 +71,7 @@ <beans:bean id="formAuthenticationEntryPoint" class="org.apache.atlas.web.filters.AtlasAuthenticationEntryPoint"> - <beans:property name="loginFormUrl" value="/login.jsp" /> + <beans:constructor-arg value="/login.jsp"/> </beans:bean> <beans:bean id="authenticationEntryPoint" @@ -108,7 +103,5 @@ <security:global-method-security pre-post-annotations="enabled" /> - <context:component-scan base-package="org.apache.atlas.web" /> - <beans:bean id = "atlasAuthorizationFilter" class="org.apache.atlas.web.filters.AtlasAuthorizationFilter"/> -</beans:beans> +</beans:beans> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/main/webapp/WEB-INF/applicationContext.xml ---------------------------------------------------------------------- diff --git a/webapp/src/main/webapp/WEB-INF/applicationContext.xml b/webapp/src/main/webapp/WEB-INF/applicationContext.xml index d4ad14e..aae2aa0 100644 --- a/webapp/src/main/webapp/WEB-INF/applicationContext.xml +++ b/webapp/src/main/webapp/WEB-INF/applicationContext.xml @@ -12,9 +12,12 @@ <beans xmlns="http://www.springframework.org/schema/beans"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xmlns:context="http://www.springframework.org/schema/context"; + xmlns:aop="http://www.springframework.org/schema/aop"; xsi:schemaLocation="http://www.springframework.org/schema/beans - http://www.springframework.org/schema/beans/spring-beans-3.1.xsd";> - - <import resource="classpath:/spring-security.xml" /> + http://www.springframework.org/schema/beans/spring-beans-3.1.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd";> + <context:annotation-config/> + <aop:config proxy-target-class="true"/> + <context:component-scan base-package="org.apache.atlas" /> </beans> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/main/webapp/WEB-INF/web.xml ---------------------------------------------------------------------- diff --git a/webapp/src/main/webapp/WEB-INF/web.xml b/webapp/src/main/webapp/WEB-INF/web.xml index f7e2028..d2d08f5 100755 --- a/webapp/src/main/webapp/WEB-INF/web.xml +++ b/webapp/src/main/webapp/WEB-INF/web.xml @@ -24,51 +24,46 @@ <display-name>Apache Atlas</display-name> <description>Metadata Management and Data Governance Platform over Hadoop</description> - <context-param> - <param-name>guice.packages</param-name> - <param-value> - org.apache.atlas.web.resources,org.apache.atlas.web.params,org.apache.atlas.web.rest,org.apache.atlas.web.errors - </param-value> - </context-param> - - <context-param> - <param-name>com.sun.jersey.api.json.POJOMappingFeature</param-name> - <param-value>true</param-value> - </context-param> + <servlet> + <servlet-name>jersey-servlet</servlet-name> + <servlet-class> + com.sun.jersey.spi.spring.container.servlet.SpringServlet + </servlet-class> + <init-param> + <param-name>com.sun.jersey.api.json.POJOMappingFeature</param-name> + <param-value>true</param-value> + </init-param> + <load-on-startup>1</load-on-startup> + </servlet> - <!-- - More information can be found here: - - https://jersey.java.net/nonav/apidocs/1.11/contribs/jersey-guice/com/sun/jersey/guice/spi/container/servlet/package-summary.html - --> + <servlet-mapping> + <servlet-name>jersey-servlet</servlet-name> + <url-pattern>/api/atlas/*</url-pattern> + </servlet-mapping> <filter> <filter-name>springSecurityFilterChain</filter-name> <filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class> </filter> - - <filter> - <filter-name>guiceFilter</filter-name> - <filter-class>com.google.inject.servlet.GuiceFilter</filter-class> - </filter> - + <filter-mapping> <filter-name>springSecurityFilterChain</filter-name> <url-pattern>/*</url-pattern> </filter-mapping> - + + <filter> + <filter-name>AuditFilter</filter-name> + <filter-class>org.apache.atlas.web.filters.AuditFilter</filter-class> + </filter> + <filter-mapping> - <filter-name>guiceFilter</filter-name> + <filter-name>AuditFilter</filter-name> <url-pattern>/*</url-pattern> </filter-mapping> <listener> <listener-class>org.springframework.web.util.Log4jConfigListener</listener-class> </listener> - - <listener> - <listener-class>org.apache.atlas.web.listeners.GuiceServletConfig</listener-class> - </listener> <listener> <listener-class>org.springframework.web.context.request.RequestContextListener</listener-class> @@ -88,5 +83,5 @@ </cookie-config> </session-config> - + </web-app> http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/test/java/org/apache/atlas/examples/QuickStartIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/examples/QuickStartIT.java b/webapp/src/test/java/org/apache/atlas/examples/QuickStartIT.java index 06c78be..592c2a6 100644 --- a/webapp/src/test/java/org/apache/atlas/examples/QuickStartIT.java +++ b/webapp/src/test/java/org/apache/atlas/examples/QuickStartIT.java @@ -22,7 +22,7 @@ import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasServiceException; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.web.resources.BaseResourceIT; +import org.apache.atlas.web.integration.BaseResourceIT; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java b/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java index 7f3192a..773a514 100644 --- a/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java +++ b/webapp/src/test/java/org/apache/atlas/examples/QuickStartV2IT.java @@ -26,9 +26,8 @@ import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.lineage.AtlasLineageInfo; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection; import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation; -import org.apache.atlas.web.resources.BaseResourceIT; +import org.apache.atlas.web.integration.BaseResourceIT; import org.codehaus.jettison.json.JSONException; -import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java b/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java index 29be942..ac3b538 100644 --- a/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java +++ b/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java @@ -19,8 +19,8 @@ package org.apache.atlas.notification; import com.google.common.collect.ImmutableSet; -import com.google.inject.Inject; import org.apache.atlas.AtlasClient; +import org.apache.atlas.kafka.NotificationProvider; import org.apache.atlas.notification.entity.EntityNotification; import org.apache.atlas.typesystem.IReferenceableInstance; import org.apache.atlas.typesystem.IStruct; @@ -32,9 +32,8 @@ import org.apache.atlas.typesystem.persistence.Id; import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; import org.apache.atlas.typesystem.types.TraitType; import org.apache.atlas.typesystem.types.utils.TypesUtil; -import org.apache.atlas.web.resources.BaseResourceIT; +import org.apache.atlas.web.integration.BaseResourceIT; import org.testng.annotations.BeforeClass; -import org.testng.annotations.Guice; import org.testng.annotations.Test; import java.util.Collections; @@ -48,13 +47,11 @@ import static org.testng.Assert.assertTrue; /** * Entity Notification Integration Tests. */ -@Guice(modules = NotificationModule.class) public class EntityNotificationIT extends BaseResourceIT { private final String DATABASE_NAME = "db" + randomString(); private final String TABLE_NAME = "table" + randomString(); - @Inject - private NotificationInterface notificationInterface; + private NotificationInterface notificationInterface = NotificationProvider.get(); private Id tableId; private Id dbId; private String traitName; http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java index 1c2cdc6..9c5597e 100644 --- a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java +++ b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java @@ -18,23 +18,21 @@ package org.apache.atlas.notification; -import com.google.inject.Inject; import org.apache.atlas.EntityAuditEvent; +import org.apache.atlas.kafka.NotificationProvider; import org.apache.atlas.notification.hook.HookNotification; import org.apache.atlas.typesystem.Referenceable; import org.apache.atlas.typesystem.persistence.Id; -import org.apache.atlas.web.resources.BaseResourceIT; +import org.apache.atlas.web.integration.BaseResourceIT; import org.codehaus.jettison.json.JSONArray; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; -import org.testng.annotations.Guice; import org.testng.annotations.Test; import java.util.List; import static org.testng.Assert.assertEquals; -@Guice(modules = NotificationModule.class) public class NotificationHookConsumerIT extends BaseResourceIT { private static final String TEST_USER = "testuser"; @@ -43,8 +41,7 @@ public class NotificationHookConsumerIT extends BaseResourceIT { public static final String QUALIFIED_NAME = "qualifiedName"; public static final String CLUSTER_NAME = "clusterName"; - @Inject - private NotificationInterface kafka; + private NotificationInterface notificationInterface = NotificationProvider.get(); @BeforeClass public void setUp() throws Exception { @@ -54,11 +51,11 @@ public class NotificationHookConsumerIT extends BaseResourceIT { @AfterClass public void teardown() throws Exception { - kafka.close(); + notificationInterface.close(); } private void sendHookMessage(HookNotification.HookNotificationMessage message) throws NotificationException { - kafka.send(NotificationInterface.NotificationType.HOOK, message); + notificationInterface.send(NotificationInterface.NotificationType.HOOK, message); } @Test http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java index e744e2e..18fd2ee 100644 --- a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java +++ b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java @@ -18,12 +18,12 @@ package org.apache.atlas.notification; -import com.google.inject.Inject; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasException; import org.apache.atlas.AtlasServiceException; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.kafka.KafkaNotification; +import org.apache.atlas.kafka.NotificationProvider; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.notification.hook.HookNotification; import org.apache.atlas.repository.converters.AtlasInstanceConverter; @@ -39,7 +39,6 @@ import org.mockito.MockitoAnnotations; import org.testng.Assert; import org.testng.annotations.AfterTest; import org.testng.annotations.BeforeTest; -import org.testng.annotations.Guice; import org.testng.annotations.Test; import static org.mockito.Matchers.any; @@ -47,14 +46,12 @@ import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.*; -@Guice(modules = NotificationModule.class) public class NotificationHookConsumerKafkaTest { public static final String NAME = "name"; public static final String DESCRIPTION = "description"; public static final String QUALIFIED_NAME = "qualifiedName"; - @Inject - private NotificationInterface notificationInterface; + private NotificationInterface notificationInterface = NotificationProvider.get(); @Mock http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java index 9272203..e753881 100644 --- a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java +++ b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java @@ -20,7 +20,7 @@ package org.apache.atlas.web.adapters; import org.apache.atlas.AtlasClient; import org.apache.atlas.RequestContext; import org.apache.atlas.RequestContextV1; -import org.apache.atlas.TestOnlyModule; +import org.apache.atlas.TestModules; import org.apache.atlas.TestUtilsV2; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; @@ -32,7 +32,6 @@ import org.apache.atlas.model.instance.ClassificationAssociateRequest; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasType; @@ -42,7 +41,6 @@ import org.apache.atlas.web.rest.EntityREST; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; -import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeClass; import org.testng.annotations.Guice; @@ -57,7 +55,7 @@ import java.util.List; import java.util.Map; -@Guice(modules = {TestOnlyModule.class}) +@Guice(modules = {TestModules.TestOnlyModule.class}) public class TestEntitiesREST { private static final Logger LOG = LoggerFactory.getLogger(TestEntitiesREST.class); @@ -105,11 +103,6 @@ public class TestEntitiesREST { RequestContextV1.clear(); } - @AfterClass - public void tearDown() throws Exception { - AtlasGraphProvider.cleanup(); - } - @Test public void testCreateOrUpdateEntities() throws Exception { AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java index cadf0ff..b90ea64 100644 --- a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java +++ b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntityREST.java @@ -17,9 +17,9 @@ */ package org.apache.atlas.web.adapters; +import org.apache.atlas.TestModules; import org.apache.atlas.RequestContext; import org.apache.atlas.RequestContextV1; -import org.apache.atlas.TestOnlyModule; import org.apache.atlas.TestUtilsV2; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasClassification.AtlasClassifications; @@ -30,7 +30,6 @@ import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.model.typedef.AtlasTypesDef; -import org.apache.atlas.repository.graph.AtlasGraphProvider; import org.apache.atlas.store.AtlasTypeDefStore; import org.apache.atlas.type.AtlasTypeUtil; import org.apache.atlas.web.rest.EntityREST; @@ -50,7 +49,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -@Guice(modules = {TestOnlyModule.class}) +@Guice(modules = {TestModules.TestOnlyModule.class}) public class TestEntityREST { @Inject @@ -73,7 +72,7 @@ public class TestEntityREST { @AfterClass public void tearDown() throws Exception { - AtlasGraphProvider.cleanup(); +// AtlasGraphProvider.cleanup(); } @AfterMethod http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/test/java/org/apache/atlas/web/integration/AdminJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/integration/AdminJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/integration/AdminJerseyResourceIT.java new file mode 100755 index 0000000..cfe09d6 --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/integration/AdminJerseyResourceIT.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.integration; + +import org.apache.atlas.AtlasClient; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.codehaus.jettison.json.JSONObject; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Integration test for Admin jersey resource. + */ +public class AdminJerseyResourceIT extends BaseResourceIT { + + @BeforeClass + public void setUp() throws Exception { + super.setUp(); + } + + @Test + public void testGetVersion() throws Exception { + JSONObject response = atlasClientV1.callAPIWithBodyAndParams(AtlasClient.API.VERSION, null, (String[]) null); + Assert.assertNotNull(response); + + PropertiesConfiguration buildConfiguration = new PropertiesConfiguration("atlas-buildinfo.properties"); + + Assert.assertEquals(response.get("Version"), buildConfiguration.getString("build.version")); + Assert.assertEquals(response.get("Name"), buildConfiguration.getString("project.name")); + Assert.assertEquals(response.get("Description"), buildConfiguration.getString("project.description")); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java new file mode 100755 index 0000000..b59d3ee --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java @@ -0,0 +1,669 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.integration; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import kafka.consumer.ConsumerTimeoutException; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasClient; +import org.apache.atlas.AtlasClientV2; +import org.apache.atlas.AtlasServiceException; +import org.apache.atlas.model.instance.AtlasClassification; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasEntityDef; +import org.apache.atlas.model.typedef.AtlasEnumDef; +import org.apache.atlas.model.typedef.AtlasStructDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality; +import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.atlas.notification.NotificationConsumer; +import org.apache.atlas.notification.entity.EntityNotification; +import org.apache.atlas.type.AtlasTypeUtil; +import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.json.TypesSerialization; +import org.apache.atlas.typesystem.persistence.Id; +import org.apache.atlas.typesystem.types.*; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.apache.atlas.utils.AuthenticationUtil; +import org.apache.atlas.utils.ParamChecker; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.lang.RandomStringUtils; +import org.codehaus.jettison.json.JSONArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ATTRIBUTE; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_INVERSE_REF; +import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_OWNED_REF; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +/** + * Base class for integration tests. + * Sets up the web resource and has helper methods to created type and entity. + */ +public abstract class BaseResourceIT { + + public static final String ATLAS_REST_ADDRESS = "atlas.rest.address"; + public static final String NAME = "name"; + public static final String QUALIFIED_NAME = "qualifiedName"; + public static final String CLUSTER_NAME = "clusterName"; + public static final String DESCRIPTION = "description"; + + // All service clients + protected AtlasClient atlasClientV1; + protected AtlasClientV2 atlasClientV2; + + public static final Logger LOG = LoggerFactory.getLogger(BaseResourceIT.class); + protected static final int MAX_WAIT_TIME = 60000; + protected String[] atlasUrls; + + @BeforeClass + public void setUp() throws Exception { + + //set high timeouts so that tests do not fail due to read timeouts while you + //are stepping through the code in a debugger + ApplicationProperties.get().setProperty("atlas.client.readTimeoutMSecs", "100000000"); + ApplicationProperties.get().setProperty("atlas.client.connectTimeoutMSecs", "100000000"); + + + Configuration configuration = ApplicationProperties.get(); + atlasUrls = configuration.getStringArray(ATLAS_REST_ADDRESS); + + if (atlasUrls == null || atlasUrls.length == 0) { + atlasUrls = new String[] { "http://localhost:21000/"; }; + } + + if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) { + atlasClientV1 = new AtlasClient(atlasUrls, new String[]{"admin", "admin"}); + atlasClientV2 = new AtlasClientV2(atlasUrls, new String[]{"admin", "admin"}); + } else { + atlasClientV1 = new AtlasClient(atlasUrls); + atlasClientV2 = new AtlasClientV2(atlasUrls); + } + } + + protected void batchCreateTypes(AtlasTypesDef typesDef) throws AtlasServiceException { + AtlasTypesDef toCreate = new AtlasTypesDef(); + for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) { + if (atlasClientV2.typeWithNameExists(enumDef.getName())) { + LOG.warn("Type with name {} already exists. Skipping", enumDef.getName()); + } else { + toCreate.getEnumDefs().add(enumDef); + } + } + + for (AtlasStructDef structDef : typesDef.getStructDefs()) { + if (atlasClientV2.typeWithNameExists(structDef.getName())) { + LOG.warn("Type with name {} already exists. Skipping", structDef.getName()); + } else { + toCreate.getStructDefs().add(structDef); + } + } + + for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) { + if (atlasClientV2.typeWithNameExists(entityDef.getName())) { + LOG.warn("Type with name {} already exists. Skipping", entityDef.getName()); + } else { + toCreate.getEntityDefs().add(entityDef); + } + } + + for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) { + if (atlasClientV2.typeWithNameExists(classificationDef.getName())) { + LOG.warn("Type with name {} already exists. Skipping", classificationDef.getName()); + } else { + toCreate.getClassificationDefs().add(classificationDef); + } + } + + atlasClientV2.createAtlasTypeDefs(toCreate); + } + + protected void createType(AtlasTypesDef typesDef) throws AtlasServiceException { + // Since the bulk create bails out on a single failure, this has to be done as a workaround + batchCreateTypes(typesDef); + } + + protected List<String> createType(TypesDef typesDef) throws Exception { + List<EnumTypeDefinition> enumTypes = new ArrayList<>(); + List<StructTypeDefinition> structTypes = new ArrayList<>(); + List<HierarchicalTypeDefinition<TraitType>> traitTypes = new ArrayList<>(); + List<HierarchicalTypeDefinition<ClassType>> classTypes = new ArrayList<>(); + + for (EnumTypeDefinition enumTypeDefinition : typesDef.enumTypesAsJavaList()) { + if (atlasClientV2.typeWithNameExists(enumTypeDefinition.name)) { + LOG.warn("Type with name {} already exists. Skipping", enumTypeDefinition.name); + } else { + enumTypes.add(enumTypeDefinition); + } + } + for (StructTypeDefinition structTypeDefinition : typesDef.structTypesAsJavaList()) { + if (atlasClientV2.typeWithNameExists(structTypeDefinition.typeName)) { + LOG.warn("Type with name {} already exists. Skipping", structTypeDefinition.typeName); + } else { + structTypes.add(structTypeDefinition); + } + } + for (HierarchicalTypeDefinition<TraitType> hierarchicalTypeDefinition : typesDef.traitTypesAsJavaList()) { + if (atlasClientV2.typeWithNameExists(hierarchicalTypeDefinition.typeName)) { + LOG.warn("Type with name {} already exists. Skipping", hierarchicalTypeDefinition.typeName); + } else { + traitTypes.add(hierarchicalTypeDefinition); + } + } + for (HierarchicalTypeDefinition<ClassType> hierarchicalTypeDefinition : typesDef.classTypesAsJavaList()) { + if (atlasClientV2.typeWithNameExists(hierarchicalTypeDefinition.typeName)) { + LOG.warn("Type with name {} already exists. Skipping", hierarchicalTypeDefinition.typeName); + } else { + classTypes.add(hierarchicalTypeDefinition); + } + } + + TypesDef toCreate = TypesUtil.getTypesDef(ImmutableList.copyOf(enumTypes), + ImmutableList.copyOf(structTypes), + ImmutableList.copyOf(traitTypes), + ImmutableList.copyOf(classTypes)); + return atlasClientV1.createType(toCreate); + } + + protected List<String> createType(String typesAsJSON) throws Exception { + return createType(TypesSerialization.fromJson(typesAsJSON)); + } + + protected Id createInstance(Referenceable referenceable) throws Exception { + String typeName = referenceable.getTypeName(); + System.out.println("creating instance of type " + typeName); + + List<String> guids = atlasClientV1.createEntity(referenceable); + System.out.println("created instance for type " + typeName + ", guid: " + guids); + + // return the reference to created instance with guid + if (guids.size() > 0) { + return new Id(guids.get(guids.size() - 1), 0, referenceable.getTypeName()); + } + return null; + } + + protected TypesDef getTypesDef(ImmutableList<EnumTypeDefinition> enums, + ImmutableList<StructTypeDefinition> structs, + ImmutableList<HierarchicalTypeDefinition<TraitType>> traits, + ImmutableList<HierarchicalTypeDefinition<ClassType>> classes){ + enums = (enums != null) ? enums : ImmutableList + .<EnumTypeDefinition>of(); + structs = + (structs != null) ? structs : ImmutableList.<StructTypeDefinition>of(); + + traits = (traits != null) ? traits : ImmutableList + .<HierarchicalTypeDefinition<TraitType>>of(); + + classes = (classes != null) ? classes : ImmutableList + .<HierarchicalTypeDefinition<ClassType>>of(); + return TypesUtil.getTypesDef(enums, structs, traits, classes); + + } + + protected AtlasEntityHeader modifyEntity(AtlasEntity atlasEntity, boolean update) { + EntityMutationResponse entity = null; + try { + if (!update) { + entity = atlasClientV2.createEntity(new AtlasEntityWithExtInfo(atlasEntity)); + assertNotNull(entity); + assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE)); + assertTrue(entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).size() > 0); + return entity.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE).get(0); + } else { + entity = atlasClientV2.updateEntity(new AtlasEntityWithExtInfo(atlasEntity)); + assertNotNull(entity); + assertNotNull(entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE)); + assertTrue(entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).size() > 0); + return entity.getEntitiesByOperation(EntityMutations.EntityOperation.UPDATE).get(0); + } + + } catch (AtlasServiceException e) { + LOG.error("Entity {} failed", update ? "update" : "creation", entity); + } + return null; + } + + protected AtlasEntityHeader createEntity(AtlasEntity atlasEntity) { + return modifyEntity(atlasEntity, false); + } + + protected AtlasEntityHeader updateEntity(AtlasEntity atlasEntity) { + return modifyEntity(atlasEntity, true); + } + + protected static final String DATABASE_TYPE_V2 = "hive_db_v2"; + protected static final String HIVE_TABLE_TYPE_V2 = "hive_table_v2"; + protected static final String COLUMN_TYPE_V2 = "hive_column_v2"; + protected static final String HIVE_PROCESS_TYPE_V2 = "hive_process_v2"; + + protected static final String DATABASE_TYPE = "hive_db_v1"; + protected static final String HIVE_TABLE_TYPE = "hive_table_v1"; + protected static final String COLUMN_TYPE = "hive_column_v1"; + protected static final String HIVE_PROCESS_TYPE = "hive_process_v1"; + + protected static final String DATABASE_TYPE_BUILTIN = "hive_db"; + protected static final String HIVE_TABLE_TYPE_BUILTIN = "hive_table"; + protected static final String COLUMN_TYPE_BUILTIN = "hive_column"; + protected static final String HIVE_PROCESS_TYPE_BUILTIN = "hive_process"; + + protected void createTypeDefinitionsV1() throws Exception { + HierarchicalTypeDefinition<ClassType> dbClsDef = TypesUtil + .createClassTypeDef(DATABASE_TYPE, null, + TypesUtil.createUniqueRequiredAttrDef(NAME, DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef(DESCRIPTION, DataTypes.STRING_TYPE), + attrDef("locationUri", DataTypes.STRING_TYPE), + attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.INT_TYPE), + new AttributeDefinition("tables", DataTypes.arrayTypeName(HIVE_TABLE_TYPE), + Multiplicity.OPTIONAL, false, "db") + ); + + HierarchicalTypeDefinition<ClassType> columnClsDef = TypesUtil + .createClassTypeDef(COLUMN_TYPE, null, attrDef(NAME, DataTypes.STRING_TYPE), + attrDef("dataType", DataTypes.STRING_TYPE), attrDef("comment", DataTypes.STRING_TYPE)); + + StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType", + new AttributeDefinition[]{TypesUtil.createRequiredAttrDef(NAME, DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE)}); + + EnumValue values[] = {new EnumValue("MANAGED", 1), new EnumValue("EXTERNAL", 2),}; + + EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values); + + HierarchicalTypeDefinition<ClassType> tblClsDef = TypesUtil + .createClassTypeDef(HIVE_TABLE_TYPE, ImmutableSet.of("DataSet"), + attrDef("owner", DataTypes.STRING_TYPE), attrDef("createTime", DataTypes.LONG_TYPE), + attrDef("lastAccessTime", DataTypes.DATE_TYPE), + attrDef("temporary", DataTypes.BOOLEAN_TYPE), + new AttributeDefinition("db", DATABASE_TYPE, Multiplicity.OPTIONAL, true, "tables"), + new AttributeDefinition("columns", DataTypes.arrayTypeName(COLUMN_TYPE), + Multiplicity.OPTIONAL, true, null), + new AttributeDefinition("tableType", "tableType", Multiplicity.OPTIONAL, false, null), + new AttributeDefinition("serde1", "serdeType", Multiplicity.OPTIONAL, false, null), + new AttributeDefinition("serde2", "serdeType", Multiplicity.OPTIONAL, false, null)); + + HierarchicalTypeDefinition<ClassType> loadProcessClsDef = TypesUtil + .createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableSet.of("Process"), + attrDef("userName", DataTypes.STRING_TYPE), attrDef("startTime", DataTypes.INT_TYPE), + attrDef("endTime", DataTypes.LONG_TYPE), + attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), + attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), + attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), + attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)); + + HierarchicalTypeDefinition<TraitType> classificationTrait = TypesUtil + .createTraitTypeDef("classification", ImmutableSet.<String>of(), + TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition<TraitType> piiTrait = + TypesUtil.createTraitTypeDef("pii", ImmutableSet.<String>of()); + HierarchicalTypeDefinition<TraitType> phiTrait = + TypesUtil.createTraitTypeDef("phi", ImmutableSet.<String>of()); + HierarchicalTypeDefinition<TraitType> pciTrait = + TypesUtil.createTraitTypeDef("pci", ImmutableSet.<String>of()); + HierarchicalTypeDefinition<TraitType> soxTrait = + TypesUtil.createTraitTypeDef("sox", ImmutableSet.<String>of()); + HierarchicalTypeDefinition<TraitType> secTrait = + TypesUtil.createTraitTypeDef("sec", ImmutableSet.<String>of()); + HierarchicalTypeDefinition<TraitType> financeTrait = + TypesUtil.createTraitTypeDef("finance", ImmutableSet.<String>of()); + HierarchicalTypeDefinition<TraitType> factTrait = + TypesUtil.createTraitTypeDef("Fact", ImmutableSet.<String>of()); + HierarchicalTypeDefinition<TraitType> etlTrait = + TypesUtil.createTraitTypeDef("ETL", ImmutableSet.<String>of()); + HierarchicalTypeDefinition<TraitType> dimensionTrait = + TypesUtil.createTraitTypeDef("Dimension", ImmutableSet.<String>of()); + HierarchicalTypeDefinition<TraitType> metricTrait = + TypesUtil.createTraitTypeDef("Metric", ImmutableSet.<String>of()); + + createType(getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition), + ImmutableList.of(classificationTrait, piiTrait, phiTrait, pciTrait, + soxTrait, secTrait, financeTrait, factTrait, etlTrait, dimensionTrait, metricTrait), + ImmutableList.of(dbClsDef, columnClsDef, tblClsDef, loadProcessClsDef))); + } + + protected void createTypeDefinitionsV2() throws Exception { + + AtlasConstraintDef isCompositeSourceConstraint = new AtlasConstraintDef(CONSTRAINT_TYPE_OWNED_REF); + + AtlasConstraintDef isCompositeTargetConstraint = new AtlasConstraintDef(CONSTRAINT_TYPE_INVERSE_REF, + Collections.<String, Object>singletonMap(CONSTRAINT_PARAM_ATTRIBUTE, "randomTable")); + + AtlasEntityDef dbClsTypeDef = AtlasTypeUtil.createClassTypeDef( + DATABASE_TYPE_V2, + null, + AtlasTypeUtil.createUniqueRequiredAttrDef(NAME, "string"), + AtlasTypeUtil.createRequiredAttrDef(DESCRIPTION, "string"), + AtlasTypeUtil.createOptionalAttrDef("locationUri", "string"), + AtlasTypeUtil.createOptionalAttrDef("owner", "string"), + AtlasTypeUtil.createOptionalAttrDef("createTime", "int"), + AtlasTypeUtil.createOptionalAttrDef("createTime", "int"), + //there is a serializ + new AtlasAttributeDef("randomTable", + DataTypes.arrayTypeName(HIVE_TABLE_TYPE_V2), + true, + Cardinality.SET, + 0, -1, false, true, Collections.singletonList(isCompositeSourceConstraint)) + ); + + AtlasEntityDef columnClsDef = AtlasTypeUtil + .createClassTypeDef(COLUMN_TYPE_V2, null, + AtlasTypeUtil.createOptionalAttrDef(NAME, "string"), + AtlasTypeUtil.createOptionalAttrDef("dataType", "string"), + AtlasTypeUtil.createOptionalAttrDef("comment", "string")); + + AtlasStructDef structTypeDef = AtlasTypeUtil.createStructTypeDef("serdeType", + AtlasTypeUtil.createRequiredAttrDef(NAME, "string"), + AtlasTypeUtil.createRequiredAttrDef("serde", "string") + ); + + AtlasEnumDef enumDef = new AtlasEnumDef("tableType", DESCRIPTION, Arrays.asList( + new AtlasEnumDef.AtlasEnumElementDef("MANAGED", null, 1), + new AtlasEnumDef.AtlasEnumElementDef("EXTERNAL", null, 2) + )); + + AtlasEntityDef tblClsDef = AtlasTypeUtil + .createClassTypeDef(HIVE_TABLE_TYPE_V2, + ImmutableSet.of("DataSet"), + AtlasTypeUtil.createOptionalAttrDef("owner", "string"), + AtlasTypeUtil.createOptionalAttrDef("createTime", "long"), + AtlasTypeUtil.createOptionalAttrDef("lastAccessTime", "date"), + AtlasTypeUtil.createOptionalAttrDef("temporary", "boolean"), + new AtlasAttributeDef("db", + DATABASE_TYPE_V2, + true, + Cardinality.SINGLE, + 0, 1, false, true, Collections.singletonList(isCompositeTargetConstraint)), + + //some tests don't set the columns field or set it to null... + AtlasTypeUtil.createOptionalAttrDef("columns", DataTypes.arrayTypeName(COLUMN_TYPE_V2)), + AtlasTypeUtil.createOptionalAttrDef("tableType", "tableType"), + AtlasTypeUtil.createOptionalAttrDef("serde1", "serdeType"), + AtlasTypeUtil.createOptionalAttrDef("serde2", "serdeType")); + + AtlasEntityDef loadProcessClsDef = AtlasTypeUtil + .createClassTypeDef(HIVE_PROCESS_TYPE_V2, + ImmutableSet.of("Process"), + AtlasTypeUtil.createOptionalAttrDef("userName", "string"), + AtlasTypeUtil.createOptionalAttrDef("startTime", "int"), + AtlasTypeUtil.createOptionalAttrDef("endTime", "long"), + AtlasTypeUtil.createRequiredAttrDef("queryText", "string"), + AtlasTypeUtil.createRequiredAttrDef("queryPlan", "string"), + AtlasTypeUtil.createRequiredAttrDef("queryId", "string"), + AtlasTypeUtil.createRequiredAttrDef("queryGraph", "string")); + + AtlasClassificationDef classificationTrait = AtlasTypeUtil + .createTraitTypeDef("classification",ImmutableSet.<String>of(), + AtlasTypeUtil.createRequiredAttrDef("tag", "string")); + AtlasClassificationDef piiTrait = + AtlasTypeUtil.createTraitTypeDef("pii", ImmutableSet.<String>of()); + AtlasClassificationDef phiTrait = + AtlasTypeUtil.createTraitTypeDef("phi", ImmutableSet.<String>of()); + AtlasClassificationDef pciTrait = + AtlasTypeUtil.createTraitTypeDef("pci", ImmutableSet.<String>of()); + AtlasClassificationDef soxTrait = + AtlasTypeUtil.createTraitTypeDef("sox", ImmutableSet.<String>of()); + AtlasClassificationDef secTrait = + AtlasTypeUtil.createTraitTypeDef("sec", ImmutableSet.<String>of()); + AtlasClassificationDef financeTrait = + AtlasTypeUtil.createTraitTypeDef("finance", ImmutableSet.<String>of()); + + AtlasTypesDef typesDef = new AtlasTypesDef(ImmutableList.of(enumDef), + ImmutableList.of(structTypeDef), + ImmutableList.of(classificationTrait, piiTrait, phiTrait, pciTrait, soxTrait, secTrait, financeTrait), + ImmutableList.of(dbClsTypeDef, columnClsDef, tblClsDef, loadProcessClsDef)); + + batchCreateTypes(typesDef); + } + + AttributeDefinition attrDef(String name, IDataType dT) { + return attrDef(name, dT, Multiplicity.OPTIONAL, false, null); + } + + AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) { + return attrDef(name, dT, m, false, null); + } + + AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m, boolean isComposite, + String reverseAttributeName) { + Preconditions.checkNotNull(name); + Preconditions.checkNotNull(dT); + return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName); + } + + protected String randomString() { + //names cannot start with a digit + return RandomStringUtils.randomAlphabetic(1) + RandomStringUtils.randomAlphanumeric(9); + } + + protected Referenceable createHiveTableInstanceBuiltIn(String dbName, String tableName, Id dbId) throws Exception { + Map<String, Object> values = new HashMap<>(); + values.put(NAME, dbName); + values.put(DESCRIPTION, "foo database"); + values.put(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, dbName); + values.put("owner", "user1"); + values.put(CLUSTER_NAME, "cl1"); + values.put("parameters", Collections.EMPTY_MAP); + values.put("location", "/tmp"); + Referenceable databaseInstance = new Referenceable(dbId._getId(), dbId.getTypeName(), values); + Referenceable tableInstance = + new Referenceable(HIVE_TABLE_TYPE_BUILTIN, "classification", "pii", "phi", "pci", "sox", "sec", "finance"); + tableInstance.set(NAME, tableName); + tableInstance.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName); + tableInstance.set("db", databaseInstance); + tableInstance.set(DESCRIPTION, "bar table"); + tableInstance.set("lastAccessTime", "2014-07-11T08:00:00.000Z"); + tableInstance.set("type", "managed"); + tableInstance.set("level", 2); + tableInstance.set("tableType", 1); // enum + tableInstance.set("compressed", false); + + Struct traitInstance = (Struct) tableInstance.getTrait("classification"); + traitInstance.set("tag", "foundation_etl"); + + Struct serde1Instance = new Struct("serdeType"); + serde1Instance.set(NAME, "serde1"); + serde1Instance.set("serde", "serde1"); + tableInstance.set("serde1", serde1Instance); + + Struct serde2Instance = new Struct("serdeType"); + serde2Instance.set(NAME, "serde2"); + serde2Instance.set("serde", "serde2"); + tableInstance.set("serde2", serde2Instance); + + List<String> traits = tableInstance.getTraits(); + Assert.assertEquals(traits.size(), 7); + + return tableInstance; + } + + protected AtlasEntity createHiveTableInstanceV2(AtlasEntity databaseInstance, String tableName) throws Exception { + AtlasEntity tableInstance = new AtlasEntity(HIVE_TABLE_TYPE_V2); + tableInstance.setClassifications( + Arrays.asList(new AtlasClassification("classification"), + new AtlasClassification("pii"), + new AtlasClassification("phi"), + new AtlasClassification("pci"), + new AtlasClassification("sox"), + new AtlasClassification("sec"), + new AtlasClassification("finance")) + ); + + tableInstance.setAttribute(NAME, tableName); + tableInstance.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableName); + tableInstance.setAttribute("db", AtlasTypeUtil.getAtlasObjectId(databaseInstance)); + tableInstance.setAttribute(DESCRIPTION, "bar table"); + tableInstance.setAttribute("lastAccessTime", "2014-07-11T08:00:00.000Z"); + tableInstance.setAttribute("type", "managed"); + tableInstance.setAttribute("level", 2); + tableInstance.setAttribute("tableType", "MANAGED"); // enum + tableInstance.setAttribute("compressed", false); + + AtlasClassification classification = tableInstance.getClassifications().get(0); + classification.setAttribute("tag", "foundation_etl"); + + AtlasStruct serde1Instance = new AtlasStruct("serdeType"); + serde1Instance.setAttribute(NAME, "serde1"); + serde1Instance.setAttribute("serde", "serde1"); + tableInstance.setAttribute("serde1", serde1Instance); + + AtlasStruct serde2Instance = new AtlasStruct("serdeType"); + serde2Instance.setAttribute(NAME, "serde2"); + serde2Instance.setAttribute("serde", "serde2"); + tableInstance.setAttribute("serde2", serde2Instance); + + List<AtlasClassification> traits = tableInstance.getClassifications(); + Assert.assertEquals(traits.size(), 7); + + return tableInstance; + } + protected Referenceable createHiveDBInstanceBuiltIn(String dbName) { + Referenceable databaseInstance = new Referenceable(DATABASE_TYPE_BUILTIN); + databaseInstance.set(NAME, dbName); + databaseInstance.set(QUALIFIED_NAME, dbName); + databaseInstance.set(CLUSTER_NAME, randomString()); + databaseInstance.set(DESCRIPTION, "foo database"); + return databaseInstance; + } + + + protected Referenceable createHiveDBInstanceV1(String dbName) { + Referenceable databaseInstance = new Referenceable(DATABASE_TYPE); + databaseInstance.set(NAME, dbName); + databaseInstance.set(DESCRIPTION, "foo database"); + databaseInstance.set(CLUSTER_NAME, "fooCluster"); + return databaseInstance; + } + + protected AtlasEntity createHiveDBInstanceV2(String dbName) { + AtlasEntity atlasEntity = new AtlasEntity(DATABASE_TYPE_V2); + atlasEntity.setAttribute(NAME, dbName); + atlasEntity.setAttribute(DESCRIPTION, "foo database"); + atlasEntity.setAttribute(CLUSTER_NAME, "fooCluster"); + atlasEntity.setAttribute("owner", "user1"); + atlasEntity.setAttribute("locationUri", "/tmp"); + atlasEntity.setAttribute("createTime",1000); + return atlasEntity; + } + + + public interface Predicate { + + /** + * Perform a predicate evaluation. + * + * @return the boolean result of the evaluation. + * @throws Exception thrown if the predicate evaluation could not evaluate. + */ + boolean evaluate() throws Exception; + } + + public interface NotificationPredicate { + + /** + * Perform a predicate evaluation. + * + * @return the boolean result of the evaluation. + * @throws Exception thrown if the predicate evaluation could not evaluate. + */ + boolean evaluate(EntityNotification notification) throws Exception; + } + + /** + * Wait for a condition, expressed via a {@link Predicate} to become true. + * + * @param timeout maximum time in milliseconds to wait for the predicate to become true. + * @param predicate predicate waiting on. + */ + protected void waitFor(int timeout, Predicate predicate) throws Exception { + ParamChecker.notNull(predicate, "predicate"); + long mustEnd = System.currentTimeMillis() + timeout; + + boolean eval; + while (!(eval = predicate.evaluate()) && System.currentTimeMillis() < mustEnd) { + LOG.info("Waiting up to {} msec", mustEnd - System.currentTimeMillis()); + Thread.sleep(100); + } + if (!eval) { + throw new Exception("Waiting timed out after " + timeout + " msec"); + } + } + + protected EntityNotification waitForNotification(final NotificationConsumer<EntityNotification> consumer, int maxWait, + final NotificationPredicate predicate) throws Exception { + final TypeUtils.Pair<EntityNotification, String> pair = TypeUtils.Pair.of(null, null); + final long maxCurrentTime = System.currentTimeMillis() + maxWait; + waitFor(maxWait, new Predicate() { + @Override + public boolean evaluate() throws Exception { + try { + while (consumer.hasNext() && System.currentTimeMillis() < maxCurrentTime) { + EntityNotification notification = consumer.next(); + if (predicate.evaluate(notification)) { + pair.left = notification; + return true; + } + } + } catch(ConsumerTimeoutException e) { + //ignore + } + return false; + } + }); + return pair.left; + } + + protected NotificationPredicate newNotificationPredicate(final EntityNotification.OperationType operationType, + final String typeName, final String guid) { + return new NotificationPredicate() { + @Override + public boolean evaluate(EntityNotification notification) throws Exception { + return notification != null && + notification.getOperationType() == operationType && + notification.getEntity().getTypeName().equals(typeName) && + notification.getEntity().getId()._getId().equals(guid); + } + }; + } + + protected JSONArray searchByDSL(String dslQuery) throws AtlasServiceException { + return atlasClientV1.searchByDSL(dslQuery, 10, 0); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/35e5828f/webapp/src/test/java/org/apache/atlas/web/integration/DataSetLineageJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/integration/DataSetLineageJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/integration/DataSetLineageJerseyResourceIT.java new file mode 100644 index 0000000..8c6ba77 --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/integration/DataSetLineageJerseyResourceIT.java @@ -0,0 +1,298 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.integration; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.apache.atlas.AtlasClient; +import org.apache.atlas.AtlasServiceException; +import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.typesystem.json.InstanceSerialization; +import org.apache.atlas.typesystem.persistence.Id; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONObject; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; + +import static org.testng.Assert.assertEquals; + +/** + * Hive Lineage Integration Tests. + */ +public class DataSetLineageJerseyResourceIT extends BaseResourceIT { + + private String salesFactTable; + private String salesMonthlyTable; + private String salesDBName; + + @BeforeClass + public void setUp() throws Exception { + super.setUp(); + + createTypeDefinitionsV1(); + setupInstances(); + } + + @Test + public void testInputsGraph() throws Exception { + JSONObject response = atlasClientV1.callAPIWithBodyAndParams(AtlasClient.API.NAME_LINEAGE_INPUTS_GRAPH, null, salesMonthlyTable, "inputs", "graph"); + Assert.assertNotNull(response); + System.out.println("inputs graph = " + response); + + Assert.assertNotNull(response.get(AtlasClient.REQUEST_ID)); + + JSONObject results = response.getJSONObject(AtlasClient.RESULTS); + Assert.assertNotNull(results); + + Struct resultsInstance = InstanceSerialization.fromJsonStruct(results.toString(), true); + Map<String, Struct> vertices = (Map<String, Struct>) resultsInstance.get("vertices"); + Assert.assertEquals(vertices.size(), 4); + + Map<String, Struct> edges = (Map<String, Struct>) resultsInstance.get("edges"); + Assert.assertEquals(edges.size(), 4); + } + + @Test + public void testInputsGraphForEntity() throws Exception { + String tableId = atlasClientV1.getEntity(HIVE_TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, + salesMonthlyTable).getId()._getId(); + JSONObject results = atlasClientV1.getInputGraphForEntity(tableId); + Assert.assertNotNull(results); + + Struct resultsInstance = InstanceSerialization.fromJsonStruct(results.toString(), true); + Map<String, Struct> vertices = (Map<String, Struct>) resultsInstance.get("vertices"); + Assert.assertEquals(vertices.size(), 4); + Struct vertex = vertices.get(tableId); + assertEquals(((Struct) vertex.get("vertexId")).get("state"), Id.EntityState.ACTIVE.name()); + + Map<String, Struct> edges = (Map<String, Struct>) resultsInstance.get("edges"); + Assert.assertEquals(edges.size(), 4); + } + + @Test + public void testOutputsGraph() throws Exception { + JSONObject response = atlasClientV1.callAPIWithBodyAndParams(AtlasClient.API.NAME_LINEAGE_OUTPUTS_GRAPH, null, salesFactTable, "outputs", "graph"); + Assert.assertNotNull(response); + System.out.println("outputs graph= " + response); + + Assert.assertNotNull(response.get(AtlasClient.REQUEST_ID)); + + JSONObject results = response.getJSONObject(AtlasClient.RESULTS); + Assert.assertNotNull(results); + + Struct resultsInstance = InstanceSerialization.fromJsonStruct(results.toString(), true); + Map<String, Struct> vertices = (Map<String, Struct>) resultsInstance.get("vertices"); + Assert.assertEquals(vertices.size(), 3); + + Map<String, Struct> edges = (Map<String, Struct>) resultsInstance.get("edges"); + Assert.assertEquals(edges.size(), 4); + } + + @Test + public void testOutputsGraphForEntity() throws Exception { + String tableId = atlasClientV1.getEntity(HIVE_TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, + salesFactTable).getId()._getId(); + JSONObject results = atlasClientV1.getOutputGraphForEntity(tableId); + Assert.assertNotNull(results); + + Struct resultsInstance = InstanceSerialization.fromJsonStruct(results.toString(), true); + Map<String, Struct> vertices = (Map<String, Struct>) resultsInstance.get("vertices"); + Assert.assertEquals(vertices.size(), 3); + Struct vertex = vertices.get(tableId); + assertEquals(((Struct) vertex.get("vertexId")).get("state"), Id.EntityState.ACTIVE.name()); + + Map<String, Struct> edges = (Map<String, Struct>) resultsInstance.get("edges"); + Assert.assertEquals(edges.size(), 4); + } + + @Test + public void testSchema() throws Exception { + JSONObject response = atlasClientV1.callAPIWithBodyAndParams(AtlasClient.API.NAME_LINEAGE_SCHEMA, null, salesFactTable, "schema"); + + Assert.assertNotNull(response); + System.out.println("schema = " + response); + + Assert.assertNotNull(response.get(AtlasClient.REQUEST_ID)); + + JSONObject results = response.getJSONObject(AtlasClient.RESULTS); + Assert.assertNotNull(results); + + JSONArray rows = results.getJSONArray("rows"); + Assert.assertEquals(rows.length(), 4); + + for (int index = 0; index < rows.length(); index++) { + final JSONObject row = rows.getJSONObject(index); + LOG.info("JsonRow - {}", row); + Assert.assertNotNull(row.getString("name")); + Assert.assertNotNull(row.getString("comment")); + Assert.assertEquals(row.getString("$typeName$"), "hive_column_v1"); + } + } + + @Test + public void testSchemaForEntity() throws Exception { + String tableId = atlasClientV1.getEntity(HIVE_TABLE_TYPE, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, salesFactTable).getId()._getId(); + JSONObject results = atlasClientV1.getSchemaForEntity(tableId); + Assert.assertNotNull(results); + + JSONArray rows = results.getJSONArray("rows"); + Assert.assertEquals(rows.length(), 4); + + for (int index = 0; index < rows.length(); index++) { + final JSONObject row = rows.getJSONObject(index); + LOG.info("JsonRow - {}", row); + Assert.assertNotNull(row.getString("name")); + Assert.assertNotNull(row.getString("comment")); + Assert.assertEquals(row.getString("$typeName$"), "hive_column_v1"); + } + } + + @Test(expectedExceptions = AtlasServiceException.class) + public void testSchemaForInvalidTable() throws Exception { + JSONObject response = atlasClientV1.callAPIWithBodyAndParams(AtlasClient.API.NAME_LINEAGE_SCHEMA, null, "blah", "schema"); + } + + @Test(expectedExceptions = AtlasServiceException.class) + public void testSchemaForDB() throws Exception { + JSONObject response = atlasClientV1.callAPIWithBodyAndParams(AtlasClient.API.NAME_LINEAGE_SCHEMA, null, salesDBName, "schema"); + } + + private void setupInstances() throws Exception { + HierarchicalTypeDefinition<TraitType> factTrait = + TypesUtil.createTraitTypeDef("Fact", ImmutableSet.<String>of()); + HierarchicalTypeDefinition<TraitType> etlTrait = + TypesUtil.createTraitTypeDef("ETL", ImmutableSet.<String>of()); + HierarchicalTypeDefinition<TraitType> dimensionTrait = + TypesUtil.createTraitTypeDef("Dimension", ImmutableSet.<String>of()); + HierarchicalTypeDefinition<TraitType> metricTrait = + TypesUtil.createTraitTypeDef("Metric", ImmutableSet.<String>of()); + createType(getTypesDef(null, null, + ImmutableList.of(factTrait, etlTrait, dimensionTrait, metricTrait), null)); + + salesDBName = "Sales" + randomString(); + Id salesDB = database(salesDBName, "Sales Database", "John ETL", + "hdfs://host:8000/apps/warehouse/sales"); + + List<Referenceable> salesFactColumns = ImmutableList + .of(column("time_id", "int", "time id"), column("product_id", "int", "product id"), + column("customer_id", "int", "customer id", "pii"), + column("sales", "double", "product id", "Metric")); + + salesFactTable = "sales_fact" + randomString(); + Id salesFact = table(salesFactTable, "sales fact table", salesDB, "Joe", "MANAGED", salesFactColumns, "Fact"); + + List<Referenceable> timeDimColumns = ImmutableList + .of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"), + column("weekDay", "int", "week Day")); + + Id timeDim = + table("time_dim" + randomString(), "time dimension table", salesDB, "John Doe", "EXTERNAL", + timeDimColumns, "Dimension"); + + Id reportingDB = + database("Reporting" + randomString(), "reporting database", "Jane BI", + "hdfs://host:8000/apps/warehouse/reporting"); + + Id salesFactDaily = + table("sales_fact_daily_mv" + randomString(), "sales fact daily materialized view", reportingDB, + "Joe BI", "MANAGED", salesFactColumns, "Metric"); + + loadProcess("loadSalesDaily" + randomString(), "John ETL", ImmutableList.of(salesFact, timeDim), + ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL"); + + salesMonthlyTable = "sales_fact_monthly_mv" + randomString(); + Id salesFactMonthly = + table(salesMonthlyTable, "sales fact monthly materialized view", reportingDB, "Jane BI", + "MANAGED", salesFactColumns, "Metric"); + + loadProcess("loadSalesMonthly" + randomString(), "John ETL", ImmutableList.of(salesFactDaily), + ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL"); + } + + Id database(String name, String description, String owner, String locationUri, String... traitNames) + throws Exception { + Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames); + referenceable.set(NAME, name); + referenceable.set(QUALIFIED_NAME, name); + referenceable.set(CLUSTER_NAME, locationUri + name); + referenceable.set("description", description); + referenceable.set("owner", owner); + referenceable.set("locationUri", locationUri); + referenceable.set("createTime", System.currentTimeMillis()); + + return createInstance(referenceable); + } + + Referenceable column(String name, String type, String comment, String... traitNames) throws Exception { + Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames); + referenceable.set(NAME, name); + referenceable.set(QUALIFIED_NAME, name); + referenceable.set("type", type); + referenceable.set("comment", comment); + + return referenceable; + } + + Id table(String name, String description, Id dbId, String owner, String tableType, List<Referenceable> columns, + String... traitNames) throws Exception { + Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames); + referenceable.set("name", name); + referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name); + referenceable.set("description", description); + referenceable.set("owner", owner); + referenceable.set("tableType", tableType); + referenceable.set("createTime", System.currentTimeMillis()); + referenceable.set("lastAccessTime", System.currentTimeMillis()); + referenceable.set("retention", System.currentTimeMillis()); + + referenceable.set("db", dbId); + referenceable.set("columns", columns); + + return createInstance(referenceable); + } + + Id loadProcess(String name, String user, List<Id> inputTables, List<Id> outputTables, String queryText, + String queryPlan, String queryId, String queryGraph, String... traitNames) throws Exception { + Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames); + referenceable.set("name", name); + referenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name); + referenceable.set("userName", user); + referenceable.set("startTime", System.currentTimeMillis()); + referenceable.set("endTime", System.currentTimeMillis() + 10000); + + referenceable.set("inputs", inputTables); + referenceable.set("outputs", outputTables); + + referenceable.set("operationType", "testOperation"); + referenceable.set("queryText", queryText); + referenceable.set("queryPlan", queryPlan); + referenceable.set("queryId", queryId); + referenceable.set("queryGraph", queryGraph); + + return createInstance(referenceable); + } +}
