http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/util/CredentialProviderUtility.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/util/CredentialProviderUtility.java b/webapp/src/main/java/org/apache/atlas/util/CredentialProviderUtility.java new file mode 100755 index 0000000..1a512a1 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/util/CredentialProviderUtility.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.util; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.security.alias.JavaKeyStoreProvider; + +import java.io.Console; +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY; +import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY; +import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY; + +/** + * A utility class for generating a credential provider containing the entries required for supporting the SSL implementation + * of the DGC server. + */ +public class CredentialProviderUtility { + private static final String[] KEYS = new String[] {KEYSTORE_PASSWORD_KEY, + TRUSTSTORE_PASSWORD_KEY, SERVER_CERT_PASSWORD_KEY}; + + public static abstract class TextDevice { + public abstract void printf(String fmt, Object... params); + + public abstract String readLine(String fmt, Object ... args); + + public abstract char[] readPassword(String fmt, Object ... args); + + } + + private static TextDevice DEFAULT_TEXT_DEVICE = new TextDevice() { + Console console = System.console(); + + @Override + public void printf(String fmt, Object... params) { + console.printf(fmt, params); + } + + @Override + public String readLine(String fmt, Object ... args) { + return console.readLine(fmt, args); + } + + @Override + public char[] readPassword(String fmt, Object ... args) { + return console.readPassword(fmt, args); + } + }; + + public static TextDevice textDevice = DEFAULT_TEXT_DEVICE; + + public static void main(String[] args) throws IOException { + // prompt for the provider name + CredentialProvider provider = getCredentialProvider(textDevice); + + char[] cred; + for (String key : KEYS) { + cred = getPassword(textDevice, key); + // create a credential entry and store it + boolean overwrite = true; + if (provider.getCredentialEntry(key) != null) { + String choice = textDevice.readLine("Entry for %s already exists. Overwrite? (y/n) [y]:", key); + overwrite = StringUtils.isEmpty(choice) || choice.equalsIgnoreCase("y"); + if (overwrite) { + provider.deleteCredentialEntry(key); + provider.flush(); + provider.createCredentialEntry(key, cred); + provider.flush(); + textDevice.printf("Entry for %s was overwritten with the new value.\n", key); + } else { + textDevice.printf("Entry for %s was not overwritten.\n", key); + } + } else { + provider.createCredentialEntry(key, cred); + provider.flush(); + } + } + } + + /** + * Retrieves a password from the command line. + * @param textDevice the system console. + * @param key the password key/alias. + * @return the password. + */ + private static char[] getPassword(TextDevice textDevice, String key) { + boolean noMatch; + char[] cred = new char[0]; + char[] passwd1; + char[] passwd2; + do { + passwd1 = textDevice.readPassword("Please enter the password value for %s:", key); + passwd2 = textDevice.readPassword("Please enter the password value for %s again:", key); + noMatch = !Arrays.equals(passwd1, passwd2); + if (noMatch) { + if (passwd1 != null) Arrays.fill(passwd1, ' '); + textDevice.printf("Password entries don't match. Please try again.\n"); + } else { + if (passwd1.length == 0) { + textDevice.printf("An empty password is not valid. Please try again.\n"); + noMatch = true; + } else { + cred = passwd1; + } + } + if (passwd2 != null) Arrays.fill(passwd2, ' '); + } while (noMatch); + return cred; + } + + /**\ + * Returns a credential provider for the entered JKS path. + * @param textDevice the system console. + * @return the Credential provider + * @throws IOException + */ + private static CredentialProvider getCredentialProvider(TextDevice textDevice) throws IOException { + String providerPath = textDevice.readLine("Please enter the full path to the credential provider:"); + File file = new File(providerPath); + if (file.exists()) { + textDevice.printf("%s already exists. You will need to specify whether existing entries should be overwritten " + + "(default is 'yes')\n", providerPath); + } + String providerURI = JavaKeyStoreProvider.SCHEME_NAME + "://file" + providerPath; + Configuration conf = new Configuration(false); + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerURI); + return CredentialProviderFactory.getProviders(conf).get(0); + } +}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/errors/LoggingExceptionMapper.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/errors/LoggingExceptionMapper.java b/webapp/src/main/java/org/apache/atlas/web/errors/LoggingExceptionMapper.java new file mode 100755 index 0000000..bdf84a6 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/errors/LoggingExceptionMapper.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.errors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import java.util.concurrent.ThreadLocalRandom; + +/** + * Exception mapper for Jersey. + * @param <E> + */ +public class LoggingExceptionMapper<E extends Throwable> implements ExceptionMapper<E> { + private static final Logger LOGGER = LoggerFactory.getLogger(LoggingExceptionMapper.class); + + @Override + public Response toResponse(E exception) { + if (exception instanceof WebApplicationException) { + return ((WebApplicationException) exception).getResponse(); + } + + final long id = ThreadLocalRandom.current().nextLong(); + logException(id, exception); + return Response.serverError() + .entity(formatErrorMessage(id, exception)) + .build(); + } + + @SuppressWarnings("UnusedParameters") + protected String formatErrorMessage(long id, E exception) { + return String.format( + "There was an error processing your request. It has been logged (ID %016x).", id); + } + + protected void logException(long id, E exception) { + LOGGER.error(formatLogMessage(id, exception), exception); + } + + @SuppressWarnings("UnusedParameters") + protected String formatLogMessage(long id, Throwable exception) { + return String.format("Error handling a request: %016x", id); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java new file mode 100755 index 0000000..0ebf22d --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.filters; + +import com.google.inject.Singleton; +import org.apache.atlas.MetadataServiceClient; +import org.apache.atlas.web.util.DateTimeHelper; +import org.apache.atlas.web.util.Servlets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.Date; +import java.util.UUID; + +/** + * This records audit information as part of the filter after processing the request + * and also introduces a UUID into request and response for tracing requests in logs. + */ +@Singleton +public class AuditFilter implements Filter { + + private static final Logger AUDIT_LOG = LoggerFactory.getLogger("AUDIT"); + private static final Logger LOG = LoggerFactory.getLogger(AuditFilter.class); + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + LOG.info("AuditFilter initialization started"); + } + + @Override + public void doFilter(ServletRequest request, + ServletResponse response, + FilterChain filterChain) throws IOException, ServletException { + final String requestTimeISO9601 = DateTimeHelper.formatDateUTC(new Date()); + final HttpServletRequest httpRequest = (HttpServletRequest) request; + final String requestId = UUID.randomUUID().toString(); + final Thread currentThread = Thread.currentThread(); + final String oldName = currentThread.getName(); + + try { + currentThread.setName(formatName(oldName, requestId)); + recordAudit(httpRequest, requestTimeISO9601); + filterChain.doFilter(request, response); + } finally { + // put the request id into the response so users can trace logs for this request + ((HttpServletResponse) response).setHeader(MetadataServiceClient.REQUEST_ID, requestId); + currentThread.setName(oldName); + } + } + + private String formatName(String oldName, String requestId) { + return oldName + " - " + requestId; + } + + private void recordAudit(HttpServletRequest httpRequest, String whenISO9601) { + final String who = getUserFromRequest(httpRequest); + final String fromHost = httpRequest.getRemoteHost(); + final String fromAddress = httpRequest.getRemoteAddr(); + final String whatURL = Servlets.getRequestURL(httpRequest); + final String whatAddrs = httpRequest.getLocalAddr(); + + LOG.debug("Audit: {}/{} performed request {} ({}) at time {}", + who, fromAddress, whatURL, whatAddrs, whenISO9601); + audit(who, fromAddress, fromHost, whatURL, whatAddrs, whenISO9601); + } + + private String getUserFromRequest(HttpServletRequest httpRequest) { + // look for the user in the request + final String userFromRequest = Servlets.getUserFromRequest(httpRequest); + return userFromRequest == null ? "UNKNOWN" : userFromRequest; + } + + private void audit(String who, String fromAddress, String fromHost, String whatURL, + String whatAddrs, String whenISO9601) { + AUDIT_LOG.info("Audit: {}/{}-{} performed request {} ({}) at time {}", + who, fromAddress, fromHost, whatURL, whatAddrs, whenISO9601); + } + + @Override + public void destroy() { + // do nothing + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/filters/MetadataAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/MetadataAuthenticationFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/MetadataAuthenticationFilter.java new file mode 100644 index 0000000..5e9e453 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/filters/MetadataAuthenticationFilter.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.filters; + +import com.google.inject.Singleton; +import org.apache.atlas.PropertiesUtil; +import org.apache.atlas.security.SecurityProperties; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.Properties; + +/** + * This enforces authentication as part of the filter before processing the request. + * todo: Subclass of {@link org.apache.hadoop.security.authentication.server.AuthenticationFilter}. + */ +@Singleton +public class MetadataAuthenticationFilter extends AuthenticationFilter { + private static final Logger LOG = LoggerFactory.getLogger(MetadataAuthenticationFilter.class); + static final String PREFIX = "atlas.http.authentication."; + + @Override + protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException { + PropertiesConfiguration configuration; + try { + configuration = PropertiesUtil.getApplicationProperties(); + } catch (Exception e) { + throw new ServletException(e); + } + + Properties config = new Properties(); + + config.put(AuthenticationFilter.COOKIE_PATH, "/"); + + // add any config passed in as init parameters + Enumeration<String> enumeration = filterConfig.getInitParameterNames(); + while (enumeration.hasMoreElements()) { + String name = enumeration.nextElement(); + config.put(name, filterConfig.getInitParameter(name)); + } + // transfer application.properties config items starting with defined prefix + Iterator<String> itor = configuration.getKeys(); + while (itor.hasNext()) { + String name = itor.next(); + if (name.startsWith(PREFIX)) { + String value = configuration.getString(name); + name = name.substring(PREFIX.length()); + config.put(name, value); + } + } + + //Resolve _HOST into bind address + String bindAddress = configuration.getString(SecurityProperties.BIND_ADDRESS); + if (bindAddress == null) { + LOG.info("No host name configured. Defaulting to local host name."); + try { + bindAddress = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + throw new ServletException("Unable to obtain host name", e); + } + } + String principal = config.getProperty(KerberosAuthenticationHandler.PRINCIPAL); + if (principal != null) { + try { + principal = SecurityUtil.getServerPrincipal(principal, bindAddress); + } catch (IOException ex) { + throw new RuntimeException("Could not resolve Kerberos principal name: " + ex.toString(), ex); + } + config.put(KerberosAuthenticationHandler.PRINCIPAL, principal); + } + + LOG.info("AuthenticationFilterConfig: {}", config); + + return config; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java b/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java new file mode 100755 index 0000000..5fb5b6b --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/listeners/GuiceServletConfig.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.listeners; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.servlet.GuiceServletContextListener; +import com.sun.jersey.api.core.PackagesResourceConfig; +import com.sun.jersey.guice.JerseyServletModule; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import org.apache.atlas.MetadataException; +import org.apache.atlas.MetadataServiceClient; +import org.apache.atlas.PropertiesUtil; +import org.apache.atlas.RepositoryMetadataModule; +import org.apache.atlas.web.filters.AuditFilter; +import org.apache.atlas.web.filters.MetadataAuthenticationFilter; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.ServletContextEvent; +import java.util.HashMap; +import java.util.Map; + +public class GuiceServletConfig extends GuiceServletContextListener { + + private static final Logger LOG = LoggerFactory.getLogger(GuiceServletConfig.class); + + private static final String GUICE_CTX_PARAM = "guice.packages"; + static final String HTTP_AUTHENTICATION_ENABLED = "atlas.http.authentication.enabled"; + private Injector injector; + + @Override + protected Injector getInjector() { + LOG.info("Loading Guice modules"); + /* + * More information on this can be found here: + * https://jersey.java.net/nonav/apidocs/1 + * .11/contribs/jersey-guice/com/sun/jersey/guice/spi/container/servlet/package-summary + * .html + */ + if (injector == null) { + injector = Guice.createInjector( + new RepositoryMetadataModule(), + new JerseyServletModule() { + @Override + protected void configureServlets() { + filter("/*").through(AuditFilter.class); + try { + configureAuthenticationFilter(); + } catch (ConfigurationException e) { + LOG.warn("Unable to add and configure authentication filter", e); + } + + String packages = getServletContext().getInitParameter(GUICE_CTX_PARAM); + + LOG.info("Jersey loading from packages: " + packages); + + Map<String, String> params = new HashMap<>(); + params.put(PackagesResourceConfig.PROPERTY_PACKAGES, packages); + serve("/" + MetadataServiceClient.BASE_URI + "*").with(GuiceContainer.class, params); + } + + private void configureAuthenticationFilter() throws ConfigurationException { + try { + PropertiesConfiguration configuration = PropertiesUtil.getApplicationProperties(); + if (Boolean.valueOf(configuration.getString(HTTP_AUTHENTICATION_ENABLED))) { + filter("/*").through(MetadataAuthenticationFilter.class); + } + } catch (MetadataException e) { + LOG.warn("Error loading configuration and initializing authentication filter", e); + } + } + }); + + LOG.info("Guice modules loaded"); + } + + return injector; + } + + @Override + public void contextInitialized(ServletContextEvent servletContextEvent) { + super.contextInitialized(servletContextEvent); + + // perform login operations + LoginProcessor loginProcessor = new LoginProcessor(); + loginProcessor.login(); + } + + @Override + public void contextDestroyed(ServletContextEvent servletContextEvent) { + super.contextDestroyed(servletContextEvent); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java b/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java new file mode 100644 index 0000000..2c6d98e --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/listeners/LoginProcessor.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.web.listeners; + +import org.apache.atlas.MetadataException; +import org.apache.atlas.PropertiesUtil; +import org.apache.atlas.security.SecurityProperties; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Shell; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * A class capable of performing a simple or kerberos login. + */ +public class LoginProcessor { + + private static final Logger LOG = LoggerFactory + .getLogger(LoginProcessor.class); + public static final String METADATA_AUTHENTICATION_PREFIX = "atlas.authentication."; + public static final String AUTHENTICATION_METHOD = METADATA_AUTHENTICATION_PREFIX + "method"; + public static final String AUTHENTICATION_PRINCIPAL = METADATA_AUTHENTICATION_PREFIX + "principal"; + public static final String AUTHENTICATION_KEYTAB = METADATA_AUTHENTICATION_PREFIX + "keytab"; + + /** + * Perform a SIMPLE login based on established OS identity or a kerberos based login using the configured + * principal and keytab (via application.properties). + */ + public void login() { + // first, let's see if we're running in a hadoop cluster and have the env configured + boolean isHadoopCluster = isHadoopCluster(); + Configuration hadoopConfig = isHadoopCluster ? getHadoopConfiguration() : new Configuration(false); + PropertiesConfiguration configuration = null; + try { + configuration = getPropertiesConfiguration(); + } catch (ConfigurationException e) { + LOG.warn("Error reading application configuration", e); + } + if (!isHadoopCluster) { + // need to read the configured authentication choice and create the UGI configuration + setupHadoopConfiguration(hadoopConfig, configuration); + } + doServiceLogin(hadoopConfig, configuration); + } + + protected void doServiceLogin(Configuration hadoopConfig, PropertiesConfiguration configuration) { + UserGroupInformation.setConfiguration(hadoopConfig); + + UserGroupInformation ugi = null; + UserGroupInformation.AuthenticationMethod authenticationMethod = + SecurityUtil.getAuthenticationMethod(hadoopConfig); + try { + if (authenticationMethod == UserGroupInformation.AuthenticationMethod.SIMPLE) { + UserGroupInformation.loginUserFromSubject(null); + } else if (authenticationMethod == UserGroupInformation.AuthenticationMethod.KERBEROS) { + String bindAddress = getHostname(configuration); + UserGroupInformation.loginUserFromKeytab( + getServerPrincipal(configuration.getString(AUTHENTICATION_PRINCIPAL), bindAddress), + configuration.getString(AUTHENTICATION_KEYTAB)); + } + LOG.info("Logged in user {}", UserGroupInformation.getLoginUser()); + } catch (IOException e) { + throw new IllegalStateException(String.format("Unable to perform %s login.", authenticationMethod), e); + } + } + + private String getHostname(PropertiesConfiguration configuration) { + String bindAddress = configuration.getString(SecurityProperties.BIND_ADDRESS); + if (bindAddress == null) { + LOG.info("No host name configured. Defaulting to local host name."); + try { + bindAddress = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + throw new IllegalStateException(e); + } + } + return bindAddress; + } + + protected void setupHadoopConfiguration(Configuration hadoopConfig, PropertiesConfiguration configuration) { + String authMethod; + authMethod = configuration != null ? configuration.getString(AUTHENTICATION_METHOD) : null; + // getString may return null, and would like to log the nature of the default setting + if (authMethod == null) { + LOG.info("No authentication method configured. Defaulting to simple authentication"); + authMethod = "simple"; + } + SecurityUtil.setAuthenticationMethod( + UserGroupInformation.AuthenticationMethod.valueOf(authMethod.toUpperCase()), + hadoopConfig); + } + + /** + * Return a server (service) principal. The token "_HOST" in the principal will be replaced with the local host + * name (e.g. dgi/_HOST will be changed to dgi/localHostName) + * @param principal the input principal containing an option "_HOST" token + * @return the service principal. + * @throws IOException + */ + private String getServerPrincipal(String principal, String host) throws IOException { + return SecurityUtil.getServerPrincipal(principal, host); + } + + /** + * Returns a Hadoop configuration instance. + * @return the configuration. + */ + protected Configuration getHadoopConfiguration() { + return new Configuration(); + } + + /** + * Returns the metadata application configuration. + * @return the metadata configuration. + * @throws ConfigurationException + */ + protected PropertiesConfiguration getPropertiesConfiguration() throws ConfigurationException { + try { + return PropertiesUtil.getApplicationProperties(); + } catch (MetadataException e) { + throw new ConfigurationException(e); + } + } + + /** + * Uses a hadoop shell to discern whether a hadoop cluster is available/configured. + * @return true if a hadoop cluster is detected. + */ + protected boolean isHadoopCluster() { + boolean isHadoopCluster = false; + try { + isHadoopCluster = Shell.getHadoopHome() != null; + } catch (IOException e) { + // ignore - false is default setting + } + return isHadoopCluster; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/params/AbstractParam.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/params/AbstractParam.java b/webapp/src/main/java/org/apache/atlas/web/params/AbstractParam.java new file mode 100755 index 0000000..f5b555e --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/params/AbstractParam.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.params; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +/** + * An abstract base class from which to build Jersey parameter classes. + * + * @param <T> the type of value wrapped by the parameter + */ +public abstract class AbstractParam<T> { + private final T value; + + /** + * Given an input value from a client, creates a parameter wrapping its parsed value. + * + * @param input an input value from a client request + */ + @SuppressWarnings({"AbstractMethodCallInConstructor", + "OverriddenMethodCallDuringObjectConstruction"}) + protected AbstractParam(String input) { + try { + this.value = parse(input); + } catch (Exception e) { + throw new WebApplicationException(error(input, e)); + } + } + + /** + * Given a string representation which was unable to be parsed and the exception thrown, produce + * a {@link javax.ws.rs.core.Response} to be sent to the client. + * + * By default, generates a {@code 400 Bad Request} with a plain text entity generated by + * {@link #errorMessage(String, Exception)}. + * + * @param input the raw input value + * @param e the exception thrown while parsing {@code input} + * @return the {@link javax.ws.rs.core.Response} to be sent to the client + */ + protected Response error(String input, Exception e) { + return Response.status(getErrorStatus()) + .entity(errorMessage(input, e)) + .type(mediaType()) + .build(); + } + + /** + * Returns the media type of the error message entity. + * + * @return the media type of the error message entity + */ + protected MediaType mediaType() { + return MediaType.TEXT_PLAIN_TYPE; + } + + /** + * Given a string representation which was unable to be parsed and the exception thrown, produce + * an entity to be sent to the client. + * + * @param input the raw input value + * @param e the exception thrown while parsing {@code input} + * @return the error message to be sent the client + */ + protected String errorMessage(String input, Exception e) { + return String.format("Invalid parameter: %s (%s)", input, e.getMessage()); + } + + /** + * Given a string representation which was unable to be parsed, produce a {@link javax.ws.rs + * .core.Response.Status} for the + * {@link Response} to be sent to the client. + * + * @return the HTTP {@link javax.ws.rs.core.Response.Status} of the error message + */ + @SuppressWarnings("MethodMayBeStatic") + protected Response.Status getErrorStatus() { + return Response.Status.BAD_REQUEST; + } + + /** + * Given a string representation, parse it and return an instance of the parameter type. + * + * @param input the raw input + * @return {@code input}, parsed as an instance of {@code T} + * @throws Exception if there is an error parsing the input + */ + protected abstract T parse(String input) throws Exception; + + /** + * Returns the underlying value. + * + * @return the underlying value + */ + public T get() { + return value; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + final AbstractParam<?> that = (AbstractParam<?>) obj; + return value.equals(that.value); + } + + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public String toString() { + return value.toString(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/params/BooleanParam.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/params/BooleanParam.java b/webapp/src/main/java/org/apache/atlas/web/params/BooleanParam.java new file mode 100755 index 0000000..d81fb6f --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/params/BooleanParam.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.params; + +/** + * A parameter encapsulating boolean values. If the query parameter value is {@code "true"}, + * regardless of case, the returned value is {@link Boolean#TRUE}. If the query parameter value is + * {@code "false"}, regardless of case, the returned value is {@link Boolean#FALSE}. All other + * values will return a {@code 400 Bad Request} response. + */ +public class BooleanParam extends AbstractParam<Boolean> { + + public BooleanParam(String input) { + super(input); + } + + @Override + protected String errorMessage(String input, Exception e) { + return '"' + input + "\" must be \"true\" or \"false\"."; + } + + @Override + protected Boolean parse(String input) throws Exception { + if ("true".equalsIgnoreCase(input)) { + return Boolean.TRUE; + } + if ("false".equalsIgnoreCase(input)) { + return Boolean.FALSE; + } + throw new Exception(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/params/DateTimeParam.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/params/DateTimeParam.java b/webapp/src/main/java/org/apache/atlas/web/params/DateTimeParam.java new file mode 100755 index 0000000..4b417e6 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/params/DateTimeParam.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.params; + +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +/** + * A parameter encapsulating date/time values. All non-parsable values will return a {@code 400 Bad + * Request} response. All values returned are in UTC. + */ +public class DateTimeParam extends AbstractParam<DateTime> { + + public DateTimeParam(String input) { + super(input); + } + + @Override + protected DateTime parse(String input) throws Exception { + return new DateTime(input, DateTimeZone.UTC); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java new file mode 100755 index 0000000..4ad27c8 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.resources; + +import org.apache.atlas.web.util.Servlets; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.commons.lang.StringUtils; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; + +import javax.inject.Singleton; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +/** + * Jersey Resource for admin operations. + */ +@Path("admin") +@Singleton +public class AdminResource { + + private Response version; + + /** + * Fetches the thread stack dump for this application. + * + * @return json representing the thread stack dump. + */ + @GET + @Path("stack") + @Produces(MediaType.TEXT_PLAIN) + public String getThreadDump() { + ThreadGroup topThreadGroup = Thread.currentThread().getThreadGroup(); + + while (topThreadGroup.getParent() != null) { + topThreadGroup = topThreadGroup.getParent(); + } + Thread[] threads = new Thread[topThreadGroup.activeCount()]; + + int nr = topThreadGroup.enumerate(threads); + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < nr; i++) { + builder.append(threads[i].getName()).append("\nState: "). + append(threads[i].getState()).append("\n"); + String stackTrace = StringUtils.join(threads[i].getStackTrace(), "\n"); + builder.append(stackTrace); + } + return builder.toString(); + } + + /** + * Fetches the version for this application. + * + * @return json representing the version. + */ + @GET + @Path("version") + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response getVersion() { + if (version == null) { + try { + PropertiesConfiguration configProperties = + new PropertiesConfiguration("atlas-buildinfo.properties"); + + JSONObject response = new JSONObject(); + response.put("Version", configProperties.getString("build.version", "UNKNOWN")); + response.put("Name", + configProperties.getString("project.name", "apache-atlas")); + response.put("Description", configProperties.getString("project.description", + "Metadata Management and Data Governance Platform over Hadoop")); + + // todo: add hadoop version? + // response.put("Hadoop", VersionInfo.getVersion() + "-r" + VersionInfo.getRevision()); + version = Response.ok(response).build(); + } catch (JSONException | ConfigurationException e) { + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + return version; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java new file mode 100755 index 0000000..deedf45 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.resources; + +import com.google.common.base.Preconditions; +import org.apache.atlas.MetadataException; +import org.apache.atlas.MetadataServiceClient; +import org.apache.atlas.ParamChecker; +import org.apache.atlas.repository.EntityNotFoundException; +import org.apache.atlas.services.MetadataService; +import org.apache.atlas.typesystem.types.ValueConversionException; +import org.apache.atlas.web.util.Servlets; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriBuilder; +import javax.ws.rs.core.UriInfo; +import java.io.IOException; +import java.net.URI; +import java.util.List; + + +/** + * Entity management operations as REST API. + * + * An entity is an "instance" of a Type. Entities conform to the definition + * of the Type they correspond with. + */ +@Path("entities") +@Singleton +public class EntityResource { + + private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class); + private static final String TRAIT_NAME = "traitName"; + + private final MetadataService metadataService; + + @Context + UriInfo uriInfo; + + /** + * Created by the Guice ServletModule and injected with the + * configured MetadataService. + * + * @param metadataService metadata service handle + */ + @Inject + public EntityResource(MetadataService metadataService) { + this.metadataService = metadataService; + } + + /** + * Submits an entity definition (instance) corresponding to a given type. + */ + @POST + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response submit(@Context HttpServletRequest request) { + try { + final String entity = Servlets.getRequestPayload(request); + LOG.debug("submitting entity {} ", entity); + + final String guid = metadataService.createEntity(entity); + + UriBuilder ub = uriInfo.getAbsolutePathBuilder(); + URI locationURI = ub.path(guid).build(); + + JSONObject response = new JSONObject(); + response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId()); + response.put(MetadataServiceClient.GUID, guid); + response.put(MetadataServiceClient.DEFINITION, + metadataService.getEntityDefinition(guid)); + + return Response.created(locationURI).entity(response).build(); + + } catch(ValueConversionException ve) { + LOG.error("Unable to persist entity instance due to a desrialization error ", ve); + throw new WebApplicationException( + Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST)); + } catch (MetadataException | IllegalArgumentException e) { + LOG.error("Unable to persist entity instance", e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to persist entity instance", e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** + * Fetch the complete definition of an entity given its GUID. + * + * @param guid GUID for the entity + */ + @GET + @Path("{guid}") + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response getEntityDefinition(@PathParam("guid") String guid) { + try { + LOG.debug("Fetching entity definition for guid={} ", guid); + ParamChecker.notEmpty(guid, "guid cannot be null"); + final String entityDefinition = metadataService.getEntityDefinition(guid); + + JSONObject response = new JSONObject(); + response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId()); + response.put(MetadataServiceClient.GUID, guid); + + Response.Status status = Response.Status.NOT_FOUND; + if (entityDefinition != null) { + response.put(MetadataServiceClient.DEFINITION, entityDefinition); + status = Response.Status.OK; + } else { + response.put(MetadataServiceClient.ERROR, Servlets.escapeJsonString( + String.format("An entity with GUID={%s} does not exist", guid))); + } + + return Response.status(status).entity(response).build(); + + } catch (EntityNotFoundException e) { + LOG.error("An entity with GUID={} does not exist", guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (MetadataException | IllegalArgumentException e) { + LOG.error("Bad GUID={}", guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to get instance definition for GUID {}", guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** + * Gets the list of entities for a given entity type. + * + * @param entityType name of a type which is unique + */ + @GET + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response getEntityListByType(@QueryParam("type") String entityType) { + try { + Preconditions.checkNotNull(entityType, "Entity type cannot be null"); + + LOG.debug("Fetching entity list for type={} ", entityType); + final List<String> entityList = metadataService.getEntityList(entityType); + + JSONObject response = new JSONObject(); + response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId()); + response.put(MetadataServiceClient.TYPENAME, entityType); + response.put(MetadataServiceClient.RESULTS, new JSONArray(entityList)); + response.put(MetadataServiceClient.COUNT, entityList.size()); + + return Response.ok(response).build(); + } catch (NullPointerException e) { + LOG.error("Entity type cannot be null", e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (MetadataException | IllegalArgumentException e) { + LOG.error("Unable to get entity list for type {}", entityType, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to get entity list for type {}", entityType, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** + * Adds property to the given entity id + * @param guid entity id + * @param property property to add + * @param value property's value + * @return response payload as json + */ + @PUT + @Path("{guid}") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response update(@PathParam("guid") String guid, + @QueryParam("property") String property, + @QueryParam("value") String value) { + try { + Preconditions.checkNotNull(property, "Entity property cannot be null"); + Preconditions.checkNotNull(value, "Entity value cannot be null"); + + metadataService.updateEntity(guid, property, value); + + JSONObject response = new JSONObject(); + response.put(MetadataServiceClient.REQUEST_ID, Thread.currentThread().getName()); + return Response.ok(response).build(); + } catch (EntityNotFoundException e) { + LOG.error("An entity with GUID={} does not exist", guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (MetadataException | IllegalArgumentException e) { + LOG.error("Unable to add property {} to entity id {}", property, guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to add property {} to entity id {}", property, guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + // Trait management functions + /** + * Gets the list of trait names for a given entity represented by a guid. + * + * @param guid globally unique identifier for the entity + * @return a list of trait names for the given entity guid + */ + @GET + @Path("{guid}/traits") + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response getTraitNames(@PathParam("guid") String guid) { + try { + LOG.debug("Fetching trait names for entity={}", guid); + final List<String> traitNames = metadataService.getTraitNames(guid); + + JSONObject response = new JSONObject(); + response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId()); + response.put(MetadataServiceClient.GUID, guid); + response.put(MetadataServiceClient.RESULTS, new JSONArray(traitNames)); + response.put(MetadataServiceClient.COUNT, traitNames.size()); + + return Response.ok(response).build(); + } catch (EntityNotFoundException e) { + LOG.error("An entity with GUID={} does not exist", guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (MetadataException | IllegalArgumentException e) { + LOG.error("Unable to get trait names for entity {}", guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to get trait names for entity {}", guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** + * Adds a new trait to an existing entity represented by a guid. + * + * @param guid globally unique identifier for the entity + */ + @POST + @Path("{guid}/traits") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response addTrait(@Context HttpServletRequest request, + @PathParam("guid") String guid) { + try { + final String traitDefinition = Servlets.getRequestPayload(request); + LOG.debug("Adding trait={} for entity={} ", traitDefinition, guid); + metadataService.addTrait(guid, traitDefinition); + + UriBuilder ub = uriInfo.getAbsolutePathBuilder(); + URI locationURI = ub.path(guid).build(); + + JSONObject response = new JSONObject(); + response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId()); + response.put(MetadataServiceClient.GUID, guid); + + return Response.created(locationURI).entity(response).build(); + } catch (EntityNotFoundException e) { + LOG.error("An entity with GUID={} does not exist", guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (MetadataException | IOException | IllegalArgumentException e) { + LOG.error("Unable to add trait for entity={}", guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to add trait for entity={}", guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** + * Deletes a given trait from an existing entity represented by a guid. + * + * @param guid globally unique identifier for the entity + * @param traitName name of the trait + */ + @DELETE + @Path("{guid}/traits/{traitName}") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response deleteTrait(@Context HttpServletRequest request, + @PathParam("guid") String guid, + @PathParam(TRAIT_NAME) String traitName) { + LOG.debug("Deleting trait={} from entity={} ", traitName, guid); + try { + metadataService.deleteTrait(guid, traitName); + + JSONObject response = new JSONObject(); + response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId()); + response.put(MetadataServiceClient.GUID, guid); + response.put(TRAIT_NAME, traitName); + + return Response.ok(response).build(); + } catch (EntityNotFoundException e) { + LOG.error("An entity with GUID={} does not exist", guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (MetadataException | IllegalArgumentException e) { + LOG.error("Unable to delete trait name={} for entity={}", traitName, guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to delete trait name={} for entity={}", traitName, guid, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java new file mode 100644 index 0000000..cfd055b --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.resources; + +import org.apache.atlas.MetadataServiceClient; +import org.apache.atlas.ParamChecker; +import org.apache.atlas.discovery.DiscoveryException; +import org.apache.atlas.discovery.LineageService; +import org.apache.atlas.repository.EntityNotFoundException; +import org.apache.atlas.web.util.Servlets; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; + +/** + * Jersey Resource for Hive Table Lineage. + */ +@Path("lineage/hive") +@Singleton +public class HiveLineageResource { + + private static final Logger LOG = LoggerFactory.getLogger(HiveLineageResource.class); + + private final LineageService lineageService; + + /** + * Created by the Guice ServletModule and injected with the + * configured LineageService. + * + * @param lineageService lineage service handle + */ + @Inject + public HiveLineageResource(LineageService lineageService) { + this.lineageService = lineageService; + } + + /** + * Returns the inputs graph for a given entity. + * + * @param tableName table name + */ + @GET + @Path("table/{tableName}/inputs/graph") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response inputsGraph(@Context HttpServletRequest request, + @PathParam("tableName") String tableName) { + LOG.info("Fetching lineage inputs graph for tableName={}", tableName); + + try { + ParamChecker.notEmpty(tableName, "table name cannot be null"); + final String jsonResult = lineageService.getInputsGraph(tableName); + + JSONObject response = new JSONObject(); + response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId()); + response.put("tableName", tableName); + response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult)); + + return Response.ok(response).build(); + } catch (EntityNotFoundException e) { + LOG.error("table entity not found for {}", tableName, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (DiscoveryException | IllegalArgumentException e) { + LOG.error("Unable to get lineage inputs graph for table {}", tableName, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to get lineage inputs graph for table {}", tableName, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** + * Returns the outputs graph for a given entity. + * + * @param tableName table name + */ + @GET + @Path("table/{tableName}/outputs/graph") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response outputsGraph(@Context HttpServletRequest request, + @PathParam("tableName") String tableName) { + LOG.info("Fetching lineage outputs graph for tableName={}", tableName); + + try { + ParamChecker.notEmpty(tableName, "table name cannot be null"); + final String jsonResult = lineageService.getOutputsGraph(tableName); + + JSONObject response = new JSONObject(); + response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId()); + response.put("tableName", tableName); + response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult)); + + return Response.ok(response).build(); + } catch (EntityNotFoundException e) { + LOG.error("table entity not found for {}", tableName, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (DiscoveryException | IllegalArgumentException e) { + LOG.error("Unable to get lineage outputs graph for table {}", tableName, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to get lineage outputs graph for table {}", tableName, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** + * Return the schema for the given tableName. + * + * @param tableName table name + */ + @GET + @Path("table/{tableName}/schema") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response schema(@Context HttpServletRequest request, + @PathParam("tableName") String tableName) { + LOG.info("Fetching schema for tableName={}", tableName); + + try { + ParamChecker.notEmpty(tableName, "table name cannot be null"); + final String jsonResult = lineageService.getSchema(tableName); + + JSONObject response = new JSONObject(); + response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId()); + response.put("tableName", tableName); + response.put(MetadataServiceClient.RESULTS, new JSONObject(jsonResult)); + + return Response.ok(response).build(); + } catch (EntityNotFoundException e) { + LOG.error("table entity not found for {}", tableName, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.NOT_FOUND)); + } catch (DiscoveryException | IllegalArgumentException e) { + LOG.error("Unable to get schema for table {}", tableName, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable e) { + LOG.error("Unable to get schema for table {}", tableName, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java new file mode 100755 index 0000000..9b0b0e8 --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java @@ -0,0 +1,322 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.resources; + +import com.google.common.base.Preconditions; +import org.apache.atlas.MetadataServiceClient; +import org.apache.atlas.ParamChecker; +import org.apache.atlas.discovery.DiscoveryException; +import org.apache.atlas.discovery.DiscoveryService; +import org.apache.atlas.web.util.Servlets; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Response; +import java.util.List; +import java.util.Map; + +/** + * Jersey Resource for metadata operations. + */ +@Path("discovery") +@Singleton +public class MetadataDiscoveryResource { + + private static final Logger LOG = LoggerFactory.getLogger(EntityResource.class); + private static final String QUERY_TYPE_DSL = "dsl"; + private static final String QUERY_TYPE_GREMLIN = "gremlin"; + private static final String QUERY_TYPE_FULLTEXT = "full-text"; + + private final DiscoveryService discoveryService; + + /** + * Created by the Guice ServletModule and injected with the + * configured DiscoveryService. + * + * @param discoveryService metadata service handle + */ + @Inject + public MetadataDiscoveryResource(DiscoveryService discoveryService) { + this.discoveryService = discoveryService; + } + + /** + * Search using a given query. + * + * @param query search query in raw gremlin or DSL format falling back to full text. + * @return JSON representing the type and results. + */ + @GET + @Path("search") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response search(@QueryParam("query") String query) { + JSONObject response; + try { // fall back to dsl + ParamChecker.notEmpty(query, "query cannot be null"); + + if (query.startsWith("g.")) { // raw gremlin query + return searchUsingGremlinQuery(query); + } + + final String jsonResultStr = discoveryService.searchByDSL(query); + response = new DSLJSONResponseBuilder().results(jsonResultStr) + .query(query) + .build(); + + } catch (IllegalArgumentException e) { + LOG.error("Unable to get entity list for empty query", e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch (Throwable throwable) { + LOG.error("Unable to get entity list for query {} using dsl", query, throwable); + + try { //fall back to full-text + final String jsonResultStr = discoveryService.searchByFullText(query); + response = new FullTextJSonResponseBuilder().results(jsonResultStr) + .query(query) + .build(); + + } catch (DiscoveryException | IllegalArgumentException e) { + LOG.error("Unable to get entity list for query {}", query, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch(Throwable e) { + LOG.error("Unable to get entity list for query {}", query, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + return Response.ok(response) + .build(); + + } + + /** + * Search using query DSL format. + * + * @param dslQuery search query in DSL format. + * @return JSON representing the type and results. + */ + @GET + @Path("search/dsl") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response searchUsingQueryDSL(@QueryParam("query") String dslQuery) { + try { + ParamChecker.notEmpty(dslQuery, "dslQuery cannot be null"); + final String jsonResultStr = discoveryService.searchByDSL(dslQuery); + + JSONObject response = new DSLJSONResponseBuilder().results(jsonResultStr) + .query(dslQuery) + .build(); + + return Response.ok(response) + .build(); + } catch (DiscoveryException | IllegalArgumentException e) { + LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch(Throwable e) { + LOG.error("Unable to get entity list for dslQuery {}", dslQuery, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** + * Search using raw gremlin query format. + * + * @param gremlinQuery search query in raw gremlin format. + * @return JSON representing the type and results. + */ + @GET + @Path("search/gremlin") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response searchUsingGremlinQuery(@QueryParam("query") String gremlinQuery) { + try { + ParamChecker.notEmpty(gremlinQuery, "gremlinQuery cannot be null or empty"); + final List<Map<String, String>> results = discoveryService + .searchByGremlin(gremlinQuery); + + JSONObject response = new JSONObject(); + response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId()); + response.put(MetadataServiceClient.QUERY, gremlinQuery); + response.put(MetadataServiceClient.QUERY_TYPE, QUERY_TYPE_GREMLIN); + + JSONArray list = new JSONArray(); + for (Map<String, String> result : results) { + list.put(new JSONObject(result)); + } + response.put(MetadataServiceClient.RESULTS, list); + response.put(MetadataServiceClient.COUNT, list.length()); + + return Response.ok(response) + .build(); + } catch (DiscoveryException | IllegalArgumentException e) { + LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch(Throwable e) { + LOG.error("Unable to get entity list for gremlinQuery {}", gremlinQuery, e); + throw new WebApplicationException( + Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + /** + * Search using full text search. + * + * @param query search query. + * @return JSON representing the type and results. + */ + @GET + @Path("search/fulltext") + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Produces(Servlets.JSON_MEDIA_TYPE) + public Response searchUsingFullText(@QueryParam("query") String query) { + try { + ParamChecker.notEmpty(query, "query cannot be null or empty"); + final String jsonResultStr = discoveryService.searchByFullText(query); + JSONArray rowsJsonArr = new JSONArray(jsonResultStr); + + JSONObject response = new FullTextJSonResponseBuilder().results(rowsJsonArr) + .query(query) + .build(); + return Response.ok(response) + .build(); + } catch (DiscoveryException | IllegalArgumentException e) { + LOG.error("Unable to get entity list for query {}", query, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); + } catch(Throwable e) { + LOG.error("Unable to get entity list for query {}", query, e); + throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); + } + } + + private class JsonResponseBuilder { + + protected int count = 0; + protected String query; + protected String queryType; + protected JSONObject response; + + JsonResponseBuilder() { + this.response = new JSONObject(); + } + + protected JsonResponseBuilder count(int count) { + this.count = count; + return this; + } + + public JsonResponseBuilder query(String query) { + this.query = query; + return this; + } + + public JsonResponseBuilder queryType(String queryType) { + this.queryType = queryType; + return this; + } + + protected JSONObject build() throws JSONException { + + Preconditions.checkNotNull(query, "Query cannot be null"); + Preconditions.checkNotNull(queryType, "Query Type must be specified"); + Preconditions.checkArgument(count >= 0, "Search Result count should be > 0"); + + response.put(MetadataServiceClient.REQUEST_ID, Servlets.getRequestId()); + response.put(MetadataServiceClient.QUERY, query); + response.put(MetadataServiceClient.QUERY_TYPE, queryType); + response.put(MetadataServiceClient.COUNT, count); + return response; + } + } + + private class DSLJSONResponseBuilder extends JsonResponseBuilder { + + DSLJSONResponseBuilder() { + super(); + } + + private JSONObject dslResults; + + public DSLJSONResponseBuilder results(JSONObject dslResults) { + this.dslResults = dslResults; + return this; + } + + public DSLJSONResponseBuilder results(String dslResults) throws JSONException { + return results(new JSONObject(dslResults)); + } + + @Override + public JSONObject build() throws JSONException { + Preconditions.checkNotNull(dslResults); + JSONArray rowsJsonArr = dslResults.getJSONArray(MetadataServiceClient.ROWS); + count(rowsJsonArr.length()); + queryType(QUERY_TYPE_DSL); + JSONObject response = super.build(); + response.put(MetadataServiceClient.RESULTS, dslResults); + return response; + } + + } + + private class FullTextJSonResponseBuilder extends JsonResponseBuilder { + + private JSONArray fullTextResults; + + public FullTextJSonResponseBuilder results(JSONArray fullTextResults) { + this.fullTextResults = fullTextResults; + return this; + } + + public FullTextJSonResponseBuilder results(String dslResults) throws JSONException { + return results(new JSONArray(dslResults)); + } + + public FullTextJSonResponseBuilder() { + super(); + } + + @Override + public JSONObject build() throws JSONException { + Preconditions.checkNotNull(fullTextResults); + count(fullTextResults.length()); + queryType(QUERY_TYPE_FULLTEXT); + + JSONObject response = super.build(); + response.put(MetadataServiceClient.RESULTS, fullTextResults); + return response; + } + } +}
