http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLevelConverter.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLevelConverter.java b/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLevelConverter.java new file mode 100644 index 0000000..14bbd4c --- /dev/null +++ b/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLevelConverter.java @@ -0,0 +1,99 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.common.logging.julbridge; + +import java.util.logging.Level; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * An utility class to convert between JUL and Log4j Levels. Mapping is as follows: + * <ul> + * <li>FINEST <-> TRACE</li> + * <li>FINER -> DEBUG</li> + * <li>FINE <-> DEBUG</li> + * <li>INFO <-> INFO</li> + * <li>WARNING <-> WARN</li> + * <li>SEVERE <-> ERROR</li> + * <li>SEVERE <- FATAL</li> + * </ul> + * + * Unknowns levels are mapped to FINE/DEBUG + */ +public class JULBridgeLevelConverter { + + private JULBridgeLevelConverter() {} + + /** + * Converts a JUL level into a Log4j level. + * + * @param level the JUL level to convert + * @return a Log4j level + * @throws NullPointerException if level is null + */ + public static org.apache.log4j.Level toLog4jLevel(Level level) { + checkNotNull(level); + + if (level == Level.FINEST) { + return org.apache.log4j.Level.TRACE; + } else if (level == Level.FINER) { + return org.apache.log4j.Level.DEBUG; + } else if (level == Level.FINE) { + return org.apache.log4j.Level.DEBUG; + } else if (level == Level.INFO) { + return org.apache.log4j.Level.INFO; + } else if (level == Level.WARNING) { + return org.apache.log4j.Level.WARN; + } else if (level == Level.SEVERE) { + return org.apache.log4j.Level.ERROR; + } else if (level == Level.ALL) { + return org.apache.log4j.Level.ALL; + } else if (level == Level.OFF) { + return org.apache.log4j.Level.OFF; + } + + return org.apache.log4j.Level.DEBUG; + } + + /** + * Converts a Log4j level into a JUL level. + * + * @param level the Log4j level to convert + * @return a JUL level + * @throws NullPointerException if level is null + */ + public static Level fromLog4jLevel(org.apache.log4j.Level level) { + checkNotNull(level); + + if (level == org.apache.log4j.Level.TRACE) { + return Level.FINEST; + } else if (level == org.apache.log4j.Level.DEBUG) { + return Level.FINE; + } else if (level == org.apache.log4j.Level.INFO) { + return Level.INFO; + } else if (level == org.apache.log4j.Level.WARN) { + return Level.WARNING; + } else if (level == org.apache.log4j.Level.ERROR) { + return Level.SEVERE; + } else if (level == org.apache.log4j.Level.FATAL) { + return Level.SEVERE; + } else if (level == org.apache.log4j.Level.ALL) { + return Level.ALL; + } else if (level == org.apache.log4j.Level.OFF) { + return Level.OFF; + } + + return Level.FINE; + } +}
http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLogManager.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLogManager.java b/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLogManager.java new file mode 100644 index 0000000..adf1a83 --- /dev/null +++ b/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLogManager.java @@ -0,0 +1,87 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.common.logging.julbridge; + +import java.util.logging.LogManager; +import java.util.logging.Logger; + +import org.apache.log4j.spi.LoggerRepository; + +/** + * A JUL LogManager which takes over the logging configuration and redirects all messages to Log4j. + * + * The approach is inspired by the apache-jul-log4j-bridge project from Paul Smith + * (<psm...@apache.org>) and available at <a + * href="http://people.apache.org/~psmith/logging.apache.org/sandbox/jul-log4j-bridge/"; /> + * + * During initialization, it resets configuration and adds a default handler to the root logger to + * perform the redirection. It also sets the root logger level to the Log4j repository threshold. + * This implies that Log4j is properly configured before this manager is taking over. + * + * To install this log manager, simply add the following property to the java command line: + * <code>-Djava.util.logging.manager=com.twitter.common.logging.julbridge.JULBridgeLogManager</code> + * + * It is possible to configure using extended location information (source filename and line info) + * by adding the following property to the java command line: + * <code>-Dcom.twitter.common.logging.julbridge.use-extended-location-info=true</code> + * + */ +public final class JULBridgeLogManager extends LogManager { + /** + * System property name to control if log messages sent from JUL to log4j should contain + * extended location information. + * + * Set @value to true to add source filename and line number to each message. + */ + public static final String USE_EXTENDED_LOCATION_INFO_PROPERTYNAME = + "com.twitter.common.logging.julbridge.use-extended-location-info"; + + /* + * LogManager requires a public no-arg constructor to be present so a new instance can be created + * when configured using the system property. A private constructor will throw an exception. + */ + public JULBridgeLogManager() {} + + @Override + public void readConfiguration() { + assimilate(org.apache.log4j.LogManager.getLoggerRepository()); + } + + /** + * Assimilates an existing JUL log manager. Equivalent to calling + * {@link #assimilate(LoggerRepository)} with <code>LogManager.getLoggerRepository</code>. + */ + public static void assimilate() { + assimilate(org.apache.log4j.LogManager.getLoggerRepository()); + } + + /** + * Assimilates an existing JUL log manager. + * + * It resets the manager configuration, and adds a bridge handler to the root logger. Messages are + * redirected to the specified Log4j logger repository. + * + * @param loggerRepository the Log4j logger repository to use to redirect messages + */ + public static void assimilate(LoggerRepository loggerRepository) { + LogManager.getLogManager().reset(); + + boolean withExtendedLocationInfos = + Boolean.getBoolean(USE_EXTENDED_LOCATION_INFO_PROPERTYNAME); + + Logger rootLogger = Logger.getLogger(""); + rootLogger.setLevel(JULBridgeLevelConverter.fromLog4jLevel(loggerRepository.getThreshold())); + rootLogger.addHandler(new JULBridgeHandler(loggerRepository, withExtendedLocationInfos)); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/logging/log4j/GlogLayout.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/logging/log4j/GlogLayout.java b/commons/src/main/java/org/apache/aurora/common/logging/log4j/GlogLayout.java new file mode 100644 index 0000000..1a90ded --- /dev/null +++ b/commons/src/main/java/org/apache/aurora/common/logging/log4j/GlogLayout.java @@ -0,0 +1,98 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.common.logging.log4j; + +import com.google.common.collect.ImmutableMap; + +import org.apache.aurora.common.logging.Glog; +import org.apache.log4j.Layout; +import org.apache.log4j.Level; +import org.apache.log4j.spi.LocationInfo; +import org.apache.log4j.spi.LoggingEvent; +import org.apache.log4j.spi.ThrowableInformation; + +/** + * Log4j Layout to match the format generated by glog. + * + * @see Glog + */ +public class GlogLayout extends Layout implements Glog.Formatter<LoggingEvent> { + + private static final ImmutableMap<Level, Glog.Level> LEVEL_LABELS = + ImmutableMap.<Level, Glog.Level>builder() + .put(Level.TRACE, Glog.Level.DEBUG) + .put(Level.DEBUG, Glog.Level.DEBUG) + .put(Level.INFO, Glog.Level.INFO) + .put(Level.WARN, Glog.Level.WARNING) + .put(Level.ERROR, Glog.Level.ERROR) + .put(Level.FATAL, Glog.Level.FATAL) + .build(); + + @Override + public String format(LoggingEvent record) { + return Glog.formatRecord(this, record); + } + + @Override + public boolean ignoresThrowable() { + return false; // We handle stack trace formatting. + } + + @Override + public void activateOptions() { + // We use no options + } + + @Override + public String getMessage(LoggingEvent record) { + return record.getRenderedMessage(); + } + + @Override + public String getClassName(LoggingEvent record) { + LocationInfo locationInformation = record.getLocationInformation(); + return (locationInformation != null) + ? locationInformation.getClassName() + : null; + } + + @Override + public String getMethodName(LoggingEvent record) { + LocationInfo locationInformation = record.getLocationInformation(); + return (locationInformation != null) + ? record.getLocationInformation().getMethodName() + : null; + } + + @Override + public Glog.Level getLevel(LoggingEvent record) { + return LEVEL_LABELS.get(record.getLevel()); + } + + @Override + public long getTimeStamp(LoggingEvent record) { + return record.getTimeStamp(); + } + + @Override + public long getThreadId(LoggingEvent record) { + return Thread.currentThread().getId(); + } + + @Override + public Throwable getThrowable(LoggingEvent record) { + ThrowableInformation throwableInformation = record.getThrowableInformation(); + return throwableInformation != null ? throwableInformation.getThrowable() : null; + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/Environment.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/net/Environment.java b/commons/src/main/java/org/apache/aurora/common/net/Environment.java new file mode 100644 index 0000000..65fd15e --- /dev/null +++ b/commons/src/main/java/org/apache/aurora/common/net/Environment.java @@ -0,0 +1,47 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.common.net; + +/** + * Represents a network environment at the granularity of a datacenter. + * + * @author John Sirois + */ +public interface Environment { + + /** + * Returns the name of this network environment's datacenter. + * + * @return the name of this environment's datacenter + */ + String dcName(); + + /** + * Creates a fully qualified hostname for a given unqualified hostname in the network + * environment's datacenter. Does not confirm that the host exists. + * + * @param hostname The simple hostname to qualify. + * @return The fully qualified hostname. + */ + String fullyQualify(String hostname); + + /** + * Checks if a given {@code hostname} is a valid hostname for a host in this network environment; + * does not guarantee that the host exists in this network environment. + * + * @param hostname The simple hostname to check for membership in this network environment. + * @return {@code true} if the hostname is a valid hostname for this network environment. + */ + boolean contains(String hostname); +} http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/InetSocketAddressHelper.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/net/InetSocketAddressHelper.java b/commons/src/main/java/org/apache/aurora/common/net/InetSocketAddressHelper.java new file mode 100644 index 0000000..7e88b27 --- /dev/null +++ b/commons/src/main/java/org/apache/aurora/common/net/InetSocketAddressHelper.java @@ -0,0 +1,136 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.common.net; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import org.apache.commons.lang.StringUtils; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.Set; + +/** + * A utility that can parse [host]:[port] pairs or :[port] designators into instances of + * {@link java.net.InetSocketAddress}. The literal '*' can be specified for port as an alternative + * to '0' to indicate any local port. + * + * @author John Sirois + */ +public final class InetSocketAddressHelper { + + /** + * A function that uses {@link #parse(String)} to map an endpoint spec to an + * {@link InetSocketAddress}. + */ + public static final Function<String, InetSocketAddress> STR_TO_INET = + new Function<String, InetSocketAddress>() { + @Override public InetSocketAddress apply(String value) { + return parse(value); + } + }; + + /** + * A function that uses {@link #getLocalAddress(int)} to map a local port number to an + * {@link InetSocketAddress}. + * If an {@link UnknownHostException} is thrown, it will be propagated as a + * {@link RuntimeException}. + */ + public static final Function<Integer, InetSocketAddress> INT_TO_INET = + new Function<Integer, InetSocketAddress>() { + @Override public InetSocketAddress apply(Integer port) { + try { + return getLocalAddress(port); + } catch (UnknownHostException e) { + throw Throwables.propagate(e); + } + } + }; + + public static final Function<InetSocketAddress, String> INET_TO_STR = + new Function<InetSocketAddress, String>() { + @Override public String apply(InetSocketAddress addr) { + return InetSocketAddressHelper.toString(addr); + } + }; + + /** + * Attempts to parse an endpoint spec into an InetSocketAddress. + * + * @param value the endpoint spec + * @return a parsed InetSocketAddress + * @throws NullPointerException if {@code value} is {@code null} + * @throws IllegalArgumentException if {@code value} cannot be parsed + */ + public static InetSocketAddress parse(String value) { + Preconditions.checkNotNull(value); + + String[] spec = value.split(":", 2); + if (spec.length != 2) { + throw new IllegalArgumentException("Invalid socket address spec: " + value); + } + + String host = spec[0]; + int port = asPort(spec[1]); + + return StringUtils.isEmpty(host) + ? new InetSocketAddress(port) + : InetSocketAddress.createUnresolved(host, port); + } + + /** + * Attempts to return a usable String given an InetSocketAddress. + * + * @param value the InetSocketAddress. + * @return the String representation of the InetSocketAddress. + */ + public static String toString(InetSocketAddress value) { + Preconditions.checkNotNull(value); + return value.getHostName() + ":" + value.getPort(); + } + + private static int asPort(String port) { + if ("*".equals(port)) { + return 0; + } + try { + return Integer.parseInt(port); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid port: " + port, e); + } + } + + public static InetSocketAddress getLocalAddress(int port) throws UnknownHostException { + String ipAddress = InetAddress.getLocalHost().getHostAddress(); + return new InetSocketAddress(ipAddress, port); + } + + private InetSocketAddressHelper() { + // utility + } + + /** + * Converts backend definitions (in host:port form) a set of socket addresses. + * + * @param backends Backends to convert. + * @return Sockets representing the provided backends. + */ + public static Set<InetSocketAddress> convertToSockets(Iterable<String> backends) { + return Sets.newHashSet(Iterables.transform(backends, STR_TO_INET)); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/ProxyAuthorizer.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/net/ProxyAuthorizer.java b/commons/src/main/java/org/apache/aurora/common/net/ProxyAuthorizer.java new file mode 100644 index 0000000..c5126dc --- /dev/null +++ b/commons/src/main/java/org/apache/aurora/common/net/ProxyAuthorizer.java @@ -0,0 +1,41 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.common.net; + +import org.apache.commons.codec.binary.Base64; + +import java.net.HttpURLConnection; + +/** + * Authorizes http connection for use over the proxy it is built with + * + * @author William Farner + */ +public class ProxyAuthorizer { + private final ProxyConfig config; + + private ProxyAuthorizer(ProxyConfig config) { + this.config = config; + } + + public static ProxyAuthorizer adapt(ProxyConfig config) { + return new ProxyAuthorizer(config); + } + + public void authorize(HttpURLConnection httpCon) { + httpCon.setRequestProperty("Proxy-Authorization", "Basic " + + new String(Base64.encodeBase64(new String(config.getProxyUser() + ":" + + config.getProxyPassword()).getBytes())).trim()); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/ProxyConfig.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/net/ProxyConfig.java b/commons/src/main/java/org/apache/aurora/common/net/ProxyConfig.java new file mode 100644 index 0000000..fa474b5 --- /dev/null +++ b/commons/src/main/java/org/apache/aurora/common/net/ProxyConfig.java @@ -0,0 +1,30 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.common.net; + +import java.net.InetSocketAddress; +import javax.annotation.Nullable; + +/** + * Proxy configuration parameters: proxy address, username, and password. + * + * @author John Corwin + */ +public interface ProxyConfig { + public InetSocketAddress getProxyAddress(); + + public @Nullable String getProxyUser(); + + public @Nullable String getProxyPassword(); +} http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/UrlHelper.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/net/UrlHelper.java b/commons/src/main/java/org/apache/aurora/common/net/UrlHelper.java new file mode 100644 index 0000000..3f06d63 --- /dev/null +++ b/commons/src/main/java/org/apache/aurora/common/net/UrlHelper.java @@ -0,0 +1,156 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.common.net; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author William Farner + */ +public class UrlHelper { + + private static final Logger LOG = Logger.getLogger(UrlHelper.class.getName()); + + /** + * Gets the domain from {@code url}. + * + * @param url A url. + * @return The domain portion of the URL, or {@code null} if the url is invalid. + */ + public static String getDomain(String url) { + try { + return getDomainChecked(url); + } catch (URISyntaxException e) { + LOG.finest("Malformed url: " + url); + return null; + } + } + + /** + * Gets the domain from {@code uri}, and throws an exception if it's not a valid uri. + * + * @param url A url. + * @throws URISyntaxException if url is not a valid {@code URI} + * @return The domain portion of the given url, or {@code null} if the host is undefined. + */ + public static String getDomainChecked(String url) throws URISyntaxException { + Preconditions.checkNotNull(url); + url = addProtocol(url); + return new URI(url).getHost(); + } + + /** + * Gets the path from {@code url}. + * + * @param url A url. + * @return The path portion of the URL, or {@code null} if the url is invalid. + */ + public static String getPath(String url) { + Preconditions.checkNotNull(url); + url = addProtocol(url); + try { + return new URI(url).getPath(); + } catch (URISyntaxException e) { + LOG.info("Malformed url: " + url); + return null; + } + } + + /** + * Strips URL parameters from a url. + * This will remove anything after and including a question mark in the URL. + * + * @param url The URL to strip parameters from. + * @return The original URL with parameters stripped, which will be the original URL if no + * parameters were found. + */ + public static String stripUrlParameters(String url) { + Preconditions.checkNotNull(url); + int paramStartIndex = url.indexOf("?"); + if (paramStartIndex == -1) { + return url; + } else { + return url.substring(0, paramStartIndex); + } + } + + /** + * Convenience method that calls #stripUrlParameters(String) for a URL. + * + * @param url The URL to strip parameters from. + * @return The original URL with parameters stripped, which will be the original URL if no + * parameters were found. + */ + public static String stripUrlParameters(URL url) { + return stripUrlParameters(url.toString()); + } + + private static final Pattern URL_PROTOCOL_REGEX = + Pattern.compile("^https?://", Pattern.CASE_INSENSITIVE); + + /** + * Checks whether a URL specifies its protocol, prepending http if it does not. + * + * @param url The URL to fix. + * @return The URL with the http protocol specified if no protocol was already specified. + */ + public static String addProtocol(String url) { + Preconditions.checkNotNull(url); + Matcher matcher = URL_PROTOCOL_REGEX.matcher(url); + if (!matcher.find()) { + url = "http://"; + url; + } + return url; + } + + /** + * Gets the domain levels for a host. + * For example, sub1.sub2.domain.co.uk would return + * [sub1.sub2.domain.co.uk, sub2.domain.co.uk, domain.co.uk, co.uk, uk]. + * + * + * @param host The host to peel subdomains off from. + * @return The domain levels in this host. + */ + public static List<String> getDomainLevels(String host) { + Preconditions.checkNotNull(host); + + // Automatically include www prefix if not present. + if (!host.startsWith("www")) { + host = "www." + host; + } + + Joiner joiner = Joiner.on("."); + List<String> domainParts = Lists.newLinkedList(Arrays.asList(host.split("\\."))); + List<String> levels = Lists.newLinkedList(); + + while (!domainParts.isEmpty()) { + levels.add(joiner.join(domainParts)); + domainParts.remove(0); + } + + return levels; + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/UrlResolver.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/net/UrlResolver.java b/commons/src/main/java/org/apache/aurora/common/net/UrlResolver.java new file mode 100644 index 0000000..96c5f07 --- /dev/null +++ b/commons/src/main/java/org/apache/aurora/common/net/UrlResolver.java @@ -0,0 +1,446 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.common.net; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.annotation.Nullable; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Functions; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ListenableFutureTask; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import org.apache.aurora.common.base.ExceptionalFunction; +import org.apache.aurora.common.net.UrlResolver.ResolvedUrl.EndState; +import org.apache.aurora.common.quantity.Amount; +import org.apache.aurora.common.quantity.Time; +import org.apache.aurora.common.stats.PrintableHistogram; +import org.apache.aurora.common.util.BackoffStrategy; +import org.apache.aurora.common.util.Clock; +import org.apache.aurora.common.util.TruncatedBinaryBackoff; +import org.apache.aurora.common.util.caching.Cache; +import org.apache.aurora.common.util.caching.LRUCache; + +/** + * Class to aid in resolving URLs by following redirects, which can optionally be performed + * asynchronously using a thread pool. + * + * @author William Farner + */ +public class UrlResolver { + private static final Logger LOG = Logger.getLogger(UrlResolver.class.getName()); + + private static final String TWITTER_UA = "Twitterbot/0.1"; + private static final UrlResolverUtil URL_RESOLVER = + new UrlResolverUtil(Functions.constant(TWITTER_UA)); + + private static final ExceptionalFunction<String, String, IOException> RESOLVER = + new ExceptionalFunction<String, String, IOException>() { + @Override public String apply(String url) throws IOException { + return URL_RESOLVER.getEffectiveUrl(url, null); + } + }; + + private static ExceptionalFunction<String, String, IOException> + getUrlResolver(final @Nullable ProxyConfig proxyConfig) { + if (proxyConfig != null) { + return new ExceptionalFunction<String, String, IOException>() { + @Override public String apply(String url) throws IOException { + return URL_RESOLVER.getEffectiveUrl(url, proxyConfig); + } + }; + } else { + return RESOLVER; + } + } + + private final ExceptionalFunction<String, String, IOException> resolver; + private final int maxRedirects; + + // Tracks the number of active tasks (threads in use). + private final Semaphore poolEntrySemaphore; + private final Integer threadPoolSize; + + // Helps with signaling the handler. + private final Executor handlerExecutor; + + // Manages the thread pool and task execution. + private ExecutorService executor; + + // Cache to store resolved URLs. + private final Cache<String, String> urlCache = LRUCache.<String, String>builder() + .maxSize(10000) + .makeSynchronized(true) + .build(); + + // Variables to track connection/request stats. + private AtomicInteger requestCount = new AtomicInteger(0); + private AtomicInteger cacheHits = new AtomicInteger(0); + private AtomicInteger failureCount = new AtomicInteger(0); + // Tracks the time (in milliseconds) required to resolve URLs. + private final PrintableHistogram urlResolutionTimesMs = new PrintableHistogram( + 1, 5, 10, 25, 50, 75, 100, 150, 200, 250, 300, 500, 750, 1000, 1500, 2000); + + private final Clock clock; + private final BackoffStrategy backoffStrategy; + + @VisibleForTesting + UrlResolver(Clock clock, BackoffStrategy backoffStrategy, + ExceptionalFunction<String, String, IOException> resolver, int maxRedirects) { + this(clock, backoffStrategy, resolver, maxRedirects, null); + } + + /** + * Creates a new asynchronous URL resolver. A thread pool will be used to resolve URLs, and + * resolved URLs will be announced via {@code handler}. + * + * @param maxRedirects The maximum number of HTTP redirects to follow. + * @param threadPoolSize The number of threads to use for resolving URLs. + * @param proxyConfig The proxy settings with which to make the HTTP request, or null for the + * default configured proxy. + */ + public UrlResolver(int maxRedirects, int threadPoolSize, @Nullable ProxyConfig proxyConfig) { + this(Clock.SYSTEM_CLOCK, + new TruncatedBinaryBackoff(Amount.of(100L, Time.MILLISECONDS), Amount.of(1L, Time.SECONDS)), + getUrlResolver(proxyConfig), maxRedirects, threadPoolSize); + } + + public UrlResolver(int maxRedirects, int threadPoolSize) { + this(maxRedirects, threadPoolSize, null); + } + + private UrlResolver(Clock clock, BackoffStrategy backoffStrategy, + ExceptionalFunction<String, String, IOException> resolver, int maxRedirects, + @Nullable Integer threadPoolSize) { + this.clock = clock; + this.backoffStrategy = backoffStrategy; + this.resolver = resolver; + this.maxRedirects = maxRedirects; + + if (threadPoolSize != null) { + this.threadPoolSize = threadPoolSize; + Preconditions.checkState(threadPoolSize > 0); + poolEntrySemaphore = new Semaphore(threadPoolSize); + + // Start up the thread pool. + reset(); + + // Executor to send notifications back to the handler. This also needs to be + // a daemon thread. + handlerExecutor = + Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setDaemon(true).build()); + } else { + this.threadPoolSize = null; + poolEntrySemaphore = null; + handlerExecutor = null; + } + } + + public Future<ResolvedUrl> resolveUrlAsync(final String url, final ResolvedUrlHandler handler) { + Preconditions.checkNotNull( + "Asynchronous URL resolution cannot be performed without a valid handler.", handler); + + try { + poolEntrySemaphore.acquire(); + } catch (InterruptedException e) { + LOG.log(Level.SEVERE, "Interrupted while waiting for thread to resolve URL: " + url, e); + return null; + } + final ListenableFutureTask<ResolvedUrl> future = + ListenableFutureTask.create( + new Callable<ResolvedUrl>() { + @Override public ResolvedUrl call() { + return resolveUrl(url); + } + }); + + future.addListener(new Runnable() { + @Override public void run() { + try { + handler.resolved(future); + } finally { + poolEntrySemaphore.release(); + } + } + }, handlerExecutor); + + executor.execute(future); + return future; + } + + private void logThreadpoolInfo() { + LOG.info("Shutting down thread pool, available permits: " + + poolEntrySemaphore.availablePermits()); + LOG.info("Queued threads? " + poolEntrySemaphore.hasQueuedThreads()); + LOG.info("Queue length: " + poolEntrySemaphore.getQueueLength()); + } + + public void reset() { + Preconditions.checkState(threadPoolSize != null); + if (executor != null) { + Preconditions.checkState(executor.isShutdown(), + "The thread pool must be shut down before resetting."); + Preconditions.checkState(executor.isTerminated(), "There may still be pending async tasks."); + } + + // Create a thread pool with daemon threads, so that they may be terminated when no + // application threads are running. + executor = Executors.newFixedThreadPool(threadPoolSize, + new ThreadFactoryBuilder().setDaemon(true).setNameFormat("UrlResolver[%d]").build()); + } + + /** + * Terminates the thread pool, waiting at most {@code waitSeconds} for active threads to complete. + * After this method is called, no more URLs may be submitted for resolution. + * + * @param waitSeconds The number of seconds to wait for active threads to complete. + */ + public void clearAsyncTasks(int waitSeconds) { + Preconditions.checkState(threadPoolSize != null, + "finish() should not be called on a synchronous URL resolver."); + + logThreadpoolInfo(); + executor.shutdown(); // Disable new tasks from being submitted. + try { + // Wait a while for existing tasks to terminate + if (!executor.awaitTermination(waitSeconds, TimeUnit.SECONDS)) { + LOG.info("Pool did not terminate, forcing shutdown."); + logThreadpoolInfo(); + List<Runnable> remaining = executor.shutdownNow(); + LOG.info("Tasks still running: " + remaining); + // Wait a while for tasks to respond to being cancelled + if (!executor.awaitTermination(waitSeconds, TimeUnit.SECONDS)) { + LOG.warning("Pool did not terminate."); + logThreadpoolInfo(); + } + } + } catch (InterruptedException e) { + LOG.log(Level.WARNING, "Interrupted while waiting for threadpool to finish.", e); + // (Re-)Cancel if current thread also interrupted + executor.shutdownNow(); + // Preserve interrupt status + Thread.currentThread().interrupt(); + } + } + + /** + * Resolves a URL synchronously. + * + * @param url The URL to resolve. + * @return The resolved URL. + */ + public ResolvedUrl resolveUrl(String url) { + ResolvedUrl resolvedUrl = new ResolvedUrl(); + resolvedUrl.setStartUrl(url); + + String cached = urlCache.get(url); + if (cached != null) { + cacheHits.incrementAndGet(); + resolvedUrl.setNextResolve(cached); + resolvedUrl.setEndState(EndState.CACHED); + return resolvedUrl; + } + + String currentUrl = url; + long backoffMs = 0L; + String next = null; + for (int i = 0; i < maxRedirects; i++) { + try { + next = resolveOnce(currentUrl); + + // If there was a 4xx or a 5xx, we''ll get a null back, so we pretend like we never advanced + // to allow for a retry within the redirect limit. + // TODO(John Sirois): we really need access to the return code here to do the right thing; ie: + // retry for internal server errors but probably not for unauthorized + if (next == null) { + if (i < maxRedirects - 1) { // don't wait if we're about to exit the loop + backoffMs = backoffStrategy.calculateBackoffMs(backoffMs); + try { + clock.waitFor(backoffMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException( + "Interrupted waiting to retry a failed resolution for: " + currentUrl, e); + } + } + continue; + } + + backoffMs = 0L; + if (next.equals(currentUrl)) { + // We've reached the end of the redirect chain. + resolvedUrl.setEndState(EndState.REACHED_LANDING); + urlCache.put(url, currentUrl); + for (String intermediateUrl : resolvedUrl.getIntermediateUrls()) { + urlCache.put(intermediateUrl, currentUrl); + } + return resolvedUrl; + } else if (!url.equals(next)) { + resolvedUrl.setNextResolve(next); + } + currentUrl = next; + } catch (IOException e) { + LOG.log(Level.INFO, "Failed to resolve url: " + url, e); + resolvedUrl.setEndState(EndState.ERROR); + return resolvedUrl; + } + } + + resolvedUrl.setEndState(next == null || url.equals(currentUrl) ? EndState.ERROR + : EndState.REDIRECT_LIMIT); + return resolvedUrl; + } + + /** + * Resolves a url, following at most one redirect. Thread-safe. + * + * @param url The URL to resolve. + * @return The result of following the URL through at most one redirect or null if the url could + * not be followed + * @throws IOException If an error occurs while resolving the URL. + */ + private String resolveOnce(String url) throws IOException { + requestCount.incrementAndGet(); + + String resolvedUrl = urlCache.get(url); + if (resolvedUrl != null) { + cacheHits.incrementAndGet(); + return resolvedUrl; + } + + try { + long startTimeMs = System.currentTimeMillis(); + resolvedUrl = resolver.apply(url); + if (resolvedUrl == null) { + return null; + } + + urlCache.put(url, resolvedUrl); + + synchronized (urlResolutionTimesMs) { + urlResolutionTimesMs.addValue(System.currentTimeMillis() - startTimeMs); + } + return resolvedUrl; + } catch (IOException e) { + failureCount.incrementAndGet(); + throw e; + } + } + + @Override + public String toString() { + return String.format("Cache: %s\nFailed requests: %d,\nResolution Times: %s", + urlCache, failureCount.get(), + urlResolutionTimesMs.toString()); + } + + /** + * Class to wrap the result of a URL resolution. + */ + public static class ResolvedUrl { + public enum EndState { + REACHED_LANDING, + ERROR, + CACHED, + REDIRECT_LIMIT + } + + private String startUrl; + private final List<String> resolveChain; + private EndState endState; + + public ResolvedUrl() { + resolveChain = Lists.newArrayList(); + } + + @VisibleForTesting + public ResolvedUrl(EndState endState, String startUrl, String... resolveChain) { + this.endState = endState; + this.startUrl = startUrl; + this.resolveChain = Lists.newArrayList(resolveChain); + } + + public String getStartUrl() { + return startUrl; + } + + void setStartUrl(String startUrl) { + this.startUrl = startUrl; + } + + /** + * Returns the last URL resolved following a redirect chain, or null if the startUrl is a + * landing URL. + */ + public String getEndUrl() { + return resolveChain.isEmpty() ? null : Iterables.getLast(resolveChain); + } + + void setNextResolve(String endUrl) { + this.resolveChain.add(endUrl); + } + + /** + * Returns any immediate URLs encountered on the resolution chain. If the startUrl redirects + * directly to the endUrl or they are the same the imtermediate URLs will be empty. + */ + public Iterable<String> getIntermediateUrls() { + return resolveChain.size() <= 1 ? ImmutableList.<String>of() + : resolveChain.subList(0, resolveChain.size() - 1); + } + + public EndState getEndState() { + return endState; + } + + void setEndState(EndState endState) { + this.endState = endState; + } + + public String toString() { + return String.format("%s -> %s [%s, %d redirects]", + startUrl, Joiner.on(" -> ").join(resolveChain), endState, resolveChain.size()); + } + } + + /** + * Interface to use for notifying the caller of resolved URLs. + */ + public interface ResolvedUrlHandler { + /** + * Signals that a URL has been resolved to its target. The implementation of this method must + * be thread safe. + * + * @param future The future that has finished resolving a URL. + */ + public void resolved(Future<ResolvedUrl> future); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/UrlResolverUtil.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/net/UrlResolverUtil.java b/commons/src/main/java/org/apache/aurora/common/net/UrlResolverUtil.java new file mode 100644 index 0000000..4b95bb7 --- /dev/null +++ b/commons/src/main/java/org/apache/aurora/common/net/UrlResolverUtil.java @@ -0,0 +1,148 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.aurora.common.net; + +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.google.common.base.Preconditions; +import org.apache.aurora.common.base.MorePreconditions; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.Proxy; +import java.net.Proxy.Type; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Map; +import java.util.logging.Logger; +import javax.annotation.Nullable; + +/** + * A utility that can resolve HTTP urls. + * + * @author John Sirois + */ +class UrlResolverUtil { + + private static final Logger LOG = Logger.getLogger(UrlResolverUtil.class.getName()); + + // Default user-agent string to user for HTTP requests. + private static final String DEFAULT_USER_AGENT = "Lynxy/6.6.6dev.8 libwww-FM/3.14159FM"; + + private static Map<String, String> checkNotBlank(Map<String, String> hostToUserAgent) { + Preconditions.checkNotNull(hostToUserAgent); + MorePreconditions.checkNotBlank(hostToUserAgent.entrySet()); + return hostToUserAgent; + } + + private final Function<? super URL, String> urlToUserAgent; + + UrlResolverUtil(Map<String, String> hostToUserAgent) { + this(Functions.compose(Functions.forMap(checkNotBlank(hostToUserAgent), DEFAULT_USER_AGENT), + new Function<URL, String>() { + @Override public String apply(URL url) { + return url.getHost(); + } + })); + } + + UrlResolverUtil(Function<? super URL, String> urlToUserAgent) { + this.urlToUserAgent = Preconditions.checkNotNull(urlToUserAgent); + } + + /** + * Returns the URL that {@code url} lands on, which will be the result of a 3xx redirect, + * or {@code url} if the url does not redirect using an HTTP 3xx response code. If there is a + * non-2xx or 3xx HTTP response code null is returned. + * + * @param url The URL to follow. + * @return The redirected URL, or {@code url} if {@code url} returns a 2XX response, otherwise + * null + * @throws java.io.IOException If an error occurs while trying to follow the url. + */ + String getEffectiveUrl(String url, @Nullable ProxyConfig proxyConfig) throws IOException { + Preconditions.checkNotNull(url); + // Don't follow https. + if (url.startsWith("https://";)) { + url = url.replace("https://";, "http://";); + } else if (!url.startsWith("http://";)) { + url = "http://"; + url; + } + + URL urlObj = new URL(url); + + HttpURLConnection con; + if (proxyConfig != null) { + Proxy proxy = new Proxy(Type.HTTP, proxyConfig.getProxyAddress()); + con = (HttpURLConnection) urlObj.openConnection(proxy); + ProxyAuthorizer.adapt(proxyConfig).authorize(con); + } else { + con = (HttpURLConnection) urlObj.openConnection(); + } + try { + + // TODO(John Sirois): several commonly tweeted hosts 406 or 400 on HEADs and only work with GETs + // fix the call chain to be able to specify retry-with-GET + con.setRequestMethod("HEAD"); + + con.setUseCaches(true); + con.setConnectTimeout(5000); + con.setReadTimeout(5000); + con.setInstanceFollowRedirects(false); + + // I hate to have to do this, but some URL shorteners don't respond otherwise. + con.setRequestProperty("User-Agent", urlToUserAgent.apply(urlObj)); + try { + con.connect(); + } catch (StringIndexOutOfBoundsException e) { + LOG.info("Got StringIndexOutOfBoundsException when fetching headers for " + url); + return null; + } + + int responseCode = con.getResponseCode(); + switch (responseCode / 100) { + case 2: + return url; + case 3: + String location = con.getHeaderField("Location"); + if (location == null) { + if (responseCode != 304 /* not modified */) { + LOG.info( + String.format("[%d] Location header was null for URL: %s", responseCode, url)); + } + return url; + } + + // HTTP 1.1 spec says this should be an absolute URI, but i see lots of instances where it + // is relative, so we need to check. + try { + String domain = UrlHelper.getDomainChecked(location); + if (domain == null || domain.isEmpty()) { + // This is a relative URI. + location = "http://"; + UrlHelper.getDomain(url) + location; + } + } catch (URISyntaxException e) { + LOG.info("location contained an invalid URI: " + location); + } + + return location; + default: + LOG.info("Failed to resolve url: " + url + " with: " + + responseCode + " -> " + con.getResponseMessage()); + return null; + } + } finally { + con.disconnect(); + } + } +}
