aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zma...@apache.org
Subject [26/51] [partial] aurora git commit: Move packages from com.twitter.common to org.apache.aurora.common
Date Wed, 26 Aug 2015 21:00:16 GMT
http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLevelConverter.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLevelConverter.java b/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLevelConverter.java
new file mode 100644
index 0000000..14bbd4c
--- /dev/null
+++ b/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLevelConverter.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.common.logging.julbridge;
+
+import java.util.logging.Level;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * An utility class to convert between JUL and Log4j Levels. Mapping is as follows:
+ * <ul>
+ * <li>FINEST <-> TRACE</li>
+ * <li>FINER -> DEBUG</li>
+ * <li>FINE <-> DEBUG</li>
+ * <li>INFO <-> INFO</li>
+ * <li>WARNING <-> WARN</li>
+ * <li>SEVERE <-> ERROR</li>
+ * <li>SEVERE <- FATAL</li>
+ * </ul>
+ *
+ * Unknowns levels are mapped to FINE/DEBUG
+ */
+public class JULBridgeLevelConverter {
+
+  private JULBridgeLevelConverter() {}
+
+  /**
+   * Converts a JUL level into a Log4j level.
+   *
+   * @param level the JUL level to convert
+   * @return a Log4j level
+   * @throws NullPointerException if level is null
+   */
+  public static org.apache.log4j.Level toLog4jLevel(Level level) {
+    checkNotNull(level);
+
+    if (level == Level.FINEST) {
+      return org.apache.log4j.Level.TRACE;
+    } else if (level == Level.FINER) {
+      return org.apache.log4j.Level.DEBUG;
+    } else if (level == Level.FINE) {
+      return org.apache.log4j.Level.DEBUG;
+    } else if (level == Level.INFO) {
+      return org.apache.log4j.Level.INFO;
+    } else if (level == Level.WARNING) {
+      return org.apache.log4j.Level.WARN;
+    } else if (level == Level.SEVERE) {
+      return org.apache.log4j.Level.ERROR;
+    } else if (level == Level.ALL) {
+      return org.apache.log4j.Level.ALL;
+    } else if (level == Level.OFF) {
+      return org.apache.log4j.Level.OFF;
+    }
+
+    return org.apache.log4j.Level.DEBUG;
+  }
+
+  /**
+   * Converts a Log4j level into a JUL level.
+   *
+   * @param level the Log4j level to convert
+   * @return a JUL level
+   * @throws NullPointerException if level is null
+   */
+  public static Level fromLog4jLevel(org.apache.log4j.Level level) {
+    checkNotNull(level);
+
+    if (level == org.apache.log4j.Level.TRACE) {
+      return Level.FINEST;
+    } else if (level == org.apache.log4j.Level.DEBUG) {
+      return Level.FINE;
+    } else if (level == org.apache.log4j.Level.INFO) {
+      return Level.INFO;
+    } else if (level == org.apache.log4j.Level.WARN) {
+      return Level.WARNING;
+    } else if (level == org.apache.log4j.Level.ERROR) {
+      return Level.SEVERE;
+    } else if (level == org.apache.log4j.Level.FATAL) {
+      return Level.SEVERE;
+    } else if (level == org.apache.log4j.Level.ALL) {
+      return Level.ALL;
+    } else if (level == org.apache.log4j.Level.OFF) {
+      return Level.OFF;
+    }
+
+    return Level.FINE;
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLogManager.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLogManager.java b/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLogManager.java
new file mode 100644
index 0000000..adf1a83
--- /dev/null
+++ b/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLogManager.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.common.logging.julbridge;
+
+import java.util.logging.LogManager;
+import java.util.logging.Logger;
+
+import org.apache.log4j.spi.LoggerRepository;
+
+/**
+ * A JUL LogManager which takes over the logging configuration and redirects all messages to Log4j.
+ *
+ * The approach is inspired by the apache-jul-log4j-bridge project from Paul Smith
+ * (<psmith@apache.org>) and available at <a
+ * href="http://people.apache.org/~psmith/logging.apache.org/sandbox/jul-log4j-bridge/" />
+ *
+ * During initialization, it resets configuration and adds a default handler to the root logger to
+ * perform the redirection. It also sets the root logger level to the Log4j repository threshold.
+ * This implies that Log4j is properly configured before this manager is taking over.
+ *
+ * To install this log manager, simply add the following property to the java command line:
+ * <code>-Djava.util.logging.manager=com.twitter.common.logging.julbridge.JULBridgeLogManager</code>
+ *
+ * It is possible to configure using extended location information (source filename and line info)
+ * by adding the following property to the java command line:
+ * <code>-Dcom.twitter.common.logging.julbridge.use-extended-location-info=true</code>
+ *
+ */
+public final class JULBridgeLogManager extends LogManager {
+  /**
+   * System property name to control if log messages sent from JUL to log4j should contain
+   * extended location information.
+   *
+   * Set @value to true to add source filename and line number to each message.
+   */
+  public static final String USE_EXTENDED_LOCATION_INFO_PROPERTYNAME =
+      "com.twitter.common.logging.julbridge.use-extended-location-info";
+
+  /*
+   * LogManager requires a public no-arg constructor to be present so a new instance can be created
+   * when configured using the system property. A private constructor will throw an exception.
+   */
+  public JULBridgeLogManager() {}
+
+  @Override
+  public void readConfiguration() {
+    assimilate(org.apache.log4j.LogManager.getLoggerRepository());
+  }
+
+  /**
+   * Assimilates an existing JUL log manager. Equivalent to calling
+   * {@link #assimilate(LoggerRepository)} with <code>LogManager.getLoggerRepository</code>.
+   */
+  public static void assimilate() {
+    assimilate(org.apache.log4j.LogManager.getLoggerRepository());
+  }
+
+  /**
+   * Assimilates an existing JUL log manager.
+   *
+   * It resets the manager configuration, and adds a bridge handler to the root logger. Messages are
+   * redirected to the specified Log4j logger repository.
+   *
+   * @param loggerRepository the Log4j logger repository to use to redirect messages
+   */
+  public static void assimilate(LoggerRepository loggerRepository) {
+    LogManager.getLogManager().reset();
+
+    boolean withExtendedLocationInfos =
+        Boolean.getBoolean(USE_EXTENDED_LOCATION_INFO_PROPERTYNAME);
+
+    Logger rootLogger = Logger.getLogger("");
+    rootLogger.setLevel(JULBridgeLevelConverter.fromLog4jLevel(loggerRepository.getThreshold()));
+    rootLogger.addHandler(new JULBridgeHandler(loggerRepository, withExtendedLocationInfos));
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/logging/log4j/GlogLayout.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/aurora/common/logging/log4j/GlogLayout.java b/commons/src/main/java/org/apache/aurora/common/logging/log4j/GlogLayout.java
new file mode 100644
index 0000000..1a90ded
--- /dev/null
+++ b/commons/src/main/java/org/apache/aurora/common/logging/log4j/GlogLayout.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.common.logging.log4j;
+
+import com.google.common.collect.ImmutableMap;
+
+import org.apache.aurora.common.logging.Glog;
+import org.apache.log4j.Layout;
+import org.apache.log4j.Level;
+import org.apache.log4j.spi.LocationInfo;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.log4j.spi.ThrowableInformation;
+
+/**
+ * Log4j Layout to match the format generated by glog.
+ *
+ * @see Glog
+ */
+public class GlogLayout extends Layout implements Glog.Formatter<LoggingEvent> {
+
+  private static final ImmutableMap<Level, Glog.Level> LEVEL_LABELS =
+      ImmutableMap.<Level, Glog.Level>builder()
+          .put(Level.TRACE, Glog.Level.DEBUG)
+          .put(Level.DEBUG, Glog.Level.DEBUG)
+          .put(Level.INFO, Glog.Level.INFO)
+          .put(Level.WARN, Glog.Level.WARNING)
+          .put(Level.ERROR, Glog.Level.ERROR)
+          .put(Level.FATAL, Glog.Level.FATAL)
+          .build();
+
+  @Override
+  public String format(LoggingEvent record) {
+    return Glog.formatRecord(this, record);
+  }
+
+  @Override
+  public boolean ignoresThrowable() {
+    return false; // We handle stack trace formatting.
+  }
+
+  @Override
+  public void activateOptions() {
+    // We use no options
+  }
+
+  @Override
+  public String getMessage(LoggingEvent record) {
+    return record.getRenderedMessage();
+  }
+
+  @Override
+  public String getClassName(LoggingEvent record) {
+    LocationInfo locationInformation = record.getLocationInformation();
+    return (locationInformation != null)
+        ? locationInformation.getClassName()
+        : null;
+  }
+
+  @Override
+  public String getMethodName(LoggingEvent record) {
+    LocationInfo locationInformation = record.getLocationInformation();
+    return (locationInformation != null)
+        ? record.getLocationInformation().getMethodName()
+        : null;
+  }
+
+  @Override
+  public Glog.Level getLevel(LoggingEvent record) {
+    return LEVEL_LABELS.get(record.getLevel());
+  }
+
+  @Override
+  public long getTimeStamp(LoggingEvent record) {
+    return record.getTimeStamp();
+  }
+
+  @Override
+  public long getThreadId(LoggingEvent record) {
+    return Thread.currentThread().getId();
+  }
+
+  @Override
+  public Throwable getThrowable(LoggingEvent record) {
+    ThrowableInformation throwableInformation = record.getThrowableInformation();
+    return throwableInformation != null ? throwableInformation.getThrowable() : null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/Environment.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/aurora/common/net/Environment.java b/commons/src/main/java/org/apache/aurora/common/net/Environment.java
new file mode 100644
index 0000000..65fd15e
--- /dev/null
+++ b/commons/src/main/java/org/apache/aurora/common/net/Environment.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.common.net;
+
+/**
+ * Represents a network environment at the granularity of a datacenter.
+ *
+ * @author John Sirois
+ */
+public interface Environment {
+
+  /**
+   * Returns the name of this network environment's datacenter.
+   *
+   * @return the name of this environment's datacenter
+   */
+  String dcName();
+
+  /**
+   * Creates a fully qualified hostname for a given unqualified hostname in the network
+   * environment's datacenter.  Does not confirm that the host exists.
+   *
+   * @param hostname The simple hostname to qualify.
+   * @return The fully qualified hostname.
+   */
+  String fullyQualify(String hostname);
+
+  /**
+   * Checks if a given {@code hostname} is a valid hostname for a host in this network environment;
+   * does not guarantee that the host exists in this network environment.
+   *
+   * @param hostname The simple hostname to check for membership in this network environment.
+   * @return {@code true} if the hostname is a valid hostname for this network environment.
+   */
+  boolean contains(String hostname);
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/InetSocketAddressHelper.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/aurora/common/net/InetSocketAddressHelper.java b/commons/src/main/java/org/apache/aurora/common/net/InetSocketAddressHelper.java
new file mode 100644
index 0000000..7e88b27
--- /dev/null
+++ b/commons/src/main/java/org/apache/aurora/common/net/InetSocketAddressHelper.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.common.net;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import org.apache.commons.lang.StringUtils;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.Set;
+
+/**
+ * A utility that can parse [host]:[port] pairs or :[port] designators into instances of
+ * {@link java.net.InetSocketAddress}. The literal '*' can be specified for port as an alternative
+ * to '0' to indicate any local port.
+ *
+ * @author John Sirois
+ */
+public final class InetSocketAddressHelper {
+
+  /**
+   * A function that uses {@link #parse(String)} to map an endpoint spec to an
+   * {@link InetSocketAddress}.
+   */
+  public static final Function<String, InetSocketAddress> STR_TO_INET =
+      new Function<String, InetSocketAddress>() {
+        @Override public InetSocketAddress apply(String value) {
+          return parse(value);
+        }
+      };
+
+  /**
+   * A function that uses {@link #getLocalAddress(int)} to map a local port number to an
+   * {@link InetSocketAddress}.
+   * If an {@link UnknownHostException} is thrown, it will be propagated as a
+   * {@link RuntimeException}.
+   */
+  public static final Function<Integer, InetSocketAddress> INT_TO_INET =
+      new Function<Integer, InetSocketAddress>() {
+        @Override public InetSocketAddress apply(Integer port) {
+          try {
+            return getLocalAddress(port);
+          } catch (UnknownHostException e) {
+            throw Throwables.propagate(e);
+          }
+        }
+      };
+
+  public static final Function<InetSocketAddress, String> INET_TO_STR =
+      new Function<InetSocketAddress, String>() {
+        @Override public String apply(InetSocketAddress addr) {
+          return InetSocketAddressHelper.toString(addr);
+        }
+      };
+
+  /**
+   * Attempts to parse an endpoint spec into an InetSocketAddress.
+   *
+   * @param value the endpoint spec
+   * @return a parsed InetSocketAddress
+   * @throws NullPointerException     if {@code value} is {@code null}
+   * @throws IllegalArgumentException if {@code value} cannot be parsed
+   */
+  public static InetSocketAddress parse(String value) {
+    Preconditions.checkNotNull(value);
+
+    String[] spec = value.split(":", 2);
+    if (spec.length != 2) {
+      throw new IllegalArgumentException("Invalid socket address spec: " + value);
+    }
+
+    String host = spec[0];
+    int port = asPort(spec[1]);
+
+    return StringUtils.isEmpty(host)
+        ? new InetSocketAddress(port)
+        : InetSocketAddress.createUnresolved(host, port);
+  }
+
+  /**
+   * Attempts to return a usable String given an InetSocketAddress.
+   *
+   * @param value the InetSocketAddress.
+   * @return the String representation of the InetSocketAddress.
+   */
+  public static String toString(InetSocketAddress value) {
+    Preconditions.checkNotNull(value);
+    return value.getHostName() + ":" + value.getPort();
+  }
+
+  private static int asPort(String port) {
+    if ("*".equals(port)) {
+      return 0;
+    }
+    try {
+      return Integer.parseInt(port);
+    } catch (NumberFormatException e) {
+      throw new IllegalArgumentException("Invalid port: " + port, e);
+    }
+  }
+
+  public static InetSocketAddress getLocalAddress(int port) throws UnknownHostException {
+    String ipAddress = InetAddress.getLocalHost().getHostAddress();
+    return new InetSocketAddress(ipAddress, port);
+  }
+
+  private InetSocketAddressHelper() {
+    // utility
+  }
+
+  /**
+   * Converts backend definitions (in host:port form) a set of socket addresses.
+   *
+   * @param backends Backends to convert.
+   * @return Sockets representing the provided backends.
+   */
+  public static Set<InetSocketAddress> convertToSockets(Iterable<String> backends) {
+    return Sets.newHashSet(Iterables.transform(backends, STR_TO_INET));
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/ProxyAuthorizer.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/aurora/common/net/ProxyAuthorizer.java b/commons/src/main/java/org/apache/aurora/common/net/ProxyAuthorizer.java
new file mode 100644
index 0000000..c5126dc
--- /dev/null
+++ b/commons/src/main/java/org/apache/aurora/common/net/ProxyAuthorizer.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.common.net;
+
+import org.apache.commons.codec.binary.Base64;
+
+import java.net.HttpURLConnection;
+
+/**
+ * Authorizes http connection for use over the proxy it is built with
+ *
+ * @author William Farner
+ */
+public class ProxyAuthorizer {
+  private final ProxyConfig config;
+
+  private ProxyAuthorizer(ProxyConfig config) {
+    this.config = config;
+  }
+
+  public static ProxyAuthorizer adapt(ProxyConfig config) {
+    return new ProxyAuthorizer(config);
+  }
+
+  public void authorize(HttpURLConnection httpCon) {
+    httpCon.setRequestProperty("Proxy-Authorization", "Basic " +
+        new String(Base64.encodeBase64(new String(config.getProxyUser() + ":" +
+          config.getProxyPassword()).getBytes())).trim());
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/ProxyConfig.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/aurora/common/net/ProxyConfig.java b/commons/src/main/java/org/apache/aurora/common/net/ProxyConfig.java
new file mode 100644
index 0000000..fa474b5
--- /dev/null
+++ b/commons/src/main/java/org/apache/aurora/common/net/ProxyConfig.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.common.net;
+
+import java.net.InetSocketAddress;
+import javax.annotation.Nullable;
+
+/**
+ * Proxy configuration parameters: proxy address, username, and password.
+ *
+ * @author John Corwin
+ */
+public interface ProxyConfig {
+  public InetSocketAddress getProxyAddress();
+
+  public @Nullable String getProxyUser();
+
+  public @Nullable String getProxyPassword();
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/UrlHelper.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/aurora/common/net/UrlHelper.java b/commons/src/main/java/org/apache/aurora/common/net/UrlHelper.java
new file mode 100644
index 0000000..3f06d63
--- /dev/null
+++ b/commons/src/main/java/org/apache/aurora/common/net/UrlHelper.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.common.net;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * @author William Farner
+ */
+public class UrlHelper {
+
+  private static final Logger LOG = Logger.getLogger(UrlHelper.class.getName());
+
+  /**
+   * Gets the domain from {@code url}.
+   *
+   * @param url A url.
+   * @return The domain portion of the URL, or {@code null} if the url is invalid.
+   */
+  public static String getDomain(String url) {
+    try {
+      return getDomainChecked(url);
+    } catch (URISyntaxException e) {
+      LOG.finest("Malformed url: " + url);
+      return null;
+    }
+  }
+
+  /**
+   * Gets the domain from {@code uri}, and throws an exception if it's not a valid uri.
+   *
+   * @param url A url.
+   * @throws URISyntaxException if url is not a valid {@code URI}
+   * @return The domain portion of the given url, or {@code null} if the host is undefined.
+   */
+  public static String getDomainChecked(String url) throws URISyntaxException {
+    Preconditions.checkNotNull(url);
+    url = addProtocol(url);
+    return new URI(url).getHost();
+  }
+
+  /**
+   * Gets the path from {@code url}.
+   *
+   * @param url A url.
+   * @return The path portion of the URL, or {@code null} if the url is invalid.
+   */
+  public static String getPath(String url) {
+    Preconditions.checkNotNull(url);
+    url = addProtocol(url);
+    try {
+      return new URI(url).getPath();
+    } catch (URISyntaxException e) {
+      LOG.info("Malformed url: " + url);
+      return null;
+    }
+  }
+
+  /**
+   * Strips URL parameters from a url.
+   * This will remove anything after and including a question mark in the URL.
+   *
+   * @param url The URL to strip parameters from.
+   * @return The original URL with parameters stripped, which will be the original URL if no
+   *   parameters were found.
+   */
+  public static String stripUrlParameters(String url) {
+    Preconditions.checkNotNull(url);
+    int paramStartIndex = url.indexOf("?");
+    if (paramStartIndex == -1) {
+      return url;
+    } else {
+      return url.substring(0, paramStartIndex);
+    }
+  }
+
+  /**
+   * Convenience method that calls #stripUrlParameters(String) for a URL.
+   *
+   * @param url The URL to strip parameters from.
+   * @return The original URL with parameters stripped, which will be the original URL if no
+   *   parameters were found.
+   */
+  public static String stripUrlParameters(URL url) {
+    return stripUrlParameters(url.toString());
+  }
+
+  private static final Pattern URL_PROTOCOL_REGEX =
+      Pattern.compile("^https?://", Pattern.CASE_INSENSITIVE);
+
+  /**
+   * Checks whether a URL specifies its protocol, prepending http if it does not.
+   *
+   * @param url The URL to fix.
+   * @return The URL with the http protocol specified if no protocol was already specified.
+   */
+  public static String addProtocol(String url) {
+    Preconditions.checkNotNull(url);
+    Matcher matcher = URL_PROTOCOL_REGEX.matcher(url);
+    if (!matcher.find()) {
+      url = "http://" + url;
+    }
+    return url;
+  }
+
+  /**
+   * Gets the domain levels for a host.
+   * For example, sub1.sub2.domain.co.uk would return
+   * [sub1.sub2.domain.co.uk, sub2.domain.co.uk, domain.co.uk, co.uk, uk].
+   *
+   *
+   * @param host The host to peel subdomains off from.
+   * @return The domain levels in this host.
+   */
+  public static List<String> getDomainLevels(String host) {
+    Preconditions.checkNotNull(host);
+
+    // Automatically include www prefix if not present.
+    if (!host.startsWith("www")) {
+      host = "www." + host;
+    }
+
+    Joiner joiner = Joiner.on(".");
+    List<String> domainParts = Lists.newLinkedList(Arrays.asList(host.split("\\.")));
+    List<String> levels = Lists.newLinkedList();
+
+    while (!domainParts.isEmpty()) {
+      levels.add(joiner.join(domainParts));
+      domainParts.remove(0);
+    }
+
+    return levels;
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/UrlResolver.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/aurora/common/net/UrlResolver.java b/commons/src/main/java/org/apache/aurora/common/net/UrlResolver.java
new file mode 100644
index 0000000..96c5f07
--- /dev/null
+++ b/commons/src/main/java/org/apache/aurora/common/net/UrlResolver.java
@@ -0,0 +1,446 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.common.net;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.annotation.Nullable;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Functions;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.ListenableFutureTask;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import org.apache.aurora.common.base.ExceptionalFunction;
+import org.apache.aurora.common.net.UrlResolver.ResolvedUrl.EndState;
+import org.apache.aurora.common.quantity.Amount;
+import org.apache.aurora.common.quantity.Time;
+import org.apache.aurora.common.stats.PrintableHistogram;
+import org.apache.aurora.common.util.BackoffStrategy;
+import org.apache.aurora.common.util.Clock;
+import org.apache.aurora.common.util.TruncatedBinaryBackoff;
+import org.apache.aurora.common.util.caching.Cache;
+import org.apache.aurora.common.util.caching.LRUCache;
+
+/**
+ * Class to aid in resolving URLs by following redirects, which can optionally be performed
+ * asynchronously using a thread pool.
+ *
+ * @author William Farner
+ */
+public class UrlResolver {
+  private static final Logger LOG = Logger.getLogger(UrlResolver.class.getName());
+
+  private static final String TWITTER_UA = "Twitterbot/0.1";
+  private static final UrlResolverUtil URL_RESOLVER =
+      new UrlResolverUtil(Functions.constant(TWITTER_UA));
+
+  private static final ExceptionalFunction<String, String, IOException> RESOLVER =
+      new ExceptionalFunction<String, String, IOException>() {
+        @Override public String apply(String url) throws IOException {
+          return URL_RESOLVER.getEffectiveUrl(url, null);
+        }
+      };
+
+  private static ExceptionalFunction<String, String, IOException>
+      getUrlResolver(final @Nullable ProxyConfig proxyConfig) {
+    if (proxyConfig != null) {
+      return new ExceptionalFunction<String, String, IOException>() {
+        @Override public String apply(String url) throws IOException {
+          return URL_RESOLVER.getEffectiveUrl(url, proxyConfig);
+        }
+      };
+    } else {
+      return RESOLVER;
+    }
+  }
+
+  private final ExceptionalFunction<String, String, IOException> resolver;
+  private final int maxRedirects;
+
+  // Tracks the number of active tasks (threads in use).
+  private final Semaphore poolEntrySemaphore;
+  private final Integer threadPoolSize;
+
+  // Helps with signaling the handler.
+  private final Executor handlerExecutor;
+
+  // Manages the thread pool and task execution.
+  private ExecutorService executor;
+
+  // Cache to store resolved URLs.
+  private final Cache<String, String> urlCache = LRUCache.<String, String>builder()
+      .maxSize(10000)
+      .makeSynchronized(true)
+      .build();
+
+  // Variables to track connection/request stats.
+  private AtomicInteger requestCount = new AtomicInteger(0);
+  private AtomicInteger cacheHits = new AtomicInteger(0);
+  private AtomicInteger failureCount = new AtomicInteger(0);
+  // Tracks the time (in milliseconds) required to resolve URLs.
+  private final PrintableHistogram urlResolutionTimesMs = new PrintableHistogram(
+      1, 5, 10, 25, 50, 75, 100, 150, 200, 250, 300, 500, 750, 1000, 1500, 2000);
+
+  private final Clock clock;
+  private final BackoffStrategy backoffStrategy;
+
+  @VisibleForTesting
+  UrlResolver(Clock clock, BackoffStrategy backoffStrategy,
+      ExceptionalFunction<String, String, IOException> resolver, int maxRedirects) {
+    this(clock, backoffStrategy, resolver, maxRedirects, null);
+  }
+
+  /**
+   * Creates a new asynchronous URL resolver.  A thread pool will be used to resolve URLs, and
+   * resolved URLs will be announced via {@code handler}.
+   *
+   * @param maxRedirects The maximum number of HTTP redirects to follow.
+   * @param threadPoolSize The number of threads to use for resolving URLs.
+   * @param proxyConfig The proxy settings with which to make the HTTP request, or null for the
+   *    default configured proxy.
+   */
+  public UrlResolver(int maxRedirects, int threadPoolSize, @Nullable ProxyConfig proxyConfig) {
+    this(Clock.SYSTEM_CLOCK,
+        new TruncatedBinaryBackoff(Amount.of(100L, Time.MILLISECONDS), Amount.of(1L, Time.SECONDS)),
+        getUrlResolver(proxyConfig), maxRedirects, threadPoolSize);
+  }
+
+  public UrlResolver(int maxRedirects, int threadPoolSize) {
+    this(maxRedirects, threadPoolSize, null);
+  }
+
+  private UrlResolver(Clock clock, BackoffStrategy backoffStrategy,
+      ExceptionalFunction<String, String, IOException> resolver, int maxRedirects,
+      @Nullable Integer threadPoolSize) {
+    this.clock = clock;
+    this.backoffStrategy = backoffStrategy;
+    this.resolver = resolver;
+    this.maxRedirects = maxRedirects;
+
+    if (threadPoolSize != null) {
+      this.threadPoolSize = threadPoolSize;
+      Preconditions.checkState(threadPoolSize > 0);
+      poolEntrySemaphore = new Semaphore(threadPoolSize);
+
+      // Start up the thread pool.
+      reset();
+
+      // Executor to send notifications back to the handler.  This also needs to be
+      // a daemon thread.
+      handlerExecutor =
+          Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setDaemon(true).build());
+    } else {
+      this.threadPoolSize = null;
+      poolEntrySemaphore = null;
+      handlerExecutor = null;
+    }
+  }
+
+  public Future<ResolvedUrl> resolveUrlAsync(final String url, final ResolvedUrlHandler handler) {
+    Preconditions.checkNotNull(
+        "Asynchronous URL resolution cannot be performed without a valid handler.", handler);
+
+    try {
+      poolEntrySemaphore.acquire();
+    } catch (InterruptedException e) {
+      LOG.log(Level.SEVERE, "Interrupted while waiting for thread to resolve URL: " + url, e);
+      return null;
+    }
+    final ListenableFutureTask<ResolvedUrl> future =
+        ListenableFutureTask.create(
+          new Callable<ResolvedUrl>() {
+            @Override public ResolvedUrl call() {
+              return resolveUrl(url);
+            }
+          });
+
+    future.addListener(new Runnable() {
+      @Override public void run() {
+        try {
+          handler.resolved(future);
+        } finally {
+          poolEntrySemaphore.release();
+        }
+      }
+    }, handlerExecutor);
+
+    executor.execute(future);
+    return future;
+  }
+
+  private void logThreadpoolInfo() {
+    LOG.info("Shutting down thread pool, available permits: "
+             + poolEntrySemaphore.availablePermits());
+    LOG.info("Queued threads? " + poolEntrySemaphore.hasQueuedThreads());
+    LOG.info("Queue length: " + poolEntrySemaphore.getQueueLength());
+  }
+
+  public void reset() {
+    Preconditions.checkState(threadPoolSize != null);
+    if (executor != null) {
+      Preconditions.checkState(executor.isShutdown(),
+          "The thread pool must be shut down before resetting.");
+      Preconditions.checkState(executor.isTerminated(), "There may still be pending async tasks.");
+    }
+
+    // Create a thread pool with daemon threads, so that they may be terminated when no
+    // application threads are running.
+    executor = Executors.newFixedThreadPool(threadPoolSize,
+        new ThreadFactoryBuilder().setDaemon(true).setNameFormat("UrlResolver[%d]").build());
+  }
+
+  /**
+   * Terminates the thread pool, waiting at most {@code waitSeconds} for active threads to complete.
+   * After this method is called, no more URLs may be submitted for resolution.
+   *
+   * @param waitSeconds The number of seconds to wait for active threads to complete.
+   */
+  public void clearAsyncTasks(int waitSeconds) {
+    Preconditions.checkState(threadPoolSize != null,
+        "finish() should not be called on a synchronous URL resolver.");
+
+    logThreadpoolInfo();
+    executor.shutdown(); // Disable new tasks from being submitted.
+    try {
+      // Wait a while for existing tasks to terminate
+      if (!executor.awaitTermination(waitSeconds, TimeUnit.SECONDS)) {
+        LOG.info("Pool did not terminate, forcing shutdown.");
+        logThreadpoolInfo();
+        List<Runnable> remaining = executor.shutdownNow();
+        LOG.info("Tasks still running: " + remaining);
+        // Wait a while for tasks to respond to being cancelled
+        if (!executor.awaitTermination(waitSeconds, TimeUnit.SECONDS)) {
+          LOG.warning("Pool did not terminate.");
+          logThreadpoolInfo();
+        }
+      }
+    } catch (InterruptedException e) {
+      LOG.log(Level.WARNING, "Interrupted while waiting for threadpool to finish.", e);
+      // (Re-)Cancel if current thread also interrupted
+      executor.shutdownNow();
+      // Preserve interrupt status
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  /**
+   * Resolves a URL synchronously.
+   *
+   * @param url The URL to resolve.
+   * @return The resolved URL.
+   */
+  public ResolvedUrl resolveUrl(String url) {
+    ResolvedUrl resolvedUrl = new ResolvedUrl();
+    resolvedUrl.setStartUrl(url);
+
+    String cached = urlCache.get(url);
+    if (cached != null) {
+      cacheHits.incrementAndGet();
+      resolvedUrl.setNextResolve(cached);
+      resolvedUrl.setEndState(EndState.CACHED);
+      return resolvedUrl;
+    }
+
+    String currentUrl = url;
+    long backoffMs = 0L;
+    String next = null;
+    for (int i = 0; i < maxRedirects; i++) {
+      try {
+        next = resolveOnce(currentUrl);
+
+        // If there was a 4xx or a 5xx, we''ll get a null back, so we pretend like we never advanced
+        // to allow for a retry within the redirect limit.
+        // TODO(John Sirois): we really need access to the return code here to do the right thing; ie:
+        // retry for internal server errors but probably not for unauthorized
+        if (next == null) {
+          if (i < maxRedirects - 1) { // don't wait if we're about to exit the loop
+            backoffMs = backoffStrategy.calculateBackoffMs(backoffMs);
+            try {
+              clock.waitFor(backoffMs);
+            } catch (InterruptedException e) {
+              Thread.currentThread().interrupt();
+              throw new RuntimeException(
+                  "Interrupted waiting to retry a failed resolution for: " + currentUrl, e);
+            }
+          }
+          continue;
+        }
+
+        backoffMs = 0L;
+        if (next.equals(currentUrl)) {
+          // We've reached the end of the redirect chain.
+          resolvedUrl.setEndState(EndState.REACHED_LANDING);
+          urlCache.put(url, currentUrl);
+          for (String intermediateUrl : resolvedUrl.getIntermediateUrls()) {
+            urlCache.put(intermediateUrl, currentUrl);
+          }
+          return resolvedUrl;
+        } else if (!url.equals(next)) {
+          resolvedUrl.setNextResolve(next);
+        }
+        currentUrl = next;
+      } catch (IOException e) {
+        LOG.log(Level.INFO, "Failed to resolve url: " + url, e);
+        resolvedUrl.setEndState(EndState.ERROR);
+        return resolvedUrl;
+      }
+    }
+
+    resolvedUrl.setEndState(next == null || url.equals(currentUrl) ? EndState.ERROR
+        : EndState.REDIRECT_LIMIT);
+    return resolvedUrl;
+  }
+
+  /**
+   * Resolves a url, following at most one redirect.  Thread-safe.
+   *
+   * @param url The URL to resolve.
+   * @return The result of following the URL through at most one redirect or null if the url could
+   *     not be followed
+   * @throws IOException If an error occurs while resolving the URL.
+   */
+  private String resolveOnce(String url) throws IOException {
+    requestCount.incrementAndGet();
+
+    String resolvedUrl = urlCache.get(url);
+    if (resolvedUrl != null) {
+      cacheHits.incrementAndGet();
+      return resolvedUrl;
+    }
+
+    try {
+      long startTimeMs = System.currentTimeMillis();
+      resolvedUrl = resolver.apply(url);
+      if (resolvedUrl == null) {
+        return null;
+      }
+
+      urlCache.put(url, resolvedUrl);
+
+      synchronized (urlResolutionTimesMs) {
+        urlResolutionTimesMs.addValue(System.currentTimeMillis() - startTimeMs);
+      }
+      return resolvedUrl;
+    } catch (IOException e) {
+      failureCount.incrementAndGet();
+      throw e;
+    }
+  }
+
+  @Override
+  public String toString() {
+    return String.format("Cache: %s\nFailed requests: %d,\nResolution Times: %s",
+        urlCache, failureCount.get(),
+        urlResolutionTimesMs.toString());
+  }
+
+  /**
+   * Class to wrap the result of a URL resolution.
+   */
+  public static class ResolvedUrl {
+    public enum EndState {
+      REACHED_LANDING,
+      ERROR,
+      CACHED,
+      REDIRECT_LIMIT
+    }
+
+    private String startUrl;
+    private final List<String> resolveChain;
+    private EndState endState;
+
+    public ResolvedUrl() {
+      resolveChain = Lists.newArrayList();
+    }
+
+    @VisibleForTesting
+    public ResolvedUrl(EndState endState, String startUrl, String... resolveChain) {
+      this.endState = endState;
+      this.startUrl = startUrl;
+      this.resolveChain = Lists.newArrayList(resolveChain);
+    }
+
+    public String getStartUrl() {
+      return startUrl;
+    }
+
+    void setStartUrl(String startUrl) {
+      this.startUrl = startUrl;
+    }
+
+    /**
+     * Returns the last URL resolved following a redirect chain, or null if the startUrl is a
+     * landing URL.
+     */
+    public String getEndUrl() {
+      return resolveChain.isEmpty() ? null : Iterables.getLast(resolveChain);
+    }
+
+    void setNextResolve(String endUrl) {
+      this.resolveChain.add(endUrl);
+    }
+
+    /**
+     * Returns any immediate URLs encountered on the resolution chain.  If the startUrl redirects
+     * directly to the endUrl or they are the same the imtermediate URLs will be empty.
+     */
+    public Iterable<String> getIntermediateUrls() {
+      return resolveChain.size() <= 1 ? ImmutableList.<String>of()
+          : resolveChain.subList(0, resolveChain.size() - 1);
+    }
+
+    public EndState getEndState() {
+      return endState;
+    }
+
+    void setEndState(EndState endState) {
+      this.endState = endState;
+    }
+
+    public String toString() {
+      return String.format("%s -> %s [%s, %d redirects]",
+          startUrl, Joiner.on(" -> ").join(resolveChain), endState, resolveChain.size());
+    }
+  }
+
+  /**
+   * Interface to use for notifying the caller of resolved URLs.
+   */
+  public interface ResolvedUrlHandler {
+    /**
+     * Signals that a URL has been resolved to its target.  The implementation of this method must
+     * be thread safe.
+     *
+     * @param future The future that has finished resolving a URL.
+     */
+    public void resolved(Future<ResolvedUrl> future);
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/06ddaadb/commons/src/main/java/org/apache/aurora/common/net/UrlResolverUtil.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/aurora/common/net/UrlResolverUtil.java b/commons/src/main/java/org/apache/aurora/common/net/UrlResolverUtil.java
new file mode 100644
index 0000000..4b95bb7
--- /dev/null
+++ b/commons/src/main/java/org/apache/aurora/common/net/UrlResolverUtil.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.common.net;
+
+import com.google.common.base.Function;
+import com.google.common.base.Functions;
+import com.google.common.base.Preconditions;
+import org.apache.aurora.common.base.MorePreconditions;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.Proxy;
+import java.net.Proxy.Type;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.Map;
+import java.util.logging.Logger;
+import javax.annotation.Nullable;
+
+/**
+ * A utility that can resolve HTTP urls.
+ *
+ * @author John Sirois
+ */
+class UrlResolverUtil {
+
+  private static final Logger LOG = Logger.getLogger(UrlResolverUtil.class.getName());
+
+  // Default user-agent string to user for HTTP requests.
+  private static final String DEFAULT_USER_AGENT = "Lynxy/6.6.6dev.8 libwww-FM/3.14159FM";
+
+  private static Map<String, String> checkNotBlank(Map<String, String> hostToUserAgent) {
+    Preconditions.checkNotNull(hostToUserAgent);
+    MorePreconditions.checkNotBlank(hostToUserAgent.entrySet());
+    return hostToUserAgent;
+  }
+
+  private final Function<? super URL, String> urlToUserAgent;
+
+  UrlResolverUtil(Map<String, String> hostToUserAgent) {
+    this(Functions.compose(Functions.forMap(checkNotBlank(hostToUserAgent), DEFAULT_USER_AGENT),
+        new Function<URL, String>() {
+          @Override public String apply(URL url) {
+            return url.getHost();
+          }
+        }));
+  }
+
+  UrlResolverUtil(Function<? super URL, String> urlToUserAgent) {
+    this.urlToUserAgent = Preconditions.checkNotNull(urlToUserAgent);
+  }
+
+  /**
+   * Returns the URL that {@code url} lands on, which will be the result of a 3xx redirect,
+   * or {@code url} if the url does not redirect using an HTTP 3xx response code.  If there is a
+   * non-2xx or 3xx HTTP response code null is returned.
+   *
+   * @param url The URL to follow.
+   * @return The redirected URL, or {@code url} if {@code url} returns a 2XX response, otherwise
+   *         null
+   * @throws java.io.IOException If an error occurs while trying to follow the url.
+   */
+  String getEffectiveUrl(String url, @Nullable ProxyConfig proxyConfig) throws IOException {
+    Preconditions.checkNotNull(url);
+    // Don't follow https.
+    if (url.startsWith("https://")) {
+      url = url.replace("https://", "http://");
+    } else if (!url.startsWith("http://")) {
+      url = "http://" + url;
+    }
+
+    URL urlObj = new URL(url);
+
+    HttpURLConnection con;
+    if (proxyConfig != null) {
+      Proxy proxy = new Proxy(Type.HTTP, proxyConfig.getProxyAddress());
+      con = (HttpURLConnection) urlObj.openConnection(proxy);
+      ProxyAuthorizer.adapt(proxyConfig).authorize(con);
+    } else {
+      con = (HttpURLConnection) urlObj.openConnection();
+    }
+    try {
+
+      // TODO(John Sirois): several commonly tweeted hosts 406 or 400 on HEADs and only work with GETs
+      // fix the call chain to be able to specify retry-with-GET
+      con.setRequestMethod("HEAD");
+
+      con.setUseCaches(true);
+      con.setConnectTimeout(5000);
+      con.setReadTimeout(5000);
+      con.setInstanceFollowRedirects(false);
+
+      // I hate to have to do this, but some URL shorteners don't respond otherwise.
+      con.setRequestProperty("User-Agent", urlToUserAgent.apply(urlObj));
+      try {
+        con.connect();
+      } catch (StringIndexOutOfBoundsException e) {
+        LOG.info("Got StringIndexOutOfBoundsException when fetching headers for " + url);
+        return null;
+      }
+
+      int responseCode = con.getResponseCode();
+      switch (responseCode / 100) {
+        case 2:
+          return url;
+        case 3:
+          String location = con.getHeaderField("Location");
+          if (location == null) {
+            if (responseCode != 304 /* not modified */) {
+              LOG.info(
+                  String.format("[%d] Location header was null for URL: %s", responseCode, url));
+            }
+            return url;
+          }
+
+          // HTTP 1.1 spec says this should be an absolute URI, but i see lots of instances where it
+          // is relative, so we need to check.
+          try {
+            String domain = UrlHelper.getDomainChecked(location);
+            if (domain == null || domain.isEmpty()) {
+              // This is a relative URI.
+              location = "http://" + UrlHelper.getDomain(url) + location;
+            }
+          } catch (URISyntaxException e) {
+            LOG.info("location contained an invalid URI: " + location);
+          }
+
+          return location;
+        default:
+          LOG.info("Failed to resolve url: " + url + " with: "
+                   + responseCode + " -> " + con.getResponseMessage());
+          return null;
+      }
+    } finally {
+      con.disconnect();
+    }
+  }
+}


Mime
View raw message