incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1365722 [7/11] - in /incubator/hcatalog/trunk: ./ ant/ conf/ hcatalog-pig-adapter/ ivy/ src/docs/src/documentation/content/xdocs/ src/docs/src/documentation/content/xdocs/images/ src/java/org/apache/hcatalog/mapreduce/ src/test/e2e/templet...
Date Wed, 25 Jul 2012 20:29:49 GMT
Added: incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh Wed Jul 25 20:29:44 2012
@@ -0,0 +1,237 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Support functions
+#
+
+# Follow symlinks on Linux and Darwin
+function real_script_name() {
+        local base=$1
+        local real
+        if readlink -f $base >/dev/null 2>&1; then
+                # Darwin/Mac OS X
+                real=`readlink -f $base`
+        fi
+        if [[ "$?" != "0" || -z "$real" ]]; then
+                # Linux
+                local bin=$(cd -P -- "$(dirname -- "$base")">/dev/null && pwd -P)
+                local script="$(basename -- "$base")"
+                real="$bin/$script"
+        fi
+        echo "$real"
+}
+
+function usage() {
+        echo "usage: $0 [start|stop|foreground]"
+        echo "  start           Start the Templeton Server"
+        echo "  stop            Stop the Templeton Server"
+        echo "  foreground      Run the Templeton Server in the foreground"
+        exit 1
+}
+
+# Print an error message and exit
+function die() {
+        echo "templeton: $@" 1>&2
+        exit 1
+}
+
+# Print an message
+function log() {
+        echo "templeton: $@"
+}
+
+# Find the templeton jar
+function find_jar_path() {
+        for dir in "." "build/templeton" "share/templeton/"; do
+                local jar="$base_dir/$dir/$TEMPLETON_JAR"
+                if [[ -f $jar ]]; then
+                        echo $jar
+                        break
+                fi
+        done
+}
+
+# Find the templeton classpath
+function find_classpath() {
+        local classpath=""
+        for dir in  "share/templeton/lib" "build/ivy/lib/templeton" "conf" ; do
+                local path="$base_dir/$dir"
+
+                if [[ -d $path ]]; then
+                        for jar_or_conf in $path/*; do
+                                if [[ -z "$classpath" ]]; then
+                                        classpath="$jar_or_conf"
+                                else
+                                        classpath="$classpath:$jar_or_conf"
+                                fi
+                        done
+                fi
+        done
+
+        if [[ -n "$TEMPLETON_CONF_DIR" ]]; then
+                if [[ -z "$classpath" ]]; then
+                        classpath="$TEMPLETON_CONF_DIR"
+                else
+                        classpath="$classpath:$TEMPLETON_CONF_DIR"
+                fi
+        fi
+
+        # Append hcat classpath
+        local hcat_classpath
+        hcat_classpath=`hcat -classpath`
+        if [[ "$?" != "0" ]]; then
+                die "Unable to get the hcatalog classpath"
+        fi
+        echo "$classpath:$hcat_classpath"
+}
+
+# Check if the pid is running
+function check_pid() {
+        local pid=$1
+        if ps -p $pid > /dev/null; then
+                return 0
+        else
+                return 1
+        fi
+}
+
+# Start the templeton server in the foreground
+function foreground_templeton() {
+        $start_cmd
+}
+
+# Start the templeton server in the background.  Record the PID for
+# later use.
+function start_templeton() {
+        if [[ -f $PID_FILE ]]; then
+                # Check if there is a server running
+                local pid=`cat $PID_FILE`
+                if check_pid $pid; then
+                        die "already running on process $pid"
+                fi
+        fi
+
+        log "starting ..."
+        log "$start_cmd"
+        nohup $start_cmd >>$CONSOLE_LOG 2>>$ERROR_LOG &
+        local pid=$!
+
+        if [[ -z "${pid}" ]] ; then # we failed right off
+                die "failed to start. Check logs in " `dirname $ERROR_LOG`
+        fi
+
+        sleep $SLEEP_TIME_AFTER_START
+
+        if check_pid $pid; then
+                echo $pid > $PID_FILE
+                log "starting ... started."
+        else
+                die "failed to start. Check logs in " `dirname $ERROR_LOG`
+        fi
+}
+
+# Stop a running server
+function stop_templeton() {
+        local pid
+        if [[ -f $PID_FILE ]]; then
+                # Check if there is a server running
+                local check=`cat $PID_FILE`
+                if check_pid $check; then
+                        pid=$check
+                fi
+        fi
+
+        if [[ -z "$pid" ]]; then
+                log "no running server found"
+        else
+                log "stopping ..."
+                kill $pid
+                sleep $SLEEP_TIME_AFTER_START
+                if check_pid $pid; then
+                        die "failed to stop"
+                else
+                        log "stopping ... stopped"
+                fi
+        fi
+}
+
+#
+# Build command line and run
+#
+
+this=`real_script_name "${BASH_SOURCE-$0}"`
+this_bin=`dirname $this`
+base_dir="$this_bin/.."
+
+if [[ -f "$base_dir/libexec/templeton_config.sh" ]]; then
+        . "$base_dir/libexec/templeton_config.sh"
+else
+        . "$this_bin/templeton_config.sh"
+fi
+
+JAR=`find_jar_path`
+if [[ -z "$JAR" ]]; then
+        die "No templeton jar found"
+fi
+
+CLASSPATH=`find_classpath`
+if [[ -z "$CLASSPATH" ]]; then
+        die "No classpath or jars found"
+fi
+CLASSPATH="$JAR:$CLASSPATH"
+
+if [[ -z "$HADOOP_CLASSPATH" ]]; then
+        export HADOOP_CLASSPATH="$CLASSPATH"
+else
+        export HADOOP_CLASSPATH="$CLASSPATH:$HADOOP_CLASSPATH"
+fi
+
+if [[ -z "$TEMPLETON_LOG4J" ]]; then
+        if [[ -f "$base_dir/conf/templeton-log4j.properties" ]]; then
+                TEMPLETON_LOG4J="$base_dir/conf/templeton-log4j.properties";
+        elif [[ -f "$base_dir/conf/templeton-log4j.properties" ]]; then
+                TEMPLETON_LOG4J="$base_dir/conf/templeton-log4j.properties";
+        else
+                TEMPLETON_LOG4J="templeton-log4j.properties";
+        fi
+fi
+
+export HADOOP_USER_CLASSPATH_FIRST=true
+export HADOOP_OPTS="-Dtempleton.log.dir=$TEMPLETON_LOG_DIR -Dlog4j.configuration=$TEMPLETON_LOG4J"
+
+start_cmd="$HADOOP_PREFIX/bin/hadoop jar $JAR org.apache.hcatalog.templeton.Main  "
+
+
+cmd=$1
+case $cmd in
+        start)
+                start_templeton
+                ;;
+        stop)
+                stop_templeton
+                ;;
+        foreground)
+                foreground_templeton
+                ;;
+        *)
+                usage
+                ;;
+esac
+
+log "done"

Propchange: incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/*
+ * Communicate with the JobTracker as a specific user.
+ */
+public class TempletonJobTracker {
+    private JobSubmissionProtocol cnx;
+
+    /**
+     * Create a connection to the Job Tracker.
+     */
+    public TempletonJobTracker(UserGroupInformation ugi,
+                               InetSocketAddress addr,
+                               Configuration conf)
+        throws IOException
+    {
+        cnx = (JobSubmissionProtocol)
+            RPC.getProxy(JobSubmissionProtocol.class,
+                         JobSubmissionProtocol.versionID,
+                         addr,
+                         ugi,
+                         conf,
+                         NetUtils.getSocketFactory(conf,
+                                                   JobSubmissionProtocol.class));
+    }
+
+    /**
+     * Grab a handle to a job that is already known to the JobTracker.
+     *
+     * @return Profile of the job, or null if not found.
+     */
+    public JobProfile getJobProfile(JobID jobid)
+        throws IOException
+    {
+        return cnx.getJobProfile(jobid);
+    }
+
+    /**
+     * Grab a handle to a job that is already known to the JobTracker.
+     *
+     * @return Status of the job, or null if not found.
+     */
+    public JobStatus getJobStatus(JobID jobid)
+        throws IOException
+    {
+        return cnx.getJobStatus(jobid);
+    }
+
+
+    /**
+     * Kill a job.
+     */
+    public void killJob(JobID jobid)
+        throws IOException
+    {
+        cnx.killJob(jobid);
+    }
+
+    /**
+     * Get all the jobs submitted.
+     */
+    public JobStatus[] getAllJobs()
+        throws IOException
+    {
+        return cnx.getAllJobs();
+    }
+
+    /**
+     * Close the connection to the Job Tracker.
+     */
+    public void close() {
+        RPC.stopProxy(cnx);
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hcatalog.templeton.tool.JobState;
+import org.apache.hcatalog.templeton.tool.ZooKeeperCleanup;
+import org.apache.hcatalog.templeton.tool.ZooKeeperStorage;
+
+/**
+ * The configuration for Templeton.  This merges the normal Hadoop
+ * configuration with the Templeton specific variables.
+ *
+ * The Templeton configuration variables are described in
+ * templeton-default.xml
+ *
+ * The Templeton specific configuration is split into two layers
+ *
+ * 1. templeton-default.xml - All the configuration variables that
+ *    Templeton needs.  These are the defaults that ship with the app
+ *    and should only be changed be the app developers.
+ *
+ * 2. templeton-site.xml - The (possibly empty) configuration that the
+ *    system administrator can set variables for their Hadoop cluster.
+ *
+ * The configuration files are loaded in this order with later files
+ * overriding earlier ones.
+ *
+ * To find the configuration files, we first attempt to load a file
+ * from the CLASSPATH and then look in the directory specified in the
+ * TEMPLETON_HOME environment variable.
+ *
+ * In addition the configuration files may access the special env
+ * variable env for all environment variables.  For example, the
+ * hadoop executable could be specified using:
+ *<pre>
+ *      ${env.HADOOP_PREFIX}/bin/hadoop
+ *</pre>
+ */
+public class AppConfig extends Configuration {
+    public static final String[] HADOOP_CONF_FILENAMES = {
+        "core-default.xml", "core-site.xml", "mapred-default.xml", "mapred-site.xml"
+    };
+
+    public static final String[] HADOOP_PREFIX_VARS = {
+        "HADOOP_PREFIX", "HADOOP_HOME"
+    };
+
+    public static final String TEMPLETON_HOME_VAR = "TEMPLETON_HOME";
+
+    public static final String[] TEMPLETON_CONF_FILENAMES = {
+        "templeton-default.xml",
+        "templeton-site.xml"
+    };
+
+    public static final String PORT                = "templeton.port";
+    public static final String EXEC_ENCODING_NAME  = "templeton.exec.encoding";
+    public static final String EXEC_ENVS_NAME      = "templeton.exec.envs";
+    public static final String EXEC_MAX_BYTES_NAME = "templeton.exec.max-output-bytes";
+    public static final String EXEC_MAX_PROCS_NAME = "templeton.exec.max-procs";
+    public static final String EXEC_TIMEOUT_NAME   = "templeton.exec.timeout";
+    public static final String HADOOP_NAME         = "templeton.hadoop";
+    public static final String HADOOP_CONF_DIR     = "templeton.hadoop.conf.dir";
+    public static final String HCAT_NAME           = "templeton.hcat";
+    public static final String HIVE_ARCHIVE_NAME   = "templeton.hive.archive";
+    public static final String HIVE_PATH_NAME      = "templeton.hive.path";
+    public static final String HIVE_PROPS_NAME     = "templeton.hive.properties";
+    public static final String LIB_JARS_NAME       = "templeton.libjars";
+    public static final String PIG_ARCHIVE_NAME    = "templeton.pig.archive";
+    public static final String PIG_PATH_NAME       = "templeton.pig.path";
+    public static final String STREAMING_JAR_NAME  = "templeton.streaming.jar";
+    public static final String TEMPLETON_JAR_NAME  = "templeton.jar";
+    public static final String OVERRIDE_JARS_NAME  = "templeton.override.jars";
+    public static final String OVERRIDE_JARS_ENABLED = "templeton.override.enabled";
+    public static final String KERBEROS_SECRET     = "templeton.kerberos.secret";
+    public static final String KERBEROS_PRINCIPAL  = "templeton.kerberos.principal";
+    public static final String KERBEROS_KEYTAB     = "templeton.kerberos.keytab";
+
+    public static final String CALLBACK_INTERVAL_NAME
+        = "templeton.callback.retry.interval";
+    public static final String CALLBACK_RETRY_NAME
+        = "templeton.callback.retry.attempts";
+    public static final String HADOOP_END_INTERVAL_NAME = "job.end.retry.interval";
+    public static final String HADOOP_END_RETRY_NAME    = "job.end.retry.attempts";
+    public static final String HADOOP_END_URL_NAME      = "job.end.notification.url";
+    public static final String HADOOP_SPECULATIVE_NAME
+        = "mapred.map.tasks.speculative.execution";
+
+    private static final Log LOG = LogFactory.getLog(AppConfig.class);
+
+    public AppConfig() {
+        init();
+        LOG.info("Using Hadoop version " + VersionInfo.getVersion());
+    }
+
+    private void init() {
+        for (Map.Entry<String, String> e : System.getenv().entrySet())
+            set("env." + e.getKey(), e.getValue());
+
+        String templetonDir = getTempletonDir();
+        for (String fname : TEMPLETON_CONF_FILENAMES)
+            if (! loadOneClasspathConfig(fname))
+                loadOneFileConfig(templetonDir, fname);
+
+        String hadoopConfDir = getHadoopConfDir();
+        for (String fname : HADOOP_CONF_FILENAMES)
+            loadOneFileConfig(hadoopConfDir, fname);
+    }
+
+    public void startCleanup() {
+        JobState.getStorageInstance(this).startCleanup(this);
+    }
+
+    public String getHadoopConfDir() {
+        return get(HADOOP_CONF_DIR);
+    }
+
+    public static String getTempletonDir() {
+        return System.getenv(TEMPLETON_HOME_VAR);
+    }
+
+    private boolean loadOneFileConfig(String dir, String fname) {
+        if (dir != null) {
+            File f = new File(dir, fname);
+            if (f.exists()) {
+                addResource(new Path(f.getAbsolutePath()));
+                LOG.debug("loaded config file " + f.getAbsolutePath());
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean loadOneClasspathConfig(String fname) {
+        URL x = getResource(fname);
+        if (x != null) {
+            addResource(x);
+            LOG.debug("loaded config from classpath " + x);
+            return true;
+        }
+
+        return false;
+    }
+
+    public String templetonJar()     { return get(TEMPLETON_JAR_NAME); }
+    public String libJars()          { return get(LIB_JARS_NAME); }
+    public String clusterHadoop()    { return get(HADOOP_NAME); }
+    public String clusterHcat()      { return get(HCAT_NAME); }
+    public String pigPath()          { return get(PIG_PATH_NAME); }
+    public String pigArchive()       { return get(PIG_ARCHIVE_NAME); }
+    public String hivePath()         { return get(HIVE_PATH_NAME); }
+    public String hiveArchive()      { return get(HIVE_ARCHIVE_NAME); }
+    public String streamingJar()     { return get(STREAMING_JAR_NAME); }
+    public String kerberosSecret()   { return get(KERBEROS_SECRET); }
+    public String kerberosPrincipal(){ return get(KERBEROS_PRINCIPAL); }
+    public String kerberosKeytab()   { return get(KERBEROS_KEYTAB); }
+
+    public String[] overrideJars() {
+        if (getBoolean(OVERRIDE_JARS_ENABLED, true))
+            return getStrings(OVERRIDE_JARS_NAME);
+        else
+            return null;
+    }
+    public String overrideJarsString() {
+        if (getBoolean(OVERRIDE_JARS_ENABLED, true))
+            return get(OVERRIDE_JARS_NAME);
+        else
+            return null;
+    }
+
+    public long zkCleanupInterval()  {
+        return getLong(ZooKeeperCleanup.ZK_CLEANUP_INTERVAL,
+                       (1000L * 60L * 60L * 12L)); }
+    public long zkMaxAge()           {
+        return getLong(ZooKeeperCleanup.ZK_CLEANUP_MAX_AGE,
+                       (1000L * 60L * 60L * 24L * 7L)); }
+    public String zkHosts()          { return get(ZooKeeperStorage.ZK_HOSTS); }
+    public int zkSessionTimeout()    { return getInt(ZooKeeperStorage.ZK_SESSION_TIMEOUT,
+                                                     30000); }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * Missing required or badly configured paramater.
+ */
+public class BadParam extends SimpleWebException {
+    public BadParam(String msg) {
+        super(400, msg);
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * Simple "we are busy, try again" exception.
+ */
+public class BusyException extends SimpleWebException {
+    public BusyException() {
+        super(503, "Busy, please retry");
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * The callback failed when it tried to reach the callback URL.
+ */
+public class CallbackFailedException extends SimpleWebException {
+    public CallbackFailedException(String msg) {
+        super(400, msg);
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Map all exceptions to the Jersey response.  This lets us have nice
+ * results in the error body.
+ */
+@Provider
+public class CatchallExceptionMapper
+    implements ExceptionMapper<Exception>
+{
+    private static final Log LOG = LogFactory.getLog(CatchallExceptionMapper.class);
+
+    public Response toResponse(Exception e) {
+        LOG.error(e.getMessage(), e);
+        return SimpleWebException.buildMessage(500, null, e.getMessage());
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * A description of the column to create.
+ */
+@XmlRootElement
+public class ColumnDesc extends GroupPermissionsDesc {
+    public String name;
+    public String type;
+    public String comment;
+
+    public ColumnDesc() {}
+
+    /**
+     * Create a new ColumnDesc
+     */
+    public ColumnDesc(String name, String type, String comment) {
+        this.name = name;
+        this.type = type;
+        this.comment = comment;
+    }
+
+    public String toString() {
+        return String.format("ColumnDesc(name=%s, type=%s, comment=%s)",
+                             name, type, comment);
+    }
+
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (! (o instanceof ColumnDesc))
+            return false;
+        ColumnDesc that = (ColumnDesc) o;
+        return xequals(this.name,       that.name)
+            && xequals(this.type,       that.type)
+            && xequals(this.comment,    that.comment)
+            && super.equals(that)
+            ;
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * CompleteBean - The results of an CompleteDelegator run.
+ */
+public class CompleteBean {
+    public String status;
+
+    public CompleteBean() {}
+
+    /**
+     * Create a new CompleteBean
+     *
+     * @param status     run status
+     */
+    public CompleteBean(String status) {
+        this.status = status;
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.MalformedURLException;
+import java.util.Date;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapred.JobProfile;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.JobTracker;
+import org.apache.hadoop.mapred.TempletonJobTracker;
+import org.apache.hcatalog.templeton.tool.JobState;
+import org.apache.hcatalog.templeton.tool.TempletonUtils;
+
+/**
+ * Complete a job.  This will run the callback if
+ *
+ * - the job is done
+ * - there is a callback
+ * - the callback has not yet been called
+ *
+ * There is a small chance for a race condition if two callers run
+ * this at the same time.  That should never happen.
+ *
+ * We use a Hadoop config var to notify this class on the completion
+ * of a job.  Hadoop will call use multiple times in the event of
+ * failure.  Even if the failure is that the client callback failed.
+ *
+ * See LauncherDelegator for the HADOOP_END_RETRY* vars that are set.
+ */
+public class CompleteDelegator extends TempletonDelegator {
+    private static final Log LOG = LogFactory.getLog(CompleteDelegator.class);
+
+    public CompleteDelegator(AppConfig appConf) {
+        super(appConf);
+    }
+
+    public CompleteBean run(String id)
+        throws CallbackFailedException, IOException
+    {
+        if (id == null)
+            acceptWithError("No jobid given");
+
+        JobState state = null;
+        try {
+            state = new JobState(id, Main.getAppConfigInstance());
+            if (state.getCompleteStatus() == null)
+                failed("Job not yet complete", null);
+
+            Long notified = state.getNotifiedTime();
+            if (notified != null)
+                return acceptWithError("Callback already run on "
+                                       + new Date(notified.longValue()));
+
+            String callback = state.getCallback();
+            if (callback == null)
+                return new CompleteBean("No callback registered");
+
+            try {
+                doCallback(state.getId(), callback);
+            } catch (Exception e) {
+                failed("Callback failed " + callback + " for " + id, e);
+            }
+
+            state.setNotifiedTime(System.currentTimeMillis());
+            return new CompleteBean("Callback sent");
+        } finally {
+            if (state != null)
+                state.close();
+        }
+    }
+
+    /**
+     * Call the callback url with the jobid to let them know it's
+     * finished.  If the url has the string $jobId in it, it will be
+     * replaced with the completed jobid.
+     */
+    public static void doCallback(String jobid, String url)
+        throws MalformedURLException, IOException
+    {
+        if (url.contains("$jobId"))
+            url = url.replace("$jobId", jobid);
+        TempletonUtils.fetchUrl(new URL(url));
+    }
+
+    private void failed(String msg, Exception e)
+        throws CallbackFailedException
+    {
+        if (e != null)
+            LOG.error(msg, e);
+        else
+            LOG.error(msg);
+        throw new CallbackFailedException(msg);
+    }
+
+    private CompleteBean acceptWithError(String msg) {
+        LOG.error(msg);
+        return new CompleteBean(msg);
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.util.Map;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * A description of the database to create.
+ */
+@XmlRootElement
+public class DatabaseDesc extends GroupPermissionsDesc {
+    public boolean ifNotExists;
+    public String database;
+    public String comment;
+    public String location;
+    public Map<String, String> properties;
+
+    public DatabaseDesc() {}
+
+    public String toString() {
+        return String.format("DatabaseDesc(database=%s, comment=%s, location=%s, " +
+                             "properties=%s)", database, comment, location, properties);
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.IOException;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapred.JobProfile;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.JobTracker;
+import org.apache.hadoop.mapred.TempletonJobTracker;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hcatalog.templeton.tool.JobState;
+
+/**
+ * Delete a job
+ */
+public class DeleteDelegator extends TempletonDelegator {
+    public DeleteDelegator(AppConfig appConf) {
+        super(appConf);
+    }
+
+    public QueueStatusBean run(String user, String id)
+        throws NotAuthorizedException, BadParam, IOException
+    {
+        UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+        TempletonJobTracker tracker = null;
+        JobState state = null;
+        try {
+            tracker = new TempletonJobTracker(ugi,
+                                              JobTracker.getAddress(appConf),
+                                              appConf);
+            JobID jobid = StatusDelegator.StringToJobID(id);
+            if (jobid == null)
+                throw new BadParam("Invalid jobid: " + id);
+            tracker.killJob(jobid);
+            state = new JobState(id, Main.getAppConfigInstance());
+            String childid = state.getChildId();
+            if (childid != null)
+                tracker.killJob(StatusDelegator.StringToJobID(childid));
+            return StatusDelegator.makeStatus(tracker, jobid, state);
+        } catch (IllegalStateException e) {
+            throw new BadParam(e.getMessage());
+        } finally {
+            if (tracker != null)
+                tracker.close();
+            if (state != null)
+                state.close();
+        }
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * EnqueueBean - The results of a call that enqueues a Hadoop job.
+ */
+public class EnqueueBean {
+    public String id;
+
+    public EnqueueBean() {}
+
+    /**
+     * Create a new EnqueueBean.
+     *
+     * @param id     job id
+     */
+    public EnqueueBean(String id) {
+        this.id = id;
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * ExecBean - The results of an exec call.
+ */
+public class ExecBean {
+    public String stdout;
+    public String stderr;
+    public int exitcode;
+
+    public ExecBean() {}
+
+    /**
+     * Create a new ExecBean.
+     *
+     * @param stdout     standard output of the the program.
+     * @param stderr     error output of the the program.
+     * @param exitcode   exit code of the program.
+     */
+    public ExecBean(String stdout, String stderr, int exitcode) {
+        this.stdout = stdout;
+        this.stderr = stderr;
+        this.exitcode = exitcode;
+    }
+
+    public String toString() {
+        return String.format("ExecBean(stdout=%s, stderr=%s, exitcode=%s)",
+                             stdout, stderr, exitcode);
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.exec.ExecuteException;
+
+public interface ExecService {
+    public ExecBean run(String program, List<String> args,
+                        Map<String, String> env)
+        throws NotAuthorizedException, BusyException, ExecuteException, IOException;
+
+    public ExecBean runUnlimited(String program, List<String> args,
+                                 Map<String, String> env)
+        throws NotAuthorizedException, ExecuteException, IOException;
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Semaphore;
+import org.apache.commons.exec.CommandLine;
+import org.apache.commons.exec.DefaultExecutor;
+import org.apache.commons.exec.ExecuteException;
+import org.apache.commons.exec.ExecuteWatchdog;
+import org.apache.commons.exec.PumpStreamHandler;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Execute a local program.  This is a singleton service that will
+ * execute programs as non-privileged users on the local box.  See
+ * ExecService.run and ExecService.runUnlimited for details.
+ */
+public class ExecServiceImpl implements ExecService {
+    private static final Log LOG = LogFactory.getLog(ExecServiceImpl.class);
+    private static AppConfig appConf = Main.getAppConfigInstance();
+
+    private static volatile ExecServiceImpl theSingleton;
+
+    /**
+     * Retrieve the singleton.
+     */
+    public static synchronized ExecServiceImpl getInstance() {
+        if (theSingleton == null) {
+            theSingleton = new ExecServiceImpl();
+        }
+        return theSingleton;
+    }
+
+    private Semaphore avail;
+
+    private ExecServiceImpl() {
+        avail = new Semaphore(appConf.getInt(AppConfig.EXEC_MAX_PROCS_NAME, 16));
+    }
+
+    /**
+     * Run the program synchronously as the given user. We rate limit
+     * the number of processes that can simultaneously created for
+     * this instance.
+     *
+     * @param program   The program to run
+     * @param args      Arguments to pass to the program
+     * @param env       Any extra environment variables to set
+     * @return          The result of the run.
+     */
+    public ExecBean run(String program, List<String> args,
+                        Map<String, String> env)
+        throws NotAuthorizedException, BusyException, ExecuteException, IOException
+    {
+        boolean aquired = false;
+        try {
+            aquired = avail.tryAcquire();
+            if (aquired) {
+                return runUnlimited(program, args, env);
+            } else {
+                throw new BusyException();
+            }
+        } finally {
+            if (aquired) {
+                avail.release();
+            }
+        }
+    }
+
+    /**
+     * Run the program synchronously as the given user.  Warning:
+     * CommandLine will trim the argument strings.
+     *
+     * @param program   The program to run.
+     * @param args      Arguments to pass to the program
+     * @param env       Any extra environment variables to set
+     * @return          The result of the run.
+     */
+    public ExecBean runUnlimited(String program, List<String> args,
+                                 Map<String, String> env)
+        throws NotAuthorizedException, ExecuteException, IOException
+    {
+        try {
+            return auxRun(program, args, env);
+        } catch (IOException e) {
+            File cwd = new java.io.File(".");
+            if (cwd.canRead() && cwd.canWrite())
+                throw e;
+            else
+                throw new IOException("Invalid permissions on Templeton directory: "
+                                      + cwd.getCanonicalPath());
+        }
+    }
+
+    private ExecBean auxRun(String program, List<String> args, Map<String, String> env)
+        throws NotAuthorizedException, ExecuteException, IOException
+    {
+        DefaultExecutor executor = new DefaultExecutor();
+        executor.setExitValues(null);
+
+        // Setup stdout and stderr
+        int nbytes = appConf.getInt(AppConfig.EXEC_MAX_BYTES_NAME, -1);
+        ByteArrayOutputStream outStream = new MaxByteArrayOutputStream(nbytes);
+        ByteArrayOutputStream errStream = new MaxByteArrayOutputStream(nbytes);
+        executor.setStreamHandler(new PumpStreamHandler(outStream, errStream));
+
+        // Only run for N milliseconds
+        int timeout = appConf.getInt(AppConfig.EXEC_TIMEOUT_NAME, 0);
+        ExecuteWatchdog watchdog = new ExecuteWatchdog(timeout);
+        executor.setWatchdog(watchdog);
+
+        CommandLine cmd = makeCommandLine(program, args);
+
+        LOG.info("Running: " + cmd);
+        ExecBean res = new ExecBean();
+        res.exitcode = executor.execute(cmd, execEnv(env));
+        String enc = appConf.get(AppConfig.EXEC_ENCODING_NAME);
+        res.stdout = outStream.toString(enc);
+        res.stderr = errStream.toString(enc);
+
+        return res;
+    }
+
+    private CommandLine makeCommandLine(String program,
+                                        List<String> args)
+        throws NotAuthorizedException, IOException
+    {
+        String path = validateProgram(program);
+        CommandLine cmd = new CommandLine(path);
+        if (args != null)
+            for (String arg : args)
+                cmd.addArgument(arg, false);
+
+        return cmd;
+    }
+
+    /**
+     * Build the environment used for all exec calls.
+     *
+     * @return The environment variables.
+     */
+    public Map<String, String> execEnv(Map<String, String> env) {
+        HashMap<String, String> res = new HashMap<String, String>();
+
+        for (String key : appConf.getStrings(AppConfig.EXEC_ENVS_NAME)) {
+            String val = System.getenv(key);
+            if (val != null) {
+                res.put(key, val);
+            }
+        }
+        if (env != null)
+            res.putAll(env);
+        for(Map.Entry<String, String> envs : res.entrySet()){
+	    LOG.info("Env " + envs.getKey() + "=" + envs.getValue());
+        }
+        return res;
+    }
+
+    /**
+     * Given a program name, lookup the fully qualified path.  Throws
+     * an exception if the program is missing or not authorized.
+     *
+     * @param path      The path of the program.
+     * @return          The path of the validated program.
+     */
+    public String validateProgram(String path)
+        throws NotAuthorizedException, IOException
+    {
+        File f = new File(path);
+        if (f.canExecute()) {
+            return f.getCanonicalPath();
+        } else {
+            throw new NotAuthorizedException("Unable to access program: " + path);
+        }
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * The base create permissions for ddl objects.
+ */
+public abstract class GroupPermissionsDesc {
+    public String group;
+    public String permissions;
+
+    public GroupPermissionsDesc() {}
+
+    protected static boolean xequals(Object a, Object b) {
+        if (a == null) {
+            if (b == null)
+                return true;
+            else
+                return false;
+        }
+
+        return a.equals(b);
+    }
+
+    protected static boolean xequals(boolean a, boolean b) { return a == b; }
+    protected static boolean xequals(int a, int b)         { return a == b; }
+    protected static boolean xequals(char a, char b)       { return a == b; }
+
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (! (o instanceof GroupPermissionsDesc))
+            return false;
+        GroupPermissionsDesc that = (GroupPermissionsDesc) o;
+        return xequals(this.group,       that.group)
+            && xequals(this.permissions, that.permissions)
+            ;
+    }
+}



Mime
View raw message