From hcatalog-commits-return-977-apmail-incubator-hcatalog-commits-archive=incubator.apache.org@incubator.apache.org Wed Jul 25 20:31:07 2012 Return-Path: X-Original-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 84D47D342 for ; Wed, 25 Jul 2012 20:31:07 +0000 (UTC) Received: (qmail 524 invoked by uid 500); 25 Jul 2012 20:31:07 -0000 Delivered-To: apmail-incubator-hcatalog-commits-archive@incubator.apache.org Received: (qmail 491 invoked by uid 500); 25 Jul 2012 20:31:07 -0000 Mailing-List: contact hcatalog-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hcatalog-dev@incubator.apache.org Delivered-To: mailing list hcatalog-commits@incubator.apache.org Received: (qmail 484 invoked by uid 99); 25 Jul 2012 20:31:07 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Jul 2012 20:31:07 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Jul 2012 20:31:04 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 6A0AA2388A91; Wed, 25 Jul 2012 20:29:56 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1365722 [7/11] - in /incubator/hcatalog/trunk: ./ ant/ conf/ hcatalog-pig-adapter/ ivy/ src/docs/src/documentation/content/xdocs/ src/docs/src/documentation/content/xdocs/images/ src/java/org/apache/hcatalog/mapreduce/ src/test/e2e/templet... Date: Wed, 25 Jul 2012 20:29:49 -0000 To: hcatalog-commits@incubator.apache.org From: gates@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120725202956.6A0AA2388A91@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Added: incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh Wed Jul 25 20:29:44 2012 @@ -0,0 +1,237 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Support functions +# + +# Follow symlinks on Linux and Darwin +function real_script_name() { + local base=$1 + local real + if readlink -f $base >/dev/null 2>&1; then + # Darwin/Mac OS X + real=`readlink -f $base` + fi + if [[ "$?" != "0" || -z "$real" ]]; then + # Linux + local bin=$(cd -P -- "$(dirname -- "$base")">/dev/null && pwd -P) + local script="$(basename -- "$base")" + real="$bin/$script" + fi + echo "$real" +} + +function usage() { + echo "usage: $0 [start|stop|foreground]" + echo " start Start the Templeton Server" + echo " stop Stop the Templeton Server" + echo " foreground Run the Templeton Server in the foreground" + exit 1 +} + +# Print an error message and exit +function die() { + echo "templeton: $@" 1>&2 + exit 1 +} + +# Print an message +function log() { + echo "templeton: $@" +} + +# Find the templeton jar +function find_jar_path() { + for dir in "." "build/templeton" "share/templeton/"; do + local jar="$base_dir/$dir/$TEMPLETON_JAR" + if [[ -f $jar ]]; then + echo $jar + break + fi + done +} + +# Find the templeton classpath +function find_classpath() { + local classpath="" + for dir in "share/templeton/lib" "build/ivy/lib/templeton" "conf" ; do + local path="$base_dir/$dir" + + if [[ -d $path ]]; then + for jar_or_conf in $path/*; do + if [[ -z "$classpath" ]]; then + classpath="$jar_or_conf" + else + classpath="$classpath:$jar_or_conf" + fi + done + fi + done + + if [[ -n "$TEMPLETON_CONF_DIR" ]]; then + if [[ -z "$classpath" ]]; then + classpath="$TEMPLETON_CONF_DIR" + else + classpath="$classpath:$TEMPLETON_CONF_DIR" + fi + fi + + # Append hcat classpath + local hcat_classpath + hcat_classpath=`hcat -classpath` + if [[ "$?" != "0" ]]; then + die "Unable to get the hcatalog classpath" + fi + echo "$classpath:$hcat_classpath" +} + +# Check if the pid is running +function check_pid() { + local pid=$1 + if ps -p $pid > /dev/null; then + return 0 + else + return 1 + fi +} + +# Start the templeton server in the foreground +function foreground_templeton() { + $start_cmd +} + +# Start the templeton server in the background. Record the PID for +# later use. +function start_templeton() { + if [[ -f $PID_FILE ]]; then + # Check if there is a server running + local pid=`cat $PID_FILE` + if check_pid $pid; then + die "already running on process $pid" + fi + fi + + log "starting ..." + log "$start_cmd" + nohup $start_cmd >>$CONSOLE_LOG 2>>$ERROR_LOG & + local pid=$! + + if [[ -z "${pid}" ]] ; then # we failed right off + die "failed to start. Check logs in " `dirname $ERROR_LOG` + fi + + sleep $SLEEP_TIME_AFTER_START + + if check_pid $pid; then + echo $pid > $PID_FILE + log "starting ... started." + else + die "failed to start. Check logs in " `dirname $ERROR_LOG` + fi +} + +# Stop a running server +function stop_templeton() { + local pid + if [[ -f $PID_FILE ]]; then + # Check if there is a server running + local check=`cat $PID_FILE` + if check_pid $check; then + pid=$check + fi + fi + + if [[ -z "$pid" ]]; then + log "no running server found" + else + log "stopping ..." + kill $pid + sleep $SLEEP_TIME_AFTER_START + if check_pid $pid; then + die "failed to stop" + else + log "stopping ... stopped" + fi + fi +} + +# +# Build command line and run +# + +this=`real_script_name "${BASH_SOURCE-$0}"` +this_bin=`dirname $this` +base_dir="$this_bin/.." + +if [[ -f "$base_dir/libexec/templeton_config.sh" ]]; then + . "$base_dir/libexec/templeton_config.sh" +else + . "$this_bin/templeton_config.sh" +fi + +JAR=`find_jar_path` +if [[ -z "$JAR" ]]; then + die "No templeton jar found" +fi + +CLASSPATH=`find_classpath` +if [[ -z "$CLASSPATH" ]]; then + die "No classpath or jars found" +fi +CLASSPATH="$JAR:$CLASSPATH" + +if [[ -z "$HADOOP_CLASSPATH" ]]; then + export HADOOP_CLASSPATH="$CLASSPATH" +else + export HADOOP_CLASSPATH="$CLASSPATH:$HADOOP_CLASSPATH" +fi + +if [[ -z "$TEMPLETON_LOG4J" ]]; then + if [[ -f "$base_dir/conf/templeton-log4j.properties" ]]; then + TEMPLETON_LOG4J="$base_dir/conf/templeton-log4j.properties"; + elif [[ -f "$base_dir/conf/templeton-log4j.properties" ]]; then + TEMPLETON_LOG4J="$base_dir/conf/templeton-log4j.properties"; + else + TEMPLETON_LOG4J="templeton-log4j.properties"; + fi +fi + +export HADOOP_USER_CLASSPATH_FIRST=true +export HADOOP_OPTS="-Dtempleton.log.dir=$TEMPLETON_LOG_DIR -Dlog4j.configuration=$TEMPLETON_LOG4J" + +start_cmd="$HADOOP_PREFIX/bin/hadoop jar $JAR org.apache.hcatalog.templeton.Main " + + +cmd=$1 +case $cmd in + start) + start_templeton + ;; + stop) + stop_templeton + ;; + foreground) + foreground_templeton + ;; + *) + usage + ;; +esac + +log "done" Propchange: incubator/hcatalog/trunk/webhcat/svr/src/main/bin/templeton_server.sh ------------------------------------------------------------------------------ svn:executable = * Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred; + +import java.io.IOException; +import java.net.InetSocketAddress; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.UserGroupInformation; + +/* + * Communicate with the JobTracker as a specific user. + */ +public class TempletonJobTracker { + private JobSubmissionProtocol cnx; + + /** + * Create a connection to the Job Tracker. + */ + public TempletonJobTracker(UserGroupInformation ugi, + InetSocketAddress addr, + Configuration conf) + throws IOException + { + cnx = (JobSubmissionProtocol) + RPC.getProxy(JobSubmissionProtocol.class, + JobSubmissionProtocol.versionID, + addr, + ugi, + conf, + NetUtils.getSocketFactory(conf, + JobSubmissionProtocol.class)); + } + + /** + * Grab a handle to a job that is already known to the JobTracker. + * + * @return Profile of the job, or null if not found. + */ + public JobProfile getJobProfile(JobID jobid) + throws IOException + { + return cnx.getJobProfile(jobid); + } + + /** + * Grab a handle to a job that is already known to the JobTracker. + * + * @return Status of the job, or null if not found. + */ + public JobStatus getJobStatus(JobID jobid) + throws IOException + { + return cnx.getJobStatus(jobid); + } + + + /** + * Kill a job. + */ + public void killJob(JobID jobid) + throws IOException + { + cnx.killJob(jobid); + } + + /** + * Get all the jobs submitted. + */ + public JobStatus[] getAllJobs() + throws IOException + { + return cnx.getAllJobs(); + } + + /** + * Close the connection to the Job Tracker. + */ + public void close() { + RPC.stopProxy(cnx); + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/AppConfig.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.File; +import java.net.URL; +import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.VersionInfo; +import org.apache.hcatalog.templeton.tool.JobState; +import org.apache.hcatalog.templeton.tool.ZooKeeperCleanup; +import org.apache.hcatalog.templeton.tool.ZooKeeperStorage; + +/** + * The configuration for Templeton. This merges the normal Hadoop + * configuration with the Templeton specific variables. + * + * The Templeton configuration variables are described in + * templeton-default.xml + * + * The Templeton specific configuration is split into two layers + * + * 1. templeton-default.xml - All the configuration variables that + * Templeton needs. These are the defaults that ship with the app + * and should only be changed be the app developers. + * + * 2. templeton-site.xml - The (possibly empty) configuration that the + * system administrator can set variables for their Hadoop cluster. + * + * The configuration files are loaded in this order with later files + * overriding earlier ones. + * + * To find the configuration files, we first attempt to load a file + * from the CLASSPATH and then look in the directory specified in the + * TEMPLETON_HOME environment variable. + * + * In addition the configuration files may access the special env + * variable env for all environment variables. For example, the + * hadoop executable could be specified using: + *
+ *      ${env.HADOOP_PREFIX}/bin/hadoop
+ *
+ */ +public class AppConfig extends Configuration { + public static final String[] HADOOP_CONF_FILENAMES = { + "core-default.xml", "core-site.xml", "mapred-default.xml", "mapred-site.xml" + }; + + public static final String[] HADOOP_PREFIX_VARS = { + "HADOOP_PREFIX", "HADOOP_HOME" + }; + + public static final String TEMPLETON_HOME_VAR = "TEMPLETON_HOME"; + + public static final String[] TEMPLETON_CONF_FILENAMES = { + "templeton-default.xml", + "templeton-site.xml" + }; + + public static final String PORT = "templeton.port"; + public static final String EXEC_ENCODING_NAME = "templeton.exec.encoding"; + public static final String EXEC_ENVS_NAME = "templeton.exec.envs"; + public static final String EXEC_MAX_BYTES_NAME = "templeton.exec.max-output-bytes"; + public static final String EXEC_MAX_PROCS_NAME = "templeton.exec.max-procs"; + public static final String EXEC_TIMEOUT_NAME = "templeton.exec.timeout"; + public static final String HADOOP_NAME = "templeton.hadoop"; + public static final String HADOOP_CONF_DIR = "templeton.hadoop.conf.dir"; + public static final String HCAT_NAME = "templeton.hcat"; + public static final String HIVE_ARCHIVE_NAME = "templeton.hive.archive"; + public static final String HIVE_PATH_NAME = "templeton.hive.path"; + public static final String HIVE_PROPS_NAME = "templeton.hive.properties"; + public static final String LIB_JARS_NAME = "templeton.libjars"; + public static final String PIG_ARCHIVE_NAME = "templeton.pig.archive"; + public static final String PIG_PATH_NAME = "templeton.pig.path"; + public static final String STREAMING_JAR_NAME = "templeton.streaming.jar"; + public static final String TEMPLETON_JAR_NAME = "templeton.jar"; + public static final String OVERRIDE_JARS_NAME = "templeton.override.jars"; + public static final String OVERRIDE_JARS_ENABLED = "templeton.override.enabled"; + public static final String KERBEROS_SECRET = "templeton.kerberos.secret"; + public static final String KERBEROS_PRINCIPAL = "templeton.kerberos.principal"; + public static final String KERBEROS_KEYTAB = "templeton.kerberos.keytab"; + + public static final String CALLBACK_INTERVAL_NAME + = "templeton.callback.retry.interval"; + public static final String CALLBACK_RETRY_NAME + = "templeton.callback.retry.attempts"; + public static final String HADOOP_END_INTERVAL_NAME = "job.end.retry.interval"; + public static final String HADOOP_END_RETRY_NAME = "job.end.retry.attempts"; + public static final String HADOOP_END_URL_NAME = "job.end.notification.url"; + public static final String HADOOP_SPECULATIVE_NAME + = "mapred.map.tasks.speculative.execution"; + + private static final Log LOG = LogFactory.getLog(AppConfig.class); + + public AppConfig() { + init(); + LOG.info("Using Hadoop version " + VersionInfo.getVersion()); + } + + private void init() { + for (Map.Entry e : System.getenv().entrySet()) + set("env." + e.getKey(), e.getValue()); + + String templetonDir = getTempletonDir(); + for (String fname : TEMPLETON_CONF_FILENAMES) + if (! loadOneClasspathConfig(fname)) + loadOneFileConfig(templetonDir, fname); + + String hadoopConfDir = getHadoopConfDir(); + for (String fname : HADOOP_CONF_FILENAMES) + loadOneFileConfig(hadoopConfDir, fname); + } + + public void startCleanup() { + JobState.getStorageInstance(this).startCleanup(this); + } + + public String getHadoopConfDir() { + return get(HADOOP_CONF_DIR); + } + + public static String getTempletonDir() { + return System.getenv(TEMPLETON_HOME_VAR); + } + + private boolean loadOneFileConfig(String dir, String fname) { + if (dir != null) { + File f = new File(dir, fname); + if (f.exists()) { + addResource(new Path(f.getAbsolutePath())); + LOG.debug("loaded config file " + f.getAbsolutePath()); + return true; + } + } + return false; + } + + private boolean loadOneClasspathConfig(String fname) { + URL x = getResource(fname); + if (x != null) { + addResource(x); + LOG.debug("loaded config from classpath " + x); + return true; + } + + return false; + } + + public String templetonJar() { return get(TEMPLETON_JAR_NAME); } + public String libJars() { return get(LIB_JARS_NAME); } + public String clusterHadoop() { return get(HADOOP_NAME); } + public String clusterHcat() { return get(HCAT_NAME); } + public String pigPath() { return get(PIG_PATH_NAME); } + public String pigArchive() { return get(PIG_ARCHIVE_NAME); } + public String hivePath() { return get(HIVE_PATH_NAME); } + public String hiveArchive() { return get(HIVE_ARCHIVE_NAME); } + public String streamingJar() { return get(STREAMING_JAR_NAME); } + public String kerberosSecret() { return get(KERBEROS_SECRET); } + public String kerberosPrincipal(){ return get(KERBEROS_PRINCIPAL); } + public String kerberosKeytab() { return get(KERBEROS_KEYTAB); } + + public String[] overrideJars() { + if (getBoolean(OVERRIDE_JARS_ENABLED, true)) + return getStrings(OVERRIDE_JARS_NAME); + else + return null; + } + public String overrideJarsString() { + if (getBoolean(OVERRIDE_JARS_ENABLED, true)) + return get(OVERRIDE_JARS_NAME); + else + return null; + } + + public long zkCleanupInterval() { + return getLong(ZooKeeperCleanup.ZK_CLEANUP_INTERVAL, + (1000L * 60L * 60L * 12L)); } + public long zkMaxAge() { + return getLong(ZooKeeperCleanup.ZK_CLEANUP_MAX_AGE, + (1000L * 60L * 60L * 24L * 7L)); } + public String zkHosts() { return get(ZooKeeperStorage.ZK_HOSTS); } + public int zkSessionTimeout() { return getInt(ZooKeeperStorage.ZK_SESSION_TIMEOUT, + 30000); } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BadParam.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * Missing required or badly configured paramater. + */ +public class BadParam extends SimpleWebException { + public BadParam(String msg) { + super(400, msg); + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/BusyException.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * Simple "we are busy, try again" exception. + */ +public class BusyException extends SimpleWebException { + public BusyException() { + super(503, "Busy, please retry"); + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CallbackFailedException.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * The callback failed when it tried to reach the callback URL. + */ +public class CallbackFailedException extends SimpleWebException { + public CallbackFailedException(String msg) { + super(400, msg); + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CatchallExceptionMapper.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Map all exceptions to the Jersey response. This lets us have nice + * results in the error body. + */ +@Provider +public class CatchallExceptionMapper + implements ExceptionMapper +{ + private static final Log LOG = LogFactory.getLog(CatchallExceptionMapper.class); + + public Response toResponse(Exception e) { + LOG.error(e.getMessage(), e); + return SimpleWebException.buildMessage(500, null, e.getMessage()); + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ColumnDesc.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * A description of the column to create. + */ +@XmlRootElement +public class ColumnDesc extends GroupPermissionsDesc { + public String name; + public String type; + public String comment; + + public ColumnDesc() {} + + /** + * Create a new ColumnDesc + */ + public ColumnDesc(String name, String type, String comment) { + this.name = name; + this.type = type; + this.comment = comment; + } + + public String toString() { + return String.format("ColumnDesc(name=%s, type=%s, comment=%s)", + name, type, comment); + } + + public boolean equals(Object o) { + if (this == o) + return true; + if (! (o instanceof ColumnDesc)) + return false; + ColumnDesc that = (ColumnDesc) o; + return xequals(this.name, that.name) + && xequals(this.type, that.type) + && xequals(this.comment, that.comment) + && super.equals(that) + ; + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteBean.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * CompleteBean - The results of an CompleteDelegator run. + */ +public class CompleteBean { + public String status; + + public CompleteBean() {} + + /** + * Create a new CompleteBean + * + * @param status run status + */ + public CompleteBean(String status) { + this.status = status; + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/CompleteDelegator.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import java.net.URL; +import java.net.MalformedURLException; +import java.util.Date; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.mapred.JobID; +import org.apache.hadoop.mapred.JobProfile; +import org.apache.hadoop.mapred.JobStatus; +import org.apache.hadoop.mapred.JobTracker; +import org.apache.hadoop.mapred.TempletonJobTracker; +import org.apache.hcatalog.templeton.tool.JobState; +import org.apache.hcatalog.templeton.tool.TempletonUtils; + +/** + * Complete a job. This will run the callback if + * + * - the job is done + * - there is a callback + * - the callback has not yet been called + * + * There is a small chance for a race condition if two callers run + * this at the same time. That should never happen. + * + * We use a Hadoop config var to notify this class on the completion + * of a job. Hadoop will call use multiple times in the event of + * failure. Even if the failure is that the client callback failed. + * + * See LauncherDelegator for the HADOOP_END_RETRY* vars that are set. + */ +public class CompleteDelegator extends TempletonDelegator { + private static final Log LOG = LogFactory.getLog(CompleteDelegator.class); + + public CompleteDelegator(AppConfig appConf) { + super(appConf); + } + + public CompleteBean run(String id) + throws CallbackFailedException, IOException + { + if (id == null) + acceptWithError("No jobid given"); + + JobState state = null; + try { + state = new JobState(id, Main.getAppConfigInstance()); + if (state.getCompleteStatus() == null) + failed("Job not yet complete", null); + + Long notified = state.getNotifiedTime(); + if (notified != null) + return acceptWithError("Callback already run on " + + new Date(notified.longValue())); + + String callback = state.getCallback(); + if (callback == null) + return new CompleteBean("No callback registered"); + + try { + doCallback(state.getId(), callback); + } catch (Exception e) { + failed("Callback failed " + callback + " for " + id, e); + } + + state.setNotifiedTime(System.currentTimeMillis()); + return new CompleteBean("Callback sent"); + } finally { + if (state != null) + state.close(); + } + } + + /** + * Call the callback url with the jobid to let them know it's + * finished. If the url has the string $jobId in it, it will be + * replaced with the completed jobid. + */ + public static void doCallback(String jobid, String url) + throws MalformedURLException, IOException + { + if (url.contains("$jobId")) + url = url.replace("$jobId", jobid); + TempletonUtils.fetchUrl(new URL(url)); + } + + private void failed(String msg, Exception e) + throws CallbackFailedException + { + if (e != null) + LOG.error(msg, e); + else + LOG.error(msg); + throw new CallbackFailedException(msg); + } + + private CompleteBean acceptWithError(String msg) { + LOG.error(msg); + return new CompleteBean(msg); + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DatabaseDesc.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.util.Map; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * A description of the database to create. + */ +@XmlRootElement +public class DatabaseDesc extends GroupPermissionsDesc { + public boolean ifNotExists; + public String database; + public String comment; + public String location; + public Map properties; + + public DatabaseDesc() {} + + public String toString() { + return String.format("DatabaseDesc(database=%s, comment=%s, location=%s, " + + "properties=%s)", database, comment, location, properties); + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import org.apache.hadoop.mapred.JobID; +import org.apache.hadoop.mapred.JobProfile; +import org.apache.hadoop.mapred.JobStatus; +import org.apache.hadoop.mapred.JobTracker; +import org.apache.hadoop.mapred.TempletonJobTracker; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hcatalog.templeton.tool.JobState; + +/** + * Delete a job + */ +public class DeleteDelegator extends TempletonDelegator { + public DeleteDelegator(AppConfig appConf) { + super(appConf); + } + + public QueueStatusBean run(String user, String id) + throws NotAuthorizedException, BadParam, IOException + { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); + TempletonJobTracker tracker = null; + JobState state = null; + try { + tracker = new TempletonJobTracker(ugi, + JobTracker.getAddress(appConf), + appConf); + JobID jobid = StatusDelegator.StringToJobID(id); + if (jobid == null) + throw new BadParam("Invalid jobid: " + id); + tracker.killJob(jobid); + state = new JobState(id, Main.getAppConfigInstance()); + String childid = state.getChildId(); + if (childid != null) + tracker.killJob(StatusDelegator.StringToJobID(childid)); + return StatusDelegator.makeStatus(tracker, jobid, state); + } catch (IllegalStateException e) { + throw new BadParam(e.getMessage()); + } finally { + if (tracker != null) + tracker.close(); + if (state != null) + state.close(); + } + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/EnqueueBean.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * EnqueueBean - The results of a call that enqueues a Hadoop job. + */ +public class EnqueueBean { + public String id; + + public EnqueueBean() {} + + /** + * Create a new EnqueueBean. + * + * @param id job id + */ + public EnqueueBean(String id) { + this.id = id; + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecBean.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +/** + * ExecBean - The results of an exec call. + */ +public class ExecBean { + public String stdout; + public String stderr; + public int exitcode; + + public ExecBean() {} + + /** + * Create a new ExecBean. + * + * @param stdout standard output of the the program. + * @param stderr error output of the the program. + * @param exitcode exit code of the program. + */ + public ExecBean(String stdout, String stderr, int exitcode) { + this.stdout = stdout; + this.stderr = stderr; + this.exitcode = exitcode; + } + + public String toString() { + return String.format("ExecBean(stdout=%s, stderr=%s, exitcode=%s)", + stdout, stderr, exitcode); + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecService.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.apache.commons.exec.ExecuteException; + +public interface ExecService { + public ExecBean run(String program, List args, + Map env) + throws NotAuthorizedException, BusyException, ExecuteException, IOException; + + public ExecBean runUnlimited(String program, List args, + Map env) + throws NotAuthorizedException, ExecuteException, IOException; +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ExecServiceImpl.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Semaphore; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.exec.ExecuteWatchdog; +import org.apache.commons.exec.PumpStreamHandler; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Execute a local program. This is a singleton service that will + * execute programs as non-privileged users on the local box. See + * ExecService.run and ExecService.runUnlimited for details. + */ +public class ExecServiceImpl implements ExecService { + private static final Log LOG = LogFactory.getLog(ExecServiceImpl.class); + private static AppConfig appConf = Main.getAppConfigInstance(); + + private static volatile ExecServiceImpl theSingleton; + + /** + * Retrieve the singleton. + */ + public static synchronized ExecServiceImpl getInstance() { + if (theSingleton == null) { + theSingleton = new ExecServiceImpl(); + } + return theSingleton; + } + + private Semaphore avail; + + private ExecServiceImpl() { + avail = new Semaphore(appConf.getInt(AppConfig.EXEC_MAX_PROCS_NAME, 16)); + } + + /** + * Run the program synchronously as the given user. We rate limit + * the number of processes that can simultaneously created for + * this instance. + * + * @param program The program to run + * @param args Arguments to pass to the program + * @param env Any extra environment variables to set + * @return The result of the run. + */ + public ExecBean run(String program, List args, + Map env) + throws NotAuthorizedException, BusyException, ExecuteException, IOException + { + boolean aquired = false; + try { + aquired = avail.tryAcquire(); + if (aquired) { + return runUnlimited(program, args, env); + } else { + throw new BusyException(); + } + } finally { + if (aquired) { + avail.release(); + } + } + } + + /** + * Run the program synchronously as the given user. Warning: + * CommandLine will trim the argument strings. + * + * @param program The program to run. + * @param args Arguments to pass to the program + * @param env Any extra environment variables to set + * @return The result of the run. + */ + public ExecBean runUnlimited(String program, List args, + Map env) + throws NotAuthorizedException, ExecuteException, IOException + { + try { + return auxRun(program, args, env); + } catch (IOException e) { + File cwd = new java.io.File("."); + if (cwd.canRead() && cwd.canWrite()) + throw e; + else + throw new IOException("Invalid permissions on Templeton directory: " + + cwd.getCanonicalPath()); + } + } + + private ExecBean auxRun(String program, List args, Map env) + throws NotAuthorizedException, ExecuteException, IOException + { + DefaultExecutor executor = new DefaultExecutor(); + executor.setExitValues(null); + + // Setup stdout and stderr + int nbytes = appConf.getInt(AppConfig.EXEC_MAX_BYTES_NAME, -1); + ByteArrayOutputStream outStream = new MaxByteArrayOutputStream(nbytes); + ByteArrayOutputStream errStream = new MaxByteArrayOutputStream(nbytes); + executor.setStreamHandler(new PumpStreamHandler(outStream, errStream)); + + // Only run for N milliseconds + int timeout = appConf.getInt(AppConfig.EXEC_TIMEOUT_NAME, 0); + ExecuteWatchdog watchdog = new ExecuteWatchdog(timeout); + executor.setWatchdog(watchdog); + + CommandLine cmd = makeCommandLine(program, args); + + LOG.info("Running: " + cmd); + ExecBean res = new ExecBean(); + res.exitcode = executor.execute(cmd, execEnv(env)); + String enc = appConf.get(AppConfig.EXEC_ENCODING_NAME); + res.stdout = outStream.toString(enc); + res.stderr = errStream.toString(enc); + + return res; + } + + private CommandLine makeCommandLine(String program, + List args) + throws NotAuthorizedException, IOException + { + String path = validateProgram(program); + CommandLine cmd = new CommandLine(path); + if (args != null) + for (String arg : args) + cmd.addArgument(arg, false); + + return cmd; + } + + /** + * Build the environment used for all exec calls. + * + * @return The environment variables. + */ + public Map execEnv(Map env) { + HashMap res = new HashMap(); + + for (String key : appConf.getStrings(AppConfig.EXEC_ENVS_NAME)) { + String val = System.getenv(key); + if (val != null) { + res.put(key, val); + } + } + if (env != null) + res.putAll(env); + for(Map.Entry envs : res.entrySet()){ + LOG.info("Env " + envs.getKey() + "=" + envs.getValue()); + } + return res; + } + + /** + * Given a program name, lookup the fully qualified path. Throws + * an exception if the program is missing or not authorized. + * + * @param path The path of the program. + * @return The path of the validated program. + */ + public String validateProgram(String path) + throws NotAuthorizedException, IOException + { + File f = new File(path); + if (f.canExecute()) { + return f.getCanonicalPath(); + } else { + throw new NotAuthorizedException("Unable to access program: " + path); + } + } +} Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java?rev=1365722&view=auto ============================================================================== --- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java (added) +++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/GroupPermissionsDesc.java Wed Jul 25 20:29:44 2012 @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hcatalog.templeton; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * The base create permissions for ddl objects. + */ +public abstract class GroupPermissionsDesc { + public String group; + public String permissions; + + public GroupPermissionsDesc() {} + + protected static boolean xequals(Object a, Object b) { + if (a == null) { + if (b == null) + return true; + else + return false; + } + + return a.equals(b); + } + + protected static boolean xequals(boolean a, boolean b) { return a == b; } + protected static boolean xequals(int a, int b) { return a == b; } + protected static boolean xequals(char a, char b) { return a == b; } + + public boolean equals(Object o) { + if (this == o) + return true; + if (! (o instanceof GroupPermissionsDesc)) + return false; + GroupPermissionsDesc that = (GroupPermissionsDesc) o; + return xequals(this.group, that.group) + && xequals(this.permissions, that.permissions) + ; + } +}