incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1365722 [9/11] - in /incubator/hcatalog/trunk: ./ ant/ conf/ hcatalog-pig-adapter/ ivy/ src/docs/src/documentation/content/xdocs/ src/docs/src/documentation/content/xdocs/images/ src/java/org/apache/hcatalog/mapreduce/ src/test/e2e/templet...
Date Wed, 25 Jul 2012 20:29:49 GMT
Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SecureProxySupport.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SecureProxySupport.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SecureProxySupport.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SecureProxySupport.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.io.Text;
+import org.apache.thrift.TException;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+
+/**
+ * Helper class to run jobs using Kerberos security.  Always safe to
+ * use these methods, it's a noop if security is not enabled.
+ */
+public class SecureProxySupport {
+    private Path tokenPath;
+    private final String HCAT_SERVICE = "hcat";
+    private boolean isEnabled;
+    private String user;
+
+    public SecureProxySupport() {
+        isEnabled = UserGroupInformation.isSecurityEnabled();
+    }
+
+    private static final Log LOG = LogFactory.getLog(SecureProxySupport.class);
+    
+    /**
+     * The file where we store the auth token
+     */
+    public Path getTokenPath() { return( tokenPath ); }
+
+    /**
+     * The token to pass to hcat.
+     */
+    public String getHcatServiceStr() { return( HCAT_SERVICE ); }
+
+    /**
+     * Create the delegation token.
+     */
+    public Path open(String user, Configuration conf)
+        throws IOException, InterruptedException
+    {
+        close();
+        if (isEnabled) {
+            this.user = user;
+            File t = File.createTempFile("templeton", null);
+            tokenPath = new Path(t.toURI());
+            Token fsToken = getFSDelegationToken(user, conf);
+            String hcatTokenStr;
+            try {
+                hcatTokenStr = buildHcatDelegationToken(user);
+            } catch (Exception e) {
+                throw new IOException(e);
+            }
+            Token<?> msToken = new Token();
+            msToken.decodeFromUrlString(hcatTokenStr);
+            msToken.setService(new Text(HCAT_SERVICE));
+            writeProxyDelegationTokens(fsToken, msToken, conf, user, tokenPath);
+            
+        }
+        return tokenPath;
+    }
+
+    /**
+     * Cleanup
+     */
+    public void close() {
+        if (tokenPath != null) {
+            new File(tokenPath.toUri()).delete();
+            tokenPath = null;
+        }
+    }
+
+    /**
+     * Add Hadoop env variables.
+     */
+    public void addEnv(Map<String, String> env) {
+        if (isEnabled) {
+            env.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION,
+                    getTokenPath().toUri().getPath());
+        }
+    }
+
+    /**
+     * Add hcat args.
+     */
+    public void addArgs(List<String> args) {
+        if (isEnabled) {
+            args.add("-D");
+            args.add("hive.metastore.token.signature=" + getHcatServiceStr());
+            args.add("-D");
+            args.add("proxy.user.name=" + user);            
+        }
+    }
+    
+    class TokenWrapper { 
+        Token<?> token;
+    }
+
+    private Token<?> getFSDelegationToken(String user,
+                                           final Configuration conf)
+        throws IOException, InterruptedException
+    {
+        LOG.info("user: " + user + " loginUser: " + UserGroupInformation.getLoginUser().getUserName());
+        final UserGroupInformation ugi = UgiFactory.getUgi(user);
+
+       final TokenWrapper twrapper = new TokenWrapper();
+       ugi.doAs(new PrivilegedExceptionAction<Object>() {
+           public Object run() throws IOException {
+               FileSystem fs = FileSystem.get(conf);
+               twrapper.token =  fs.getDelegationToken(ugi.getShortUserName());
+               return null;
+           }
+       });
+       return twrapper.token;
+       
+    }
+
+    private void writeProxyDelegationTokens(final Token<?> fsToken,
+            final Token<?> msToken,
+            final Configuration conf,
+            String user,
+            final Path tokenPath)
+                    throws IOException, InterruptedException{
+        
+        
+        LOG.info("user: " + user + " loginUser: " + UserGroupInformation.getLoginUser().getUserName());
+        final UserGroupInformation ugi  =  UgiFactory.getUgi(user);
+
+        
+        ugi.doAs(new PrivilegedExceptionAction<Object>() {
+                     public Object run() throws IOException {
+                         Credentials cred = new Credentials();
+                         cred.addToken(fsToken.getService(), fsToken);
+                         cred.addToken(msToken.getService(), msToken);
+                         cred.writeTokenStorageFile(tokenPath, conf);
+                         return null;
+                     }
+                 });
+        
+    }
+    
+    private String buildHcatDelegationToken(String user)
+        throws IOException, InterruptedException, MetaException, TException
+    {
+        HiveConf c = new HiveConf();
+        final HiveMetaStoreClient client = new HiveMetaStoreClient(c);
+        LOG.info("user: " + user + " loginUser: " + UserGroupInformation.getLoginUser().getUserName());
+        final TokenWrapper twrapper = new TokenWrapper();
+        final UserGroupInformation ugi = UgiFactory.getUgi(user);
+        String s = ugi.doAs(new PrivilegedExceptionAction<String>() {
+                                public String run()
+                                    throws IOException, MetaException, TException
+                                {
+                                    String u = ugi.getUserName();
+                                    return client.getDelegationToken(u);
+                                }
+                            });
+        return s;
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,827 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.FormParam;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.ws.rs.core.UriInfo;
+import org.apache.commons.exec.ExecuteException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
+import org.apache.hcatalog.templeton.tool.TempletonUtils;
+
+/**
+ * The Templeton Web API server.
+ */
+@Path("/v1")
+public class Server {
+    public static final String VERSION = "v1";
+
+    /**
+     * The status message.  Always "ok"
+     */
+    public static final Map<String, String> STATUS_OK = createStatusMsg();
+
+    /**
+     * The list of supported api versions.
+     */
+    public static final Map<String, Object> SUPPORTED_VERSIONS = createVersions();
+
+    /**
+     * The list of supported return formats.  Always json.
+     */
+    public static final Map<String, Object> SUPPORTED_FORMATS = createFormats();
+
+    // Build the status message for the /status call.
+    private static Map<String, String> createStatusMsg() {
+        HashMap<String, String> res = new HashMap<String, String>();
+        res.put("status", "ok");
+        res.put("version", VERSION);
+
+        return Collections.unmodifiableMap(res);
+    }
+
+    // Build the versions list.
+    private static Map<String, Object> createVersions() {
+        ArrayList<String> versions = new ArrayList<String>();
+        versions.add(VERSION);
+
+        HashMap<String, Object> res = new HashMap<String, Object>();
+        res.put("supportedVersions", versions);
+        res.put("version", VERSION);
+
+        return Collections.unmodifiableMap(res);
+    }
+
+    // Build the supported formats list
+    private static Map<String, Object> createFormats() {
+        ArrayList<String> formats = new ArrayList<String>();
+        formats.add(MediaType.APPLICATION_JSON);
+        HashMap<String, Object> res = new HashMap<String, Object>();
+        res.put("responseTypes", formats);
+
+        return Collections.unmodifiableMap(res);
+    }
+
+    protected static ExecService execService = ExecServiceImpl.getInstance();
+    private static AppConfig appConf = Main.getAppConfigInstance();
+
+    // The SecurityContext set by AuthFilter
+    private @Context SecurityContext theSecurityContext;
+
+    // The uri requested
+    private @Context UriInfo theUriInfo;
+
+    private static final Log LOG = LogFactory.getLog(Server.class);
+
+    /**
+     * Check the status of this server.  Always OK.
+     */
+    @GET
+    @Path("status")
+    @Produces({MediaType.APPLICATION_JSON})
+    public Map<String, String> status() {
+        return STATUS_OK;
+    }
+
+    /**
+     * Check the supported request formats of this server.
+     */
+    @GET
+    @Produces({MediaType.APPLICATION_JSON})
+    public Map<String, Object> requestFormats() {
+        return SUPPORTED_FORMATS;
+    }
+
+    /**
+     * Check the version(s) supported by this server.
+     */
+    @GET
+    @Path("version")
+    @Produces({MediaType.APPLICATION_JSON})
+    public Map<String, Object> version() {
+        return SUPPORTED_VERSIONS;
+    }
+
+    /**
+     * Execute an hcat ddl expression on the local box.  It is run
+     * as the authenticated user and rate limited.
+     */
+    @POST
+    @Path("ddl")
+    @Produces({MediaType.APPLICATION_JSON})
+    public ExecBean ddl(@FormParam("exec") String exec,
+                        @FormParam("group") String group,
+                        @FormParam("permissions") String permissions)
+        throws NotAuthorizedException, BusyException, BadParam,
+        ExecuteException, IOException
+    {
+        verifyUser();
+        verifyParam(exec, "exec");
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.run(getUser(), exec, false, group, permissions);
+    }
+
+    /**
+     * List all the tables in an hcat database.
+     */
+    @GET
+    @Path("ddl/database/{db}/table")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response listTables(@PathParam("db") String db,
+                               @QueryParam("like") String tablePattern)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        if (! TempletonUtils.isset(tablePattern))
+            tablePattern = "*";
+        return d.listTables(getUser(), db, tablePattern);
+    }
+
+    /**
+     * Create a new table.
+     */
+    @PUT
+    @Path("ddl/database/{db}/table/{table}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response createTable(@PathParam("db") String db,
+                                @PathParam("table") String table,
+                                TableDesc desc)
+        throws SimpleWebException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+        desc.table = table;
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.createTable(getUser(), db, desc);
+    }
+
+    /**
+     * Create a new table like another table.
+     */
+    @PUT
+    @Path("ddl/database/{db}/table/{existingTable}/like/{newTable}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response createTableLike(@PathParam("db") String db,
+                                    @PathParam("existingTable") String existingTable,
+                                    @PathParam("newTable") String newTable,
+                                    TableLikeDesc desc)
+        throws SimpleWebException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(existingTable, ":existingTable");
+        verifyDdlParam(newTable, ":newTable");
+        desc.existingTable = existingTable;
+        desc.newTable = newTable;
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.createTableLike(getUser(), db, desc);
+    }
+
+    /**
+     * Describe an hcat table.  This is normally a simple list of
+     * columns (using "desc table"), but the extended format will show
+     * more information (using "show table extended like").
+     */
+    @GET
+    @Path("ddl/database/{db}/table/{table}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response descTable(@PathParam("db") String db,
+                              @PathParam("table") String table,
+                              @QueryParam("format") String format)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        if ("extended".equals(format))
+            return d.descExtendedTable(getUser(), db, table);
+        else
+            return d.descTable(getUser(), db, table, false);
+    }
+
+    /**
+     * Drop an hcat table.
+     */
+    @DELETE
+    @Path("ddl/database/{db}/table/{table}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response dropTable(@PathParam("db") String db,
+                              @PathParam("table") String table,
+                              @QueryParam("ifExists") boolean ifExists,
+                              @QueryParam("group") String group,
+                              @QueryParam("permissions") String permissions)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.dropTable(getUser(), db, table, ifExists, group, permissions);
+    }
+
+    /**
+     * Rename an hcat table.
+     */
+    @POST
+    @Path("ddl/database/{db}/table/{table}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response renameTable(@PathParam("db") String db,
+                                @PathParam("table") String oldTable,
+                                @FormParam("rename") String newTable,
+                                @FormParam("group") String group,
+                                @FormParam("permissions") String permissions)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(oldTable, ":table");
+        verifyDdlParam(newTable, "rename");
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.renameTable(getUser(), db, oldTable, newTable, group, permissions);
+    }
+
+    /**
+     * Describe a single property on an hcat table.
+     */
+    @GET
+    @Path("ddl/database/{db}/table/{table}/property/{property}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response descOneTableProperty(@PathParam("db") String db,
+                                         @PathParam("table") String table,
+                                         @PathParam("property") String property)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+        verifyDdlParam(property, ":property");
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.descTableProperty(getUser(), db, table, property);
+    }
+
+    /**
+     * List all the properties on an hcat table.
+     */
+    @GET
+    @Path("ddl/database/{db}/table/{table}/property")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response listTableProperties(@PathParam("db") String db,
+                                        @PathParam("table") String table)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.listTableProperties(getUser(), db, table);
+    }
+
+    /**
+     * Add a single property on an hcat table.
+     */
+    @PUT
+    @Path("ddl/database/{db}/table/{table}/property/{property}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response addOneTableProperty(@PathParam("db") String db,
+                                        @PathParam("table") String table,
+                                        @PathParam("property") String property,
+                                        TablePropertyDesc desc)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+        verifyDdlParam(property, ":property");
+        desc.name = property;
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.addOneTableProperty(getUser(), db, table, desc);
+    }
+
+    /**
+     * List all the partitions in an hcat table.
+     */
+    @GET
+    @Path("ddl/database/{db}/table/{table}/partition")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response listPartitions(@PathParam("db") String db,
+                                   @PathParam("table") String table)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.listPartitions(getUser(), db, table);
+    }
+
+    /**
+     * Describe a single partition in an hcat table.
+     */
+    @GET
+    @Path("ddl/database/{db}/table/{table}/partition/{partition}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response descPartition(@PathParam("db") String db,
+                                  @PathParam("table") String table,
+                                  @PathParam("partition") String partition)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+        verifyParam(partition, ":partition");
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.descOnePartition(getUser(), db, table, partition);
+    }
+
+    /**
+     * Create a partition in an hcat table.
+     */
+    @PUT
+    @Path("ddl/database/{db}/table/{table}/partition/{partition}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response addOnePartition(@PathParam("db") String db,
+                                    @PathParam("table") String table,
+                                    @PathParam("partition") String partition,
+                                    PartitionDesc desc)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+        verifyParam(partition, ":partition");
+        desc.partition = partition;
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.addOnePartition(getUser(), db, table, desc);
+    }
+
+    /**
+     * Drop a partition in an hcat table.
+     */
+    @DELETE
+    @Path("ddl/database/{db}/table/{table}/partition/{partition}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response dropPartition(@PathParam("db") String db,
+                                  @PathParam("table") String table,
+                                  @PathParam("partition") String partition,
+                                  @QueryParam("ifExists") boolean ifExists,
+                                  @QueryParam("group") String group,
+                                  @QueryParam("permissions") String permissions)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+        verifyParam(partition, ":partition");
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.dropPartition(getUser(), db, table, partition, ifExists,
+                               group, permissions);
+    }
+
+    /**
+     * List all databases, or those that match a pattern.
+     */
+    @GET
+    @Path("ddl/database/")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response listDatabases(@QueryParam("like") String dbPattern)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        if (! TempletonUtils.isset(dbPattern))
+            dbPattern = "*";
+        return d.listDatabases(getUser(), dbPattern);
+    }
+
+    /**
+     * Describe a database
+     */
+    @GET
+    @Path("ddl/database/{db}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response descDatabase(@PathParam("db") String db,
+                                 @QueryParam("format") String format)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.descDatabase(getUser(), db, "extended".equals(format));
+    }
+
+    /**
+     * Create a database
+     */
+    @PUT
+    @Path("ddl/database/{db}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response createDatabase(@PathParam("db") String db,
+                                   DatabaseDesc desc)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        desc.database = db;
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.createDatabase(getUser(), desc);
+    }
+
+    /**
+     * Drop a database
+     */
+    @DELETE
+    @Path("ddl/database/{db}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response dropDatabase(@PathParam("db") String db,
+                                 @QueryParam("ifExists") boolean ifExists,
+                                 @QueryParam("option") String option,
+                                 @QueryParam("group") String group,
+                                 @QueryParam("permissions") String permissions)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        if (TempletonUtils.isset(option))
+            verifyDdlParam(option, "option");
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.dropDatabase(getUser(), db, ifExists, option,
+                              group, permissions);
+    }
+
+    /**
+     * List the columns in an hcat table.  Currently the same as
+     * describe table.
+     */
+    @GET
+    @Path("ddl/database/{db}/table/{table}/column")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response listColumns(@PathParam("db") String db,
+                                @PathParam("table") String table)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.listColumns(getUser(), db, table);
+    }
+
+    /**
+     * Describe a single column in an hcat table.
+     */
+    @GET
+    @Path("ddl/database/{db}/table/{table}/column/{column}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response descColumn(@PathParam("db") String db,
+                               @PathParam("table") String table,
+                               @PathParam("column") String column)
+        throws SimpleWebException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+        verifyParam(column, ":column");
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.descOneColumn(getUser(), db, table, column);
+    }
+
+    /**
+     * Create a column in an hcat table.
+     */
+    @PUT
+    @Path("ddl/database/{db}/table/{table}/column/{column}")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response addOneColumn(@PathParam("db") String db,
+                                 @PathParam("table") String table,
+                                 @PathParam("column") String column,
+                                 ColumnDesc desc)
+        throws HcatException, NotAuthorizedException, BusyException,
+        BadParam, ExecuteException, IOException
+    {
+        verifyUser();
+        verifyDdlParam(db, ":db");
+        verifyDdlParam(table, ":table");
+        verifyParam(column, ":column");
+        verifyParam(desc.type, "type");
+        desc.name = column;
+
+        HcatDelegator d = new HcatDelegator(appConf, execService);
+        return d.addOneColumn(getUser(), db, table, desc);
+    }
+
+    /**
+     * Run a MapReduce Streaming job.
+     */
+    @POST
+    @Path("mapreduce/streaming")
+    @Produces({MediaType.APPLICATION_JSON})
+    public EnqueueBean mapReduceStreaming(@FormParam("input") List<String> inputs,
+                                          @FormParam("output") String output,
+                                          @FormParam("mapper") String mapper,
+                                          @FormParam("reducer") String reducer,
+                                          @FormParam("file") List<String> files,
+                                          @FormParam("define") List<String> defines,
+                                          @FormParam("cmdenv") List<String> cmdenvs,
+                                          @FormParam("arg") List<String> args,
+                                          @FormParam("statusdir") String statusdir,
+                                          @FormParam("callback") String callback)
+        throws NotAuthorizedException, BusyException, BadParam, QueueException,
+        ExecuteException, IOException, InterruptedException
+    {
+        verifyUser();
+        verifyParam(inputs, "input");
+        verifyParam(mapper, "mapper");
+        verifyParam(reducer, "reducer");
+
+        StreamingDelegator d = new StreamingDelegator(appConf);
+        return d.run(getUser(), inputs, output, mapper, reducer,
+                     files, defines, cmdenvs, args,
+                     statusdir, callback, getCompletedUrl());
+    }
+
+    /**
+     * Run a MapReduce Jar job.
+     */
+    @POST
+    @Path("mapreduce/jar")
+    @Produces({MediaType.APPLICATION_JSON})
+    public EnqueueBean mapReduceJar(@FormParam("jar") String jar,
+                                    @FormParam("class") String mainClass,
+                                    @FormParam("libjars") String libjars,
+                                    @FormParam("files") String files,
+                                    @FormParam("arg") List<String> args,
+                                    @FormParam("define") List<String> defines,
+                                    @FormParam("statusdir") String statusdir,
+                                    @FormParam("callback") String callback)
+        throws NotAuthorizedException, BusyException, BadParam, QueueException,
+        ExecuteException, IOException, InterruptedException
+    {
+        verifyUser();
+        verifyParam(jar, "jar");
+        verifyParam(mainClass, "class");
+
+        JarDelegator d = new JarDelegator(appConf);
+        return d.run(getUser(),
+                     jar, mainClass,
+                     libjars, files, args, defines,
+                     statusdir, callback, getCompletedUrl());
+    }
+
+    /**
+     * Run a Pig job.
+     */
+    @POST
+    @Path("pig")
+    @Produces({MediaType.APPLICATION_JSON})
+    public EnqueueBean pig(@FormParam("execute") String execute,
+                           @FormParam("file") String srcFile,
+                           @FormParam("arg") List<String> pigArgs,
+                           @FormParam("files") String otherFiles,
+                           @FormParam("statusdir") String statusdir,
+                           @FormParam("callback") String callback)
+        throws NotAuthorizedException, BusyException, BadParam, QueueException,
+        ExecuteException, IOException, InterruptedException
+    {
+        verifyUser();
+        if (execute == null && srcFile == null)
+            throw new BadParam("Either execute or file parameter required");
+
+        PigDelegator d = new PigDelegator(appConf);
+        return d.run(getUser(),
+                     execute, srcFile,
+                     pigArgs, otherFiles,
+                     statusdir, callback, getCompletedUrl());
+    }
+
+    /**
+     * Run a Hive job.
+     */
+    @POST
+    @Path("hive")
+    @Produces({MediaType.APPLICATION_JSON})
+    public EnqueueBean hive(@FormParam("execute") String execute,
+                            @FormParam("file") String srcFile,
+                            @FormParam("define") List<String> defines,
+                            @FormParam("statusdir") String statusdir,
+                            @FormParam("callback") String callback)
+        throws NotAuthorizedException, BusyException, BadParam, QueueException,
+        ExecuteException, IOException, InterruptedException
+    {
+        verifyUser();
+        if (execute == null && srcFile == null)
+            throw new BadParam("Either execute or file parameter required");
+
+        HiveDelegator d = new HiveDelegator(appConf);
+        return d.run(getUser(), execute, srcFile, defines,
+                     statusdir, callback, getCompletedUrl());
+    }
+
+    /**
+     * Return the status of the jobid.
+     */
+    @GET
+    @Path("queue/{jobid}")
+    @Produces({MediaType.APPLICATION_JSON})
+    public QueueStatusBean showQueueId(@PathParam("jobid") String jobid)
+        throws NotAuthorizedException, BadParam, IOException
+    {
+        verifyUser();
+        verifyParam(jobid, ":jobid");
+
+        StatusDelegator d = new StatusDelegator(appConf);
+        return d.run(getUser(), jobid);
+    }
+
+    /**
+     * Kill a job in the queue.
+     */
+    @DELETE
+    @Path("queue/{jobid}")
+    @Produces({MediaType.APPLICATION_JSON})
+    public QueueStatusBean deleteQueueId(@PathParam("jobid") String jobid)
+        throws NotAuthorizedException, BadParam, IOException
+    {
+        verifyUser();
+        verifyParam(jobid, ":jobid");
+
+        DeleteDelegator d = new DeleteDelegator(appConf);
+        return d.run(getUser(), jobid);
+    }
+
+    /**
+     * Return all the known job ids for this user.
+     */
+    @GET
+    @Path("queue")
+    @Produces({MediaType.APPLICATION_JSON})
+    public List<String> showQueueList()
+        throws NotAuthorizedException, BadParam, IOException
+    {
+        verifyUser();
+
+        ListDelegator d = new ListDelegator(appConf);
+        return d.run(getUser());
+    }
+
+    /**
+     * Notify on a completed job.
+     */
+    @GET
+    @Path("internal/complete/{jobid}")
+    @Produces({MediaType.APPLICATION_JSON})
+    public CompleteBean completeJob(@PathParam("jobid") String jobid)
+        throws CallbackFailedException, IOException
+    {
+        CompleteDelegator d = new CompleteDelegator(appConf);
+        return d.run(jobid);
+    }
+
+    /**
+     * Verify that we have a valid user.  Throw an exception if invalid.
+     */
+    public void verifyUser()
+        throws NotAuthorizedException
+    {
+        if (getUser() == null) {
+            String msg = "No user found.";
+            if (! UserGroupInformation.isSecurityEnabled())
+                msg += "  Missing " + PseudoAuthenticator.USER_NAME + " parameter.";
+            throw new NotAuthorizedException(msg);
+        }
+    }
+
+    /**
+     * Verify that the parameter exists.  Throw an exception if invalid.
+     */
+    public void verifyParam(String param, String name)
+        throws BadParam
+    {
+        if (param == null)
+            throw new BadParam("Missing " + name + " parameter");
+    }
+
+    /**
+     * Verify that the parameter exists.  Throw an exception if invalid.
+     */
+    public void verifyParam(List<String> param, String name)
+        throws BadParam
+    {
+        if (param == null || param.isEmpty())
+            throw new BadParam("Missing " + name + " parameter");
+    }
+
+    public static final Pattern DDL_ID = Pattern.compile("[a-zA-Z]\\w*");
+
+    /**
+     * Verify that the parameter exists and is a simple DDL identifier
+     * name.  Throw an exception if invalid.
+     *
+     * Bug: This needs to allow for quoted ddl identifiers.
+     */
+    public void verifyDdlParam(String param, String name)
+        throws BadParam
+    {
+        verifyParam(param, name);
+        Matcher m = DDL_ID.matcher(param);
+        if (! m.matches())
+            throw new BadParam("Invalid DDL identifier " + name );
+    }
+
+    /**
+     * Get the user name from the security context.
+     */
+    public String getUser() {
+        if (theSecurityContext == null)
+            return null;
+        if (theSecurityContext.getUserPrincipal() == null)
+            return null;
+        return theSecurityContext.getUserPrincipal().getName();
+    }
+
+    /**
+     * The callback url on this server when a task is completed.
+     */
+    public String getCompletedUrl() {
+        if (theUriInfo == null)
+            return null;
+        if (theUriInfo.getBaseUri() == null)
+            return null;
+        return theUriInfo.getBaseUri() + VERSION
+            + "/internal/complete/$jobId";
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleExceptionMapper.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleExceptionMapper.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleExceptionMapper.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleExceptionMapper.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+
+/**
+ * Map our exceptions to the Jersey response.  This lets us have nice
+ * results in the error body.
+ */
+@Provider
+public class SimpleExceptionMapper
+    implements ExceptionMapper<SimpleWebException>
+{
+    public Response toResponse(SimpleWebException e) {
+        return e.getResponse();
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleWebException.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleWebException.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleWebException.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/SimpleWebException.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.HashMap;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response.ResponseBuilder;
+import javax.ws.rs.core.Response;
+import org.codehaus.jackson.map.ObjectMapper;
+
+/**
+ * Simple exception that will return a json error payload if thrown
+ * from a JAX web server.  We skip using WebApplicationException and
+ * instead map our own so that Jersey doesn't log our exceptions as
+ * error in the output log.  See SimpleExceptionMapper.
+ */
+public class SimpleWebException extends Throwable {
+    public int httpCode;
+    public Map<String, Object> params;
+
+    public SimpleWebException(int httpCode, String msg) {
+        super(msg);
+        this.httpCode = httpCode;
+    }
+
+    public SimpleWebException(int httpCode, String msg, Map<String, Object> params) {
+        super(msg);
+        this.httpCode = httpCode;
+        this.params = params;
+    }
+
+    public Response getResponse() {
+        return buildMessage(httpCode, params, getMessage());
+    }
+
+    public static Response buildMessage(int httpCode, Map<String, Object> params,
+                                        String msg)
+    {
+        HashMap<String,Object> err = new HashMap<String,Object>();
+        err.put("error", msg);
+        if (params != null)
+            err.putAll(params);
+
+        String json = "\"error\"";
+        try {
+            json = new ObjectMapper().writeValueAsString(err);
+        } catch (IOException e) {
+        }
+
+        return Response.status(httpCode)
+            .entity(json)
+            .type(MediaType.APPLICATION_JSON)
+            .build();
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapred.JobProfile;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.JobTracker;
+import org.apache.hadoop.mapred.TempletonJobTracker;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hcatalog.templeton.tool.JobState;
+
+/**
+ * Fetch the status of a given job id in the queue.
+ */
+public class StatusDelegator extends TempletonDelegator {
+    private static final Log LOG = LogFactory.getLog(StatusDelegator.class);
+
+    public StatusDelegator(AppConfig appConf) {
+        super(appConf);
+    }
+
+    public QueueStatusBean run(String user, String id)
+        throws NotAuthorizedException, BadParam, IOException
+    {
+        UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+        TempletonJobTracker tracker = null;
+        JobState state = null;
+        try {
+            tracker = new TempletonJobTracker(ugi,
+                                              JobTracker.getAddress(appConf),
+                                              appConf);
+            JobID jobid = StatusDelegator.StringToJobID(id);
+            if (jobid == null)
+                throw new BadParam("Invalid jobid: " + id);
+            state = new JobState(id, Main.getAppConfigInstance());
+            return StatusDelegator.makeStatus(tracker, jobid, state);
+        } catch (IllegalStateException e) {
+            throw new BadParam(e.getMessage());
+        } finally {
+            if (tracker != null)
+                tracker.close();
+            if (state != null)
+                state.close();
+        }
+    }
+
+    public static QueueStatusBean makeStatus(TempletonJobTracker tracker,
+                                             JobID jobid,
+                                             String childid,
+                                             JobState state)
+        throws BadParam, IOException
+    {
+        JobID bestid = jobid;
+        if (childid != null)
+            bestid = StatusDelegator.StringToJobID(childid);
+
+        JobStatus status = tracker.getJobStatus(bestid);
+        JobProfile profile = tracker.getJobProfile(bestid);
+
+        if (status == null || profile == null) {
+            if (bestid != jobid) { // Corrupt childid, retry.
+                LOG.error("Corrupt child id " + childid + " for " + jobid);
+                bestid = jobid;
+                status = tracker.getJobStatus(bestid);
+                profile = tracker.getJobProfile(bestid);
+            }
+        }
+
+        if (status == null || profile == null) // No such job.
+            throw new BadParam("Could not find job " + bestid);
+
+        return new QueueStatusBean(state, status, profile);
+    }
+
+    public static QueueStatusBean makeStatus(TempletonJobTracker tracker,
+                                             JobID jobid,
+                                             JobState state)
+        throws BadParam, IOException
+    {
+        return makeStatus(tracker, jobid, state.getChildId(), state);
+    }
+
+    /**
+     * A version of JobID.forName with our app specific error handling.
+     */
+    public static JobID StringToJobID(String id)
+        throws BadParam
+    {
+        try {
+            return JobID.forName(id);
+        } catch (IllegalArgumentException e) {
+            throw new BadParam(e.getMessage());
+        }
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StreamingDelegator.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StreamingDelegator.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StreamingDelegator.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StreamingDelegator.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.exec.ExecuteException;
+
+/**
+ * Submit a streaming job to the MapReduce queue.  Really just a front
+   end to the JarDelegator.
+ *
+ * This is the backend of the mapreduce/streaming web service.
+ */
+public class StreamingDelegator extends LauncherDelegator {
+    public StreamingDelegator(AppConfig appConf) {
+        super(appConf);
+    }
+
+    public EnqueueBean run(String user,
+                           List<String> inputs, String output,
+                           String mapper, String reducer,
+                           List<String> files, List<String> defines,
+                           List<String> cmdenvs,
+                           List<String> jarArgs,
+                           String statusdir,
+                           String callback,
+                           String completedUrl)
+        throws NotAuthorizedException, BadParam, BusyException, QueueException,
+        ExecuteException, IOException, InterruptedException
+    {
+        List<String> args = makeArgs(inputs, output, mapper, reducer,
+                                     files, defines, cmdenvs, jarArgs);
+
+        JarDelegator d = new JarDelegator(appConf);
+        return d.run(user,
+                     appConf.streamingJar(), null,
+                     null, null, args, defines,
+                     statusdir, callback, completedUrl);
+    }
+
+    private List<String> makeArgs(List<String> inputs,
+                                  String output,
+                                  String mapper,
+                                  String reducer,
+                                  List<String> files,
+                                  List<String> defines,
+                                  List<String> cmdenvs,
+                                  List<String> jarArgs)
+    {
+        ArrayList<String> args = new ArrayList<String>();
+        for (String input : inputs) {
+            args.add("-input");
+            args.add(input);
+        }
+        args.add("-output");
+        args.add(output);
+        args.add("-mapper");
+        args.add(mapper);
+        args.add("-reducer");
+        args.add(reducer);
+
+        for (String f : files)
+            args.add("-file" + f);
+        for (String d : defines)
+            args.add("-D" + d);
+        for (String e : cmdenvs)
+            args.add("-cmdenv" + e);
+        args.addAll(jarArgs);
+
+        return args;
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableDesc.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableDesc.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableDesc.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableDesc.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.util.List;
+import java.util.Map;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * A description of the table to create.
+ */
+@XmlRootElement
+public class TableDesc extends GroupPermissionsDesc {
+    public boolean external = false;
+    public boolean ifNotExists = false;
+    public String table;
+    public String comment;
+    public List<ColumnDesc> columns;
+    public List<ColumnDesc> partitionedBy;
+    public ClusteredByDesc clusteredBy;
+    public StorageFormatDesc format;
+    public String location;
+    public Map<String, String> tableProperties;
+
+    /**
+     * Create a new TableDesc
+     */
+    public TableDesc() {}
+
+    public String toString() {
+        return String.format("TableDesc(table=%s, columns=%s)", table, columns);
+    }
+
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (! (o instanceof TableDesc))
+            return false;
+        TableDesc that = (TableDesc) o;
+        return xequals(this.external,        that.external)
+            && xequals(this.ifNotExists,     that.ifNotExists)
+            && xequals(this.table,           that.table)
+            && xequals(this.comment,         that.comment)
+            && xequals(this.columns,         that.columns)
+            && xequals(this.partitionedBy,   that.partitionedBy)
+            && xequals(this.clusteredBy,     that.clusteredBy)
+            && xequals(this.format,          that.format)
+            && xequals(this.location,        that.location)
+            && xequals(this.tableProperties, that.tableProperties)
+            && super.equals(that)
+            ;
+    }
+
+    /**
+     * How to cluster the table.
+     */
+    @XmlRootElement
+    public static class ClusteredByDesc {
+        public List<String> columnNames;
+        public List<ClusterSortOrderDesc> sortedBy;
+        public int numberOfBuckets;
+
+        public ClusteredByDesc() {}
+
+        public String toString() {
+            String fmt
+                = "ClusteredByDesc(columnNames=%s, sortedBy=%s, numberOfBuckets=%s)";
+            return String.format(fmt, columnNames, sortedBy, numberOfBuckets);
+        }
+
+        public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (! (o instanceof ClusteredByDesc))
+                return false;
+            ClusteredByDesc that = (ClusteredByDesc) o;
+            return xequals(this.columnNames,     that.columnNames)
+                && xequals(this.sortedBy,        that.sortedBy)
+                && xequals(this.numberOfBuckets, that.numberOfBuckets)
+                ;
+        }
+    }
+
+    /**
+     * The clustered sort order.
+     */
+    @XmlRootElement
+    public static class ClusterSortOrderDesc {
+        public String columnName;
+        public SortDirectionDesc order;
+
+        public ClusterSortOrderDesc() {}
+
+        public ClusterSortOrderDesc(String columnName, SortDirectionDesc order) {
+            this.columnName = columnName;
+            this.order = order;
+        }
+
+        public String toString() {
+            return String
+                .format("ClusterSortOrderDesc(columnName=%s, order=%s)",
+                        columnName, order);
+        }
+
+        public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (! (o instanceof ClusterSortOrderDesc))
+                return false;
+            ClusterSortOrderDesc that = (ClusterSortOrderDesc) o;
+            return xequals(this.columnName,     that.columnName)
+                && xequals(this.order,          that.order)
+                ;
+        }
+    }
+
+    /**
+     * Ther ASC or DESC sort order.
+     */
+    @XmlRootElement
+    public static enum SortDirectionDesc {
+        ASC, DESC
+    }
+
+    /**
+     * The storage format.
+     */
+    @XmlRootElement
+    public static class StorageFormatDesc {
+        public RowFormatDesc rowFormat;
+        public String storedAs;
+        public StoredByDesc storedBy;
+
+        public StorageFormatDesc() {}
+
+        public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (! (o instanceof StorageFormatDesc))
+                return false;
+            StorageFormatDesc that = (StorageFormatDesc) o;
+            return xequals(this.rowFormat,      that.rowFormat)
+                && xequals(this.storedAs,       that.storedAs)
+                && xequals(this.storedBy,       that.storedBy)
+                ;
+        }
+    }
+
+    /**
+     * The Row Format.
+     */
+    @XmlRootElement
+    public static class RowFormatDesc {
+        public String fieldsTerminatedBy;
+        public String collectionItemsTerminatedBy;
+        public String mapKeysTerminatedBy;
+        public String linesTerminatedBy;
+        public SerdeDesc serde;
+
+        public RowFormatDesc() {}
+
+        public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (! (o instanceof RowFormatDesc))
+                return false;
+            RowFormatDesc that = (RowFormatDesc) o;
+            return xequals(this.fieldsTerminatedBy,     that.fieldsTerminatedBy)
+                && xequals(this.collectionItemsTerminatedBy,
+                           that.collectionItemsTerminatedBy)
+                && xequals(this.mapKeysTerminatedBy,    that.mapKeysTerminatedBy)
+                && xequals(this.linesTerminatedBy,      that.linesTerminatedBy)
+                && xequals(this.serde,                  that.serde)
+                ;
+        }
+    }
+
+    /**
+     * The SERDE Row Format.
+     */
+    @XmlRootElement
+    public static class SerdeDesc {
+        public String name;
+        public Map<String, String> properties;
+
+        public SerdeDesc() {}
+
+        public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (! (o instanceof SerdeDesc))
+                return false;
+            SerdeDesc that = (SerdeDesc) o;
+            return xequals(this.name,           that.name)
+                && xequals(this.properties,     that.properties)
+                ;
+        }
+    }
+
+    /**
+     * How to store the table.
+     */
+    @XmlRootElement
+    public static class StoredByDesc {
+        public String className;
+        public Map<String, String> properties;
+
+        public StoredByDesc() {}
+
+        public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (! (o instanceof StoredByDesc))
+                return false;
+            StoredByDesc that = (StoredByDesc) o;
+            return xequals(this.className,      that.className)
+                && xequals(this.properties,     that.properties)
+                ;
+        }
+    }
+
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableLikeDesc.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableLikeDesc.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableLikeDesc.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TableLikeDesc.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * A description of the table to create that's like another table.
+ */
+@XmlRootElement
+public class TableLikeDesc extends GroupPermissionsDesc {
+    public boolean external = false;
+    public boolean ifNotExists = false;
+    public String location;
+    public String existingTable;
+    public String newTable;
+
+    public TableLikeDesc() {}
+
+    public String toString() {
+        return String.format("TableLikeDesc(existingTable=%s, newTable=%s, location=%s",
+                             existingTable, newTable, location);
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TablePropertyDesc.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TablePropertyDesc.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TablePropertyDesc.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TablePropertyDesc.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * A description of a table property.
+ */
+@XmlRootElement
+public class TablePropertyDesc extends GroupPermissionsDesc {
+    public String name;
+    public String value;
+
+    public TablePropertyDesc() {}
+
+    public String toString() {
+        return String.format("TablePropertyDesc(name=%s, value=%s)",
+                             name, value);
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+/**
+ * The helper class for all the Templeton delegator classes. A
+ * delegator will call the underlying Templeton service such as hcat
+ * or hive.
+ */
+public class TempletonDelegator {
+    protected AppConfig appConf;
+
+    public TempletonDelegator(AppConfig appConf) {
+        this.appConf = appConf;
+    }
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/UgiFactory.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.security.UserGroupInformation;
+
+public class UgiFactory {
+    private static ConcurrentHashMap<String, UserGroupInformation> userUgiMap =
+            new ConcurrentHashMap<String, UserGroupInformation>();
+    
+    static UserGroupInformation getUgi(String user) throws IOException{
+        UserGroupInformation ugi = userUgiMap.get(user);
+        if(ugi == null){
+            //create new ugi and add to map
+            final UserGroupInformation newUgi = 
+                    UserGroupInformation.createProxyUser(user,
+                            UserGroupInformation.getLoginUser());
+
+            //if another thread adds an entry before the check in this one
+            // the one created here will not be added.
+            userUgiMap.putIfAbsent(user, newUgi);
+
+            //use the UGI object that got added
+            return userUgiMap.get(user);
+            
+        }
+        return ugi;
+    }
+    
+    
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/WadlConfig.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/WadlConfig.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/WadlConfig.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/WadlConfig.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton;
+
+import java.util.List;
+
+import com.sun.jersey.api.wadl.config.WadlGeneratorConfig;
+import com.sun.jersey.api.wadl.config.WadlGeneratorDescription;
+import com.sun.jersey.server.wadl.generators.resourcedoc.WadlGeneratorResourceDocSupport;
+
+/**
+ * Simple class that incorporates javadoc information into the
+ * wadl produced by jersey.
+ * 
+ */
+public class WadlConfig extends WadlGeneratorConfig {
+    
+    @Override
+    public List<WadlGeneratorDescription> configure() {
+        return generator( WadlGeneratorResourceDocSupport.class ) 
+            .prop( "resourceDocStream", "resourcedoc.xml" ) 
+        .descriptions();
+    }
+ 
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSCleanup.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSCleanup.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSCleanup.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSCleanup.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton.tool;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hcatalog.templeton.tool.TempletonStorage.Type;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * This does periodic cleanup
+ */
+public class HDFSCleanup extends Thread {
+    protected Configuration appConf;
+
+    // The interval to wake up and check the queue
+    public static final String HDFS_CLEANUP_INTERVAL =
+        "templeton.hdfs.cleanup.interval"; // 12 hours
+
+    // The max age of a task allowed
+    public static final String HDFS_CLEANUP_MAX_AGE =
+        "templeton.hdfs.cleanup.maxage"; // ~ 1 week
+
+    protected static long interval = 1000L * 60L * 60L * 12L;
+    protected static long maxage = 1000L * 60L * 60L * 24L * 7L;
+
+    // The logger
+    private static final Log LOG = LogFactory.getLog(HDFSCleanup.class);
+
+    // Handle to cancel loop
+    private boolean stop = false;
+
+    // The instance
+    private static HDFSCleanup thisclass = null;
+
+    // Whether the cycle is running
+    private static boolean isRunning = false;
+    
+    // The storage root
+    private String storage_root;
+    
+    /**
+     * Create a cleanup object. 
+     */
+    private HDFSCleanup(Configuration appConf) {
+        this.appConf = appConf;
+        interval = appConf.getLong(HDFS_CLEANUP_INTERVAL, interval);
+        maxage = appConf.getLong(HDFS_CLEANUP_MAX_AGE, maxage);
+        storage_root = appConf.get(TempletonStorage.STORAGE_ROOT);
+    }
+    
+    public static HDFSCleanup getInstance(Configuration appConf) {
+        if (thisclass != null) {
+            return thisclass;
+        }
+        thisclass = new HDFSCleanup(appConf);
+        return thisclass;
+    }
+
+    public static void startInstance(Configuration appConf) throws IOException {
+        if (!isRunning) {
+            getInstance(appConf).start();
+        }
+    }
+
+    /**
+     * Run the cleanup loop.
+     *
+     */
+    public void run() {
+        FileSystem fs = null;
+        while (!stop) {
+            try {
+                // Put each check in a separate try/catch, so if that particular
+                // cycle fails, it'll try again on the next cycle.
+                try {
+                    if (fs == null) {
+                        fs = FileSystem.get(appConf);
+                    }
+                    checkFiles(fs);
+                } catch (Exception e) {
+                    LOG.error("Cleanup cycle failed: " + e.getMessage());
+                }
+
+                long sleepMillis = (long) (Math.random() * interval);
+                LOG.info("Next execution: " + new Date(new Date().getTime()
+                                                       + sleepMillis));
+                Thread.sleep(sleepMillis);
+
+            } catch (Exception e) {
+                // If sleep fails, we should exit now before things get worse.
+                isRunning = false;
+                LOG.error("Cleanup failed: " + e.getMessage(), e);
+            }
+        }
+        isRunning = false;
+    }
+    
+    /**
+     * Loop through all the files, deleting any that are older than
+     * maxage.
+     * 
+     * @param fs
+     * @throws IOException
+     */
+    private void checkFiles(FileSystem fs) throws IOException {
+        long now = new Date().getTime();
+        for (Type type : Type.values()) {
+            try {
+                for (FileStatus status : fs.listStatus(new Path(
+                        HDFSStorage.getPath(type, storage_root)))) {
+                    if (now - status.getModificationTime() > maxage) {
+                        LOG.info("Deleting " + status.getPath().toString());
+                        fs.delete(status.getPath(), true);
+                    }
+                }
+            } catch (Exception e) {
+                // Nothing to find for this type.
+            }
+        }
+    }
+
+    // Handle to stop this process from the outside if needed.
+    public void exit() {
+        stop = true;
+    }
+
+}

Added: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java (added)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/HDFSStorage.java Wed Jul 25 20:29:44 2012
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hcatalog.templeton.tool;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ *  HDFS implementation of templeton storage.
+ *
+ *  This implementation assumes that all keys in key/value pairs are
+ *  chosen such that they don't have any newlines in them.
+ *
+ */
+public class HDFSStorage implements TempletonStorage {
+    FileSystem fs = null;
+
+    public String storage_root = null;
+
+    public static final String JOB_PATH = "/jobs";
+    public static final String JOB_TRACKINGPATH = "/created";
+    public static final String OVERHEAD_PATH = "/overhead";
+
+    private static final Log LOG = LogFactory.getLog(HDFSStorage.class);
+
+    public void startCleanup(Configuration config) {
+        try {
+            HDFSCleanup.startInstance(config);
+        } catch (Exception e) {
+            LOG.warn("Cleanup instance didn't start.");
+        }
+    }
+
+    @Override
+    public void saveField(Type type, String id, String key, String val)
+        throws NotFoundException {
+        if (val == null) {
+            return;
+        }
+        PrintWriter out = null;
+        try {
+            Path keyfile = new Path(getPath(type) + "/" + id + "/" + key);
+            // This will replace the old value if there is one
+            // Overwrite the existing file
+            out = new PrintWriter(new OutputStreamWriter(fs.create(keyfile)));
+            out.write(val);
+        } catch (IOException e) {
+            LOG.info("Couldn't write to " + getPath(type) + "/" + id + ": "
+                     + e.getMessage());
+        } finally {
+            try {
+                out.flush();
+                out.close();
+            } catch (Exception e) {
+                // fail
+            }
+        }
+    }
+
+    @Override
+    public String getField(Type type, String id, String key) {
+        BufferedReader in = null;
+        try {
+            in = new BufferedReader(new InputStreamReader
+                                    (fs.open(new Path(getPath(type) + "/" +
+                                                      id + "/" + key))));
+            String line = null;
+            String val = "";
+            while ((line = in.readLine()) != null) {
+                if (! val.equals("")) {
+                    val += "\n";
+                }
+                val += line;
+            }
+            return val;
+        } catch (IOException e) {
+            LOG.trace("Couldn't find " + getPath(type) + "/" + id + "/" + key
+                      + ": " + e.getMessage());
+        } finally {
+            try {
+                in.close();
+            } catch (Exception e) {
+                // fail
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Map<String, String> getFields(Type type, String id) {
+        HashMap<String, String> map = new HashMap<String, String>();
+        BufferedReader in = null;
+        try {
+            for (FileStatus status : fs.listStatus
+                     (new Path(getPath(type) + "/" + id))) {
+                in = new BufferedReader(new InputStreamReader
+                                        (fs.open(status.getPath())));
+                String line = null;
+                String val = "";
+                while ((line = in.readLine()) != null) {
+                    if (!val.equals("")) {
+                        val += "\n";
+                    }
+                    val += line;
+                }
+                map.put(status.getPath().getName(), val);
+            }
+        } catch (IOException e) {
+            LOG.trace("Couldn't find " + getPath(type) + "/" + id);
+        } finally {
+            try {
+                in.close();
+            } catch (Exception e) {
+                // fail
+            }
+        }
+        return map;
+    }
+
+    @Override
+    public boolean delete(Type type, String id) throws NotFoundException {
+        try {
+            fs.delete(new Path(getPath(type) + "/" + id), true);
+        } catch (IOException e) {
+            throw new NotFoundException("Node " + id + " was not found: " +
+                                        e.getMessage());
+        }
+        return false;
+    }
+
+    @Override
+    public List<String> getAll() {
+        ArrayList<String> allNodes = new ArrayList<String>();
+        for (Type type: Type.values()) {
+            allNodes.addAll(getAllForType(type));
+        }
+        return allNodes;
+    }
+
+    @Override
+    public List<String> getAllForType(Type type) {
+        ArrayList<String> allNodes = new ArrayList<String>();
+        try {
+            for (FileStatus status : fs.listStatus(new Path(getPath(type)))) {
+                allNodes.add(status.getPath().getName());
+            }
+            return null;
+        } catch (Exception e) {
+            LOG.trace("Couldn't find children for type " + type.toString());
+        }
+        return allNodes;
+    }
+
+    @Override
+    public List<String> getAllForKey(String key, String value) {
+        ArrayList<String> allNodes = new ArrayList<String>();
+        try {
+            for (Type type : Type.values()) {
+                allNodes.addAll(getAllForTypeAndKey(type, key, value));
+            }
+        } catch (Exception e) {
+            LOG.trace("Couldn't find children for key " + key + ": " +
+                      e.getMessage());
+        }
+        return allNodes;
+    }
+
+    @Override
+    public List<String> getAllForTypeAndKey(Type type, String key, String value) {
+        ArrayList<String> allNodes = new ArrayList<String>();
+        HashMap<String, String> map = new HashMap<String, String>();
+        try {
+            for (FileStatus status :
+                     fs.listStatus(new Path(getPath(type)))) {
+                map = (HashMap<String, String>)
+                    getFields(type, status.getPath().getName());
+                if (map.get(key).equals(value)) {
+                    allNodes.add(status.getPath().getName());
+                }
+            }
+        } catch (Exception e) {
+            LOG.trace("Couldn't find children for key " + key + ": " +
+                      e.getMessage());
+        }
+        return allNodes;
+    }
+
+    @Override
+    public void openStorage(Configuration config) throws IOException {
+        storage_root = config.get(TempletonStorage.STORAGE_ROOT);
+        if (fs == null) {
+            fs = FileSystem.get(config);
+        }
+    }
+
+    @Override
+    public void closeStorage() throws IOException {
+        // Nothing to do here
+    }
+
+    /**
+     * Get the path to storage based on the type.
+     * @param type
+     */
+    public String getPath(Type type) {
+        return getPath(type, storage_root);
+    }
+
+    /**
+     * Static method to get the path based on the type.
+     *
+     * @param type
+     * @param root
+     */
+    public static String getPath(Type type, String root) {
+        String typepath = root + OVERHEAD_PATH;
+        switch (type) {
+        case JOB:
+            typepath = root + JOB_PATH;
+            break;
+        case JOBTRACKING:
+            typepath = root + JOB_TRACKINGPATH;
+            break;
+        }
+        return typepath;
+    }
+}



Mime
View raw message