incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1365722 [3/11] - in /incubator/hcatalog/trunk: ./ ant/ conf/ hcatalog-pig-adapter/ ivy/ src/docs/src/documentation/content/xdocs/ src/docs/src/documentation/content/xdocs/images/ src/java/org/apache/hcatalog/mapreduce/ src/test/e2e/templet...
Date Wed, 25 Jul 2012 20:29:49 GMT
Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/queuelist.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/queuelist.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/queuelist.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/queuelist.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+  <header>
+    <title>GET queue </title>
+  </header>
+
+  <body>
+  <section>
+   <title>Description</title>
+    <p>Return a list of all job IDs registered to the user.</p>
+  </section>
+
+  <section>
+   <title>URL</title>
+    <p><code>http://</code>www.myserver.com<code>/templeton/v1/queue</code></p>
+  </section>
+
+  <section>
+   <title>Parameters</title>
+    <p>Only the <a href="index.html#Standard+Parameters">standard parameters</a>
+       are accepted.</p>
+  </section>
+
+  <section>
+   <title>Results</title>
+    <table>
+    <tr><th>Name</th><th>Description</th></tr>
+    <tr><td><strong>ids</strong></td>
+        <td>A list of all job IDs registered to the user.</td>
+    </tr>
+    </table>
+ </section>
+
+  <section>
+   <title>Example</title>
+
+   <p><strong>Curl Command</strong></p>
+<source>
+% curl -s 'http://localhost:50111/templeton/v1/queue?user.name=ctdean'
+</source>
+
+   <p><strong>JSON Output</strong></p>
+<source>
+{
+ "job_201111111311_0008",
+ "job_201111111311_0012"
+}
+</source>
+  </section>
+ </body>
+</document>

Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/renametable.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/renametable.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/renametable.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/renametable.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,120 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+  <header>
+    <title>POST ddl/database/:db/table/:table</title>
+  </header>
+
+  <body>
+  <section>
+   <title>Description</title>
+    <p>Rename an HCatalog table.</p>
+  </section>
+
+  <section>
+   <title>URL</title>
+    <p><code>http://</code>www.myserver.com<code>/templeton/v1/ddl/database/</code>:db<code>/table/</code>:table</p>
+  </section>
+
+  <section>
+   <title>Parameters</title>
+    <table>
+    <tr><th>Name</th><th>Description</th><th>Required?</th><th>Default</th></tr>
+
+    <tr><td><strong>:db</strong></td>
+        <td>The database name</td>
+        <td>Required</td>
+        <td>None</td>
+    </tr>
+
+    <tr><td><strong>:table</strong></td>
+        <td>The existing (old) table name</td>
+        <td>Required</td>
+        <td>None</td>
+    </tr>
+
+    <tr><td><strong>rename</strong></td>
+        <td>The new table name</td>
+        <td>Required</td>
+        <td>None</td>
+    </tr>
+
+    <tr><td><strong>group</strong></td>
+        <td>The user group to use</td>
+        <td>Optional</td>
+        <td>None</td>
+    </tr>
+
+    <tr><td><strong>permissions</strong></td>
+        <td>The permissions string to use.  The format is
+          "<code>rwxrw-r-x</code>".</td>
+        <td>Optional</td>
+        <td>None</td>
+    </tr>
+
+    </table>
+  </section>
+
+  <section>
+   <title>Results</title>
+    <table>
+    <tr><th>Name</th><th>Description</th></tr>
+
+    <tr><td><strong>table</strong></td>
+        <td>The new table name</td>
+    </tr>
+
+    <tr><td><strong>database</strong></td>
+        <td>The database name</td>
+    </tr>
+
+    </table>
+ </section>
+
+  <section>
+   <title>Example</title>
+
+   <p><strong>Curl Command</strong></p>
+<source>
+% curl -s -d user.name=ctdean \
+       -d rename=test_table_2 \
+       'http://localhost:50111/templeton/v1/ddl/database/default/table/test_table'
+</source>
+
+   <p><strong>JSON Output</strong></p>
+<source>
+{
+ "table": "test_table_2",
+ "database": "default"
+}
+</source>
+
+   <p><strong>JSON Output (error)</strong></p>
+<source>
+{
+  "error": "Table test_table does not exist",
+  "errorCode": 404,
+  "database": "default",
+  "table": "test_table_2"
+}
+</source>
+  </section>
+ </body>
+</document>

Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/resources.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/resources.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/resources.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/resources.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,123 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+  <header>
+    <title>Templeton Resources </title>
+  </header>
+  <body>
+     <table>
+     <tr><th>Resource</th><th>Description</th></tr>
+
+     <tr><td><a href="responsetypes.html">:version</a></td>
+         <td>Returns a list of supported response types.</td></tr>
+
+     <tr><td><a href="status.html">status</a></td>
+         <td>Returns the Templeton server status.</td></tr>
+
+    <tr><td><a href="versions.html">version</a></td>
+         <td>Returns the a list of supported versions and the current version.</td></tr>
+
+     <tr><td><a href="ddl.html">ddl</a></td>
+         <td>Performs an HCatalog DDL command.</td></tr>
+
+     <tr><td><a href="listdbs.html">ddl/database</a></td>
+         <td>List HCatalog databases.</td></tr>
+
+     <tr><td><a href="descdb.html">ddl/database/:db (GET)</a></td>
+         <td>Describe an HCatalog database.</td></tr>
+
+     <tr><td><a href="createdb.html">ddl/database/:db (PUT)</a></td>
+         <td>Create an HCatalog database.</td></tr>
+
+     <tr><td><a href="deletedb.html">ddl/database/:db (DELETE)</a></td>
+         <td>Delete (drop) an HCatalog database.</td></tr>
+
+     <tr><td><a href="listtables.html">ddl/database/:db/table</a></td>
+         <td>List the tables in an HCatalog database.</td></tr>
+
+     <tr><td><a href="desctable.html">ddl/database/:db/table/:table (GET)</a></td>
+         <td>Describe an HCatalog table.</td></tr>
+
+     <tr><td><a href="createtable.html">ddl/database/:db/table/:table (PUT)</a></td>
+         <td>Create a new HCatalog table.</td></tr>
+
+     <tr><td><a href="renametable.html">ddl/database/:db/table/:table (POST)</a></td>
+         <td>Rename an HCatalog table.</td></tr>
+
+     <tr><td><a href="deletetable.html">ddl/database/:db/table/:table (DELETE)</a></td>
+         <td>Delete (drop) an HCatalog table.</td></tr>
+
+     <tr><td><a href="createtablelike.html">ddl/database/:db/table/:existingtable/like/:newtable (PUT)</a></td>
+         <td>Create a new HCatalog table like an existing one.</td></tr>
+
+     <tr><td><a href="listpartitions.html">ddl/database/:db/table/:table/partion</a></td>
+         <td>List all partitions in an HCatalog table.</td></tr>
+
+     <tr><td><a href="descpartition.html">ddl/database/:db/table/:table/partion/:partition (GET)</a></td>
+         <td>Describe a single partition in an HCatalog table.</td></tr>
+
+     <tr><td><a href="createpartition.html">ddl/database/:db/table/:table/partion/:partition (PUT)</a></td>
+         <td>Create a partition in an HCatalog table.</td></tr>
+
+     <tr><td><a href="deletepartition.html">ddl/database/:db/table/:table/partion/:partition (DELETE)</a></td>
+         <td>Delete (drop) a partition in an HCatalog table.</td></tr>
+
+     <tr><td><a href="listcolumns.html">ddl/database/:db/table/:table/column</a></td>
+         <td>List the columns in an HCatalog table.</td></tr>
+
+     <tr><td><a href="desccolumn.html">ddl/database/:db/table/:table/column/:column (GET)</a></td>
+         <td>Describe a single column in an HCatalog table.</td></tr>
+
+     <tr><td><a href="createcolumn.html">ddl/database/:db/table/:table/column/:column (PUT)</a></td>
+         <td>Create a column in an HCatalog table.</td></tr>
+
+     <tr><td><a href="listproperties.html">ddl/database/:db/table/:table/property (GET)</a></td>
+         <td>List table properties.</td></tr>
+
+     <tr><td><a href="descproperty.html">ddl/database/:db/table/:table/property/:property (GET)</a></td>
+         <td>Return the value of a single table property.</td></tr>
+
+     <tr><td><a href="addproperty.html">ddl/database/:db/table/:table/property/:property (PUT)</a></td>
+         <td>Set a table property.</td></tr>
+
+     <tr><td><a href="mapreducestreaming.html">mapreduce/streaming</a></td>
+         <td>Creates and queues Hadoop streaming MapReduce jobs.</td></tr>
+
+     <tr><td><a href="mapreducejar.html">mapreduce/jar</a></td>
+         <td>Creates and queues standard Hadoop MapReduce jobs.</td></tr>
+
+     <tr><td><a href="pig.html">pig</a></td>
+         <td>Creates and queues Pig jobs.</td></tr>
+
+     <tr><td><a href="hive.html">hive</a></td>
+         <td>Runs Hive queries and commands.</td></tr>
+
+     <tr><td><a href="queuelist.html">queue</a></td>
+         <td>Returns a list of all jobids registered for the user.</td></tr>
+
+     <tr><td><a href="queue.html">queue/:jobid (GET)</a></td>
+         <td>Returns the status of a job given its ID.</td></tr>
+
+     <tr><td><a href="queuedelete.html">queue/:jobid (DELETE)</a></td>
+         <td>Kill a job given its ID.</td></tr>
+
+     </table>
+  </body>
+</document>

Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/responsetypes.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/responsetypes.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/responsetypes.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/responsetypes.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+  <header>
+    <title>GET :version </title>
+  </header>
+
+  <body>
+  <section>
+   <title>Description</title>
+    <p>Returns a list of the response types supported by Templeton.</p>
+  </section>
+
+  <section>
+   <title>URL</title>
+    <p><code>http://</code>www.myserver.com<code>/templeton/</code>:version</p>
+  </section>
+
+  <section>
+   <title>Parameters</title>
+    <table>
+    <tr><th>Name</th><th>Description</th><th>Required?</th><th>Default</th></tr>
+
+    <tr><td><strong>:version</strong></td>
+        <td>The Templeton version number.  (Currently this must be "v1")</td>
+        <td>Required</td>
+        <td>None</td>
+    </tr>
+    </table>
+  </section>
+
+  <section>
+   <title>Results</title>
+    <table>
+    <tr><th>Name</th><th>Description</th></tr>
+
+    <tr><td><strong>responseTypes</strong></td>
+        <td>A list of all supported response types</td>
+    </tr>
+
+    </table>
+  </section>
+
+  <section>
+   <title>Example</title>
+
+   <p><strong>Curl Command</strong></p>
+<source>
+% curl -s 'http://localhost:50111/templeton/v1'
+</source>
+
+   <p><strong>JSON Output</strong></p>
+<source>
+{
+  "responseTypes": [
+    "application/json"
+  ]
+}
+</source>
+
+   <p><strong>JSON Output (error)</strong></p>
+<source>
+{
+  "error": "null for uri: http://localhost:50111/templeton/v2"
+}
+</source>
+  </section>
+ </body>
+</document>

Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,140 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+  <header>
+    <title>HCatalog REST API </title>
+  </header>
+  <body>
+
+   <section>
+      <title>Introduction </title>
+       <p>This document describes HCatalog REST API.
+        As shown in the figure below, developers make HTTP requests to access
+        <a href="http://hadoop.apache.org/mapreduce/">Hadoop MapReduce</a>,
+        <a href="http://pig.apache.org/">Pig</a>,
+        <a href="http://hive.apache.org/">Hive</a>, and
+        <a href="http://incubator.apache.org/hcatalog/docs/r0.2.0/cli.html">
+        HCatalog DDL</a> from within applications.
+        Data and code used by this API is maintained in
+        <a href="http://hadoop.apache.org/hdfs/">HDFS</a>.  HCatalog DDL commands
+        are executed directly when requested.
+        MapReduce, Pig, and Hive jobs are placed in queue by
+        and can be monitored for progress or stopped as required.
+        Developers specify a location
+        in HDFS into which Pig, Hive, and MapReduce results should be placed.</p>
+       <figure src="images/TempletonArch.jpg" align="left" alt="Templeton Architecture"/>
+   </section>
+
+    <section>
+       <title>URL format </title>
+       <p>HCatalog's REST resources are accessed using the following URL format:</p>
+       <p><code>http://</code><em>yourserver</em><code>/templeton/v1/</code><em>resource</em></p>
+       <p>where "<em>yourserver</em>" is replaced with your server name, and
+          "<em>resource</em>" is replaced by the HCatalog
+          resource name.</p>
+       <p>For example, to check if the server is running you could
+          access the following URL:</p>
+       <p><code>http://www.myserver.com/templeton/v1/status</code></p>
+    </section>
+
+   <section>
+      <title>Security </title>
+       <p>The current version supports two types of security:</p>
+       <ul>
+       <li>Default security (without additional authentication)</li>
+       <li>Authentication via
+           <a href="http://web.mit.edu/kerberos/">Kerberos</a></li>
+       </ul>
+     <section>
+       <title>Standard Parameters </title>
+       <p>Every REST resource can accept the following parameters to
+         aid in authentication: </p>
+         <ul>
+         <li>user.name: The user name as a string.
+             Only valid when using default security. </li>
+         <li>SPNEGO credentials: When running with Kerberos authentication. </li>
+         </ul>
+     </section>
+
+     <section>
+       <title>Security Error Response</title>
+       <p>If the user.name parameter is not supplied when required, 
+        the following error will be returned: </p>
+<source>
+{
+  "error": "No user found.  Missing user.name parameter."
+}
+</source>
+     </section>
+
+   </section>
+
+   <section>
+      <title>WebHDFS and Code Push</title>
+      <p>Data and code that are used by HCatalog's REST resources must first be placed in
+         Hadoop.  When placing files into HDFS is required you can use
+         whatever method is most convienient for you.  We suggest WebHDFS since it provides
+         a REST interface for moving files into and out of HDFS.</p>
+   </section>
+
+   <section>
+        <title>Error Codes and Responses</title>
+        <p>The server returns the following HTTP status codes.</p>
+        <ul>
+        <li><strong>200 OK:</strong> Success!</li>
+        <li><strong>400 Bad Request:</strong> The request was invalid.</li>
+        <li><strong>401 Unauthorized:</strong> Credentials were missing or incorrect.</li>
+        <li><strong>404 Not Found:</strong> The URI requested is invalid or the
+                    resource requested does not exist.</li>
+        <li><strong>500 Internal Server Error:</strong> We received an unexpected result.</li>
+        <li><strong>503 Busy, please retry:</strong> The server is busy.</li>
+        </ul>
+        <p>Other data returned directly by the server is returned in JSON format.
+         JSON responses are limited to 1MB in size.  Responses over this limit must be
+         stored into HDFS using provided options instead of being directly returned.
+         If an HCatalog DDL command might return results greater than 1MB, it's
+         suggested that a corresponding Hive request be executed instead.</p>
+   </section>
+
+   <section>
+       <title>Log Files</title>
+       <p>The server creates three log files when in operation:</p>
+       <ul>
+       <li><strong>templeton.log</strong> is the log4j log.  This the main log the application
+        writes to.</li>
+       <li><strong>templeton-console.log</strong> is what Java writes to stdout when the server is
+        started.  It is a small amount of data, similar to "hcat.out".</li>
+       <li><strong>tempelton-console-error.log</strong> is what Java writes to stderr, similar to
+        "hcat.err".</li>
+       </ul>
+       <p>In the tempelton-log4j.properties file you can set the location of these logs using the
+        variable templeton.log.dir.  This log4j.properties file is set in the server startup script.</p>
+   </section>
+
+   <section>
+    <title>Project Name</title>
+    <p>The original work to add REST APIs to HCatalog was called Templeton.  For backward compatibility
+       the name still appears in URLs, log file names, etc.  The Templeton name is taken from a character
+       in the award-winning children's novel Charlotte's Web, by E. B. White.  The novel's protagonist is a pig named
+       Wilber.  Templeton is a rat who helps Wilber by running errands and making deliveries as 
+       requested by Charlotte while spinning her web.</p>
+   </section>
+  </body>
+</document>

Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest_server_install.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest_server_install.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest_server_install.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/rest_server_install.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,258 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+  <header>
+    <title>Installation </title>
+  </header>
+  <body>
+
+  <section>
+  <title>Procedure</title>
+  <ol>
+  <li>Ensure that the
+       <a href="rest_server_install.html#Requirements">required related installations</a>
+      are in place, and place required files into the
+      <a href="rest_server_install.html#Hadoop+Distributed+Cache">Hadoop distributed cache.</a></li>
+  <li>Download and unpack the HCatalog distribution.</li>
+  <li>Set the <code>TEMPLETON_HOME</code> environment variable to the base of the
+      HCatalog REST server installation.  This will usually be <code>HCATALOG_HOME/webhcat</code>.
+      This is used to find the Templeton
+      configuration.</li>
+  <li>Review the <a href="configuration.html">configuration</a>
+      and update or create <code>templeton-site.xml</code> as required. Ensure that
+      site specific component installation locations are accurate, especially
+      the Hadoop configuration path.  Configuration variables that use a filesystem
+      path try to have reasonable defaults, but it's always safe to specify a full
+      and complete path.</li>
+  <li>Verify that HCatalog is installed and that the <code>hcat</code>
+      executable is in the PATH.</li>
+  <li>Build HCatalog using the command <code>ant jar</code> from the
+      top level HCatalog directory.</li>
+  <li>Start the REST server with the command
+      <code>bin/templeton_server.sh start</code>.</li>
+  <li>Check that your local install works.  Assuming that the server is running
+      on port 8080, the following command would give output similar to that shown.
+<source>
+% curl -i http://localhost:50111/templeton/v1/status
+HTTP/1.1 200 OK
+Content-Type: application/json
+Transfer-Encoding: chunked
+Server: Jetty(7.6.0.v20120127)
+
+{"status":"ok","version":"v1"}
+%
+</source></li>
+  </ol>
+  </section>
+
+  <section>
+  <title>Server Commands</title>
+  <ul>
+   <li><strong>Start the server:</strong> <code>bin/templeton_server.sh start</code></li>
+   <li><strong>Stop the server:</strong> <code>bin/templeton_server.sh stop</code></li>
+   <li><strong>End-to-end build, run, test:</strong> <code>ant e2e</code></li>
+  </ul>
+  </section>
+
+  <section>
+  <title>Requirements</title>
+  <ul>
+   <li><a href="http://ant.apache.org/">Ant</a>, version 1.8 or higher</li>
+   <li><a href="http://hadoop.apache.org/">Hadoop</a>, version 0.20.205.0</li>
+   <li><a href="http://zookeeper.apache.org/">ZooKeeper</a> is required if you are using
+    the ZooKeeper storage class. (Be sure to review
+    and update the ZooKeeper related <a href="configuration.html">Templeton
+    configuration</a>.)</li>
+   <li>
+     <a href="http://incubator.apache.org/hcatalog/">HCatalog</a>.
+     Version 0.4.1 or higher.  The hcat executable must be both in the
+     PATH and properly configured in the <a
+     href="configuration.html">Templeton configuration</a>.
+   </li>
+   <li>
+     Permissions must be given to the user running the server.
+     (see below)
+   </li>
+   <li>
+     If running a secure cluster, Kerberos keys and principals must be
+     created.  (see below)
+   </li>
+   <li><a href="rest_server_install.html#Hadoop+Distributed+Cache">Hadoop Distributed Cache</a>.
+    To use the <a href="http://hive.apache.org/">Hive</a>,
+    <a href="http://pig.apache.org/">Pig</a>, or
+    <a href="http://hadoop.apache.org/common/docs/r1.0.0/streaming.html">hadoop/streaming</a>
+    resources, see instructions below for placing the required files in the
+    Hadoop Distributed Cache.</li>
+  </ul>
+  </section>
+
+  <section>
+  <title>Hadoop Distributed Cache</title>
+
+  <p>The server requires some files be accessible on the
+  <a href="http://hadoop.apache.org/common/docs/current/mapred_tutorial.html#DistributedCache">
+   Hadoop distributed cache</a>.  For example, to avoid the installation of Pig and Hive
+   everywhere on the cluster, the server gathers a version of Pig or Hive from the
+   Hadoop distributed cache whenever those resources are invoked. After placing the
+   following components into HDFS please update the site configuration as required for
+   each.</p>
+
+  <ul>
+  <li><strong>Hive</strong>:
+   <a href="http://www.apache.org/dyn/closer.cgi/incubator/hcatalog/">Download</a>
+   the HCatalog tar.gz file and place it in HDFS. (If you need a version
+   that is not yet released, you may need to build it yourself following the HCatalog
+   instructions.)
+<source>
+hadoop fs -put /tmp/hcatalog-0.3.0.tar.gz /user/templeton/hcatalog-0.3.0.tar.gz
+</source>
+  </li>
+
+  <li><strong>Pig</strong>:
+   <a href="http://www.apache.org/dyn/closer.cgi/pig">Download</a>
+   the Pig tar.gz file and place it into HDFS.  For example:
+<source>
+hadoop fs -put /tmp/pig-0.9.2.tar.gz /user/templeton/pig-0.9.2.tar.gz
+</source>
+  </li>
+
+  <li><strong>Hadoop Streaming</strong>:
+   Place <code>hadoop-streaming.jar</code> into HDFS. For example, use the
+   following command, substituting your path to the jar for the one below.
+<source>
+hadoop fs -put $HADOOP_PREFIX/hadoop-0.20.205.0/contrib/streaming/hadoop-streaming-0.20.205.0.jar \
+ /user/templeton/hadoop-streaming.jar
+</source>
+  </li>
+
+  <li><strong>Override Jars</strong>:
+   Place override jars required (if any) into HDFS. <em>Note</em>: As of this writing, all
+   released versions of Hadoop require a patch to properly run Templeton.  This patch is
+   distributed with Templeton (located at
+   <code>templeton/src/hadoop_temp_fix/ugi.jar</code>)
+   and should be placed into HDFS, as reflected in the current default configuration.
+<source>
+hadoop fs -put ugi.jar /user/templeton/ugi.jar
+</source>
+  </li>
+  </ul>
+
+  <p>The location of these files in the cache, and the location
+     of the installations inside the archives, can be specified using the following
+     Templeton configuration variables.  (See the
+     <a href="configuration.html">Configuration</a> documentation for more information
+     on changing Templeton configuration parameters.)</p>
+
+  <table>
+  <tr><th>Name</th><th>Default</th><th>Description</th></tr>
+
+  <tr>
+    <td><strong>templeton.pig.archive</strong></td>
+    <td><code>hdfs:///user/templeton/pig-0.9.2.tar.gz</code></td>
+    <td>The path to the Pig archive.</td>
+  </tr>
+
+  <tr>
+    <td><strong>templeton.pig.path</strong></td>
+    <td><code>pig-0.9.2.tar.gz/pig-0.9.2/bin/pig</code></td>
+    <td>The path to the Pig executable.</td>
+  </tr>
+
+  <tr>
+    <td><strong>templeton.hive.archive</strong></td>
+    <td><code>hdfs:///user/templeton/hcatalog-0.3.0.tar.gz</code></td>
+    <td>The path to the Hive archive.</td>
+  </tr>
+
+  <tr>
+    <td><strong>templeton.hive.path</strong></td>
+    <td><code>hcatalog-0.3.0.tar.gz/hcatalog-0.3.0/bin/hive</code></td>
+    <td>The path to the Hive executable.</td>
+  </tr>
+
+  <tr>
+    <td><strong>templeton.streaming.jar</strong></td>
+    <td><code>hdfs:///user/templeton/hadoop-streaming.jar</code></td>
+    <td>The path to the Hadoop streaming jar file.</td>
+  </tr>
+
+  <tr>
+    <td><strong>templeton.override.jars</strong></td>
+    <td><code>hdfs:///user/templeton/ugi.jar</code></td>
+    <td>Jars to add to the HADOOP_CLASSPATH for all Map Reduce jobs.
+        These jars must exist on HDFS.</td>
+  </tr>
+  </table>
+  </section>
+
+  <section>
+  <title>Permissions</title>
+  <p>
+    Permission must given for the user running the templeton
+    executable to run jobs for other users.  That is, the templeton
+    server will impersonate users on the Hadoop cluster.
+  </p>
+
+  <p>
+    Create (or assign) a Unix user who will run the Templeton server.
+    Call this USER.  See the Secure Cluster section below for choosing
+    a user on a Kerberos cluster.
+  </p>
+
+  <p>
+    Modify the Hadoop core-site.xml file and set these properties:
+  </p>
+
+  <table>
+    <tr><th>Variable</th><th>Value</th></tr>
+    <tr>
+      <td>hadoop.proxyuser.USER.groups</td>
+      <td>
+        A comma separated list of the Unix groups whose users will be
+        impersonated.
+      </td>
+    </tr>
+    <tr>
+      <td>hadoop.proxyuser.USER.hosts</td>
+      <td>
+        A comma separated list of the hosts that will run the hcat and
+        JobTracker servers.
+      </td>
+    </tr>
+  </table>
+  </section>
+
+  <section>
+  <title>Secure Cluster</title>
+  <p>
+    To run Templeton on a secure cluster follow the Permissions
+    instructions above but create a Kerberos principal for the
+    Templeton server with the name <code>USER/host@realm</code>
+  </p>
+
+  <p>
+    Also, set the templeton configuration variables
+    <code>templeton.kerberos.principal</code> and
+    <code>templeton.kerberos.keytab</code>
+  </p>
+  </section>
+
+  </body>
+</document>

Modified: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/site.xml?rev=1365722&r1=1365721&r2=1365722&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/site.xml Wed Jul 25 20:29:44 2012
@@ -47,6 +47,48 @@ See http://forrest.apache.org/docs/linki
     <index label="Storage Formats" href="supportedformats.html" />
     <index label="Dynamic Partitioning" href="dynpartition.html" />
     <index label="Notification" href="notification.html" />    
+    <index label="WebHCat">
+      <index label="Overview" href="rest.html" />
+      <index label="Installation" href="rest_server_install.html" />
+      <index label="Configuration" href="configuration.html" />
+      <index label="Reference">
+        <index label="Resource List" href="resources.html" />
+        <index label=":version" href="responsetypes.html" />
+        <index label="status" href="status.html" />
+        <index label="version" href="versions.html" />
+        <index label="ddl">
+          <index label="ddl (summary)" href="ddlsummary.html" />
+          <index label="ddl" href="ddl.html" />
+          <index label="database" href="listdbs.html" />
+          <index label=":db (GET)" href="descdb.html" />
+          <index label=":db (PUT)" href="createdb.html" />
+          <index label=":db (DELETE)" href="deletedb.html" />
+          <index label="table" href="listtables.html" />
+          <index label=":table (GET)" href="desctable.html" />
+          <index label=":table (PUT)" href="createtable.html" />
+          <index label=":table (POST)" href="renametable.html" />
+          <index label=":table (DELETE)" href="deletetable.html" />
+          <index label="like/:table (PUT)" href="createtablelike.html" />
+          <index label="partition" href="listpartitions.html" />
+          <index label=":partition (GET)" href="descpartition.html" />
+          <index label=":partition (PUT)" href="createpartition.html" />
+          <index label=":partition (DELETE)" href="deletepartition.html" />
+          <index label="column" href="listcolumns.html" />
+          <index label=":column (GET)" href="desccolumn.html" />
+          <index label=":column (PUT)" href="createcolumn.html" />
+          <index label="property" href="listproperties.html" />
+          <index label=":property (GET)" href="descproperty.html" />
+          <index label=":property (PUT)" href="addproperty.html" />
+        </index>
+        <index label="mapreduce/streaming" href="mapreducestreaming.html" />
+        <index label="mapreduce/jar" href="mapreducejar.html" />
+        <index label="pig" href="pig.html" />
+        <index label="hive" href="hive.html" />
+        <index label="queue" href="queuelist.html" />
+        <index label=":jobid (GET)" href="queue.html" />
+        <index label=":jobid (DELETE)" href="queuedelete.html" />
+      </index>
+    </index>
 
     <api   label="API Docs" href="api/index.html"/>
   </docs>  

Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/status.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/status.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/status.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/status.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+  <header>
+    <title>GET status </title>
+  </header>
+
+  <body>
+  <section>
+   <title>Description</title>
+    <p>Returns the current status of the Templeton server.
+       Useful for heartbeat monitoring.</p>
+  </section>
+
+  <section>
+   <title>URL</title>
+    <p><code>http://</code>www.myserver.com<code>/templeton/v1/status</code></p>
+  </section>
+
+  <section>
+   <title>Parameters</title>
+    <p>Only the <a href="index.html#Standard+Parameters">standard parameters</a>
+       are accepted.</p>
+  </section>
+
+  <section>
+   <title>Results</title>
+    <table>
+    <tr><th>Name</th><th>Description</th></tr>
+
+    <tr><td><strong>status</strong></td>
+        <td>"ok" if the Templeton server was contacted.</td>
+    </tr>
+
+    <tr><td><strong>version</strong></td>
+        <td>String containing the version number similar to "v1".</td>
+    </tr>
+    </table>
+  </section>
+
+  <section>
+   <title>Example</title>
+
+   <p><strong>Curl Command</strong></p>
+<source>
+% curl -s 'http://localhost:50111/templeton/v1/status'
+</source>
+
+   <p><strong>JSON Output</strong></p>
+<source>
+{
+ "status": "ok",
+ "version": "v1"
+}
+</source>
+  </section>
+ </body>
+</document>

Added: incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/versions.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/versions.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/versions.xml (added)
+++ incubator/hcatalog/trunk/src/docs/src/documentation/content/xdocs/versions.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+  <header>
+    <title>GET version </title>
+  </header>
+
+  <body>
+  <section>
+   <title>Description</title>
+    <p>Returns a list of supported versions and the current version.</p>
+  </section>
+
+  <section>
+   <title>URL</title>
+    <p><code>http://</code>www.myserver.com<code>/templeton/v1/version</code></p>
+  </section>
+
+  <section>
+   <title>Parameters</title>
+    <p>Only the <a href="index.html#Standard+Parameters">standard parameters</a>
+       are accepted.</p>
+  </section>
+
+  <section>
+   <title>Results</title>
+    <table>
+    <tr><th>Name</th><th>Description</th></tr>
+
+    <tr><td><strong>supportedVersions</strong></td>
+        <td>A list of all supported versions.</td>
+    </tr>
+
+    <tr><td><strong>version</strong></td>
+        <td>The current version.</td>
+    </tr>
+
+    </table>
+  </section>
+
+  <section>
+   <title>Example</title>
+
+   <p><strong>Curl Command</strong></p>
+<source>
+% curl -s 'http://localhost:50111/templeton/v1/version'
+</source>
+
+   <p><strong>JSON Output</strong></p>
+<source>
+{
+ "supportedVersions": [
+   "v1"
+ ],
+ "version": "v1"
+}
+</source>
+  </section>
+ </body>
+</document>

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java?rev=1365722&r1=1365721&r2=1365722&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java Wed Jul 25 20:29:44 2012
@@ -160,7 +160,7 @@ public class MultiOutputFormat extends O
      * for multiple output formats.
      *
      * @param job the mapreduce job to be submitted
-     * @return
+     * @return JobConfigurer
      */
     public static JobConfigurer createConfigurer(Job job) {
         return JobConfigurer.create(job);
@@ -361,7 +361,7 @@ public class MultiOutputFormat extends O
          *
          * @param alias the name used for the OutputFormat during
          *            addOutputFormat
-         * @return
+         * @return Job
          */
         public Job getJob(String alias) {
             Job copy = outputConfigs.get(alias);

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/README.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/README.txt?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/README.txt (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/README.txt Wed Jul 25 20:29:44 2012
@@ -0,0 +1,104 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+End to end tests
+---------------
+End to end tests in templeton runs tests against an existing templeton server.
+It runs hcat, mapreduce, streaming, hive and pig tests.
+
+Test cases
+----------
+The tests are defined in src/test/e2e/templeton/tests/*.conf
+
+Test framework
+--------------
+The test framework is derived from the one used in pig, there is more documentation here on the framework -
+https://cwiki.apache.org/confluence/display/PIG/HowToTest
+
+
+Setup
+-----
+1. Templeton needs to be installed and setup to be able to run hcat, maprduce, hive and pig commands. 
+
+2. Install perl and following perl modules  (cpan -i <MODULE_NAME>)
+* IPC::Run
+* JSON
+* Data::Dump
+* Number::Compare
+* Text::Glob
+* Data::Compare
+* File::Find::Rule
+* HTTP::Daemon
+
+Tips:
+* Using perlbrew (http://perlbrew.pl) should make installing perl modules easier. 
+* Use 'yes | cpan -i <MODULE_NAME>' to avoid answering the 100's of questions cpan asks.
+
+
+
+3. Copy contents of src/test/e2e/templeton/inpdir to hdfs
+
+
+Running the tests
+-----------------
+Use the following command to run tests -
+
+ant test -Dinpdir.hdfs=<location of inpdir on hdfs>  -Dtest.user.name=<user the tests should run as> \
+ -Dsecure.mode=<yes/no>   -Dharness.webhdfs.url=<webhdfs url upto port num>  -Dharness.templeton.url=<templeton url upto port num> 
+
+If you want to run specific test group you can specify the group, for example:  -Dtests.to.run='-t TestHive'
+
+If you want to run specific test in a group group you can specify the test, for example:  -Dtests.to.run='-t TestHive_1'
+
+
+Running the hcat authorization tests
+------------------------------------
+Hcat authorization tests run commands as different users to test if authorization is done right.
+
+ant test-hcat-authorization -Dkeytab.dir=<keytab files dir> 
+  -Dsecure.mode=<yes/no>  -Dtest.group.name=<common group> -Dinpdir.hdfs=<location of inpdir on hdfs>  
+  -Dtest.user.name=<user 1 belonging to common group> -Dtest.group.user.name=<user 2 belonging to common group>  
+  -Dtest.other.user.name=<user 3 who does not belong to common group> 
+  -Dharness.webhdfs.url=<webhdfs url upto port num>  -Dharness.templeton.url=<templeton url upto port num> 
+
+The <keytab files dir> is expected to have keytab filenames of the form - user_name.*keytab .
+
+
+Notes
+-----
+
+
+
+Enable webhdfs by adding the following to your hadoop hdfs-site.xml :
+
+<property>
+  <name>dfs.webhdfs.enabled</name>
+  <value>true</value>
+</property>
+<property>
+  <name>dfs.http.address</name>
+  <value>127.0.0.1:8085</value>
+  <final>true</final>
+</property>
+
+You can build a server that will measure test coverage by using templeton:
+ant clean; ant e2e
+This assumes you've got webhdfs at the address above, the inpdir info in /user/templeton, and templeton running on the default port.  You can change any of those properties in the build file.
+
+It's best to set HADOOP_HOME_WARN_SUPPRESS=true everywhere you can.

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/build.xml?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/build.xml (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/build.xml Wed Jul 25 20:29:44 2012
@@ -0,0 +1,95 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<project name="TestHarnessTempletonTests" default="test">
+
+  <!-- Separate property name for udfs' build.xml -->
+  <property name="e2e.lib.dir" value="${basedir}/lib"/>
+
+  <property name="test.src" value="${basedir}/tests"/>
+  <property name="driver.src" value="${basedir}/drivers"/>
+  <property name="harness.dir" value="${basedir}/../harness"/>
+  <property name="inpdir.local" value="${basedir}/inpdir/"/>
+  <property name="test.location" value="${basedir}/testWorkDir"/>
+  <property name="driver.src" value="${basedir}/drivers"/>
+
+  <!-- Check that the necessary properties are setup -->
+  <target name="property-check">
+    <fail message="Please set the property harness.webhdfs.url to the namenode base url of the cluster"
+      unless="harness.webhdfs.url"/>
+    <fail message="Please set the property harness.templeton.url to the templeton server base url of the cluster"
+      unless="harness.templeton.url"/>
+    <fail message="Please set the property inpdir.hdfs to the test input directory on hdfs"
+      unless="inpdir.hdfs"/>
+  </target>
+
+  <!-- Prep the test area -->
+  <target name="init-test">
+    <mkdir dir="${test.location}"/>
+  </target>
+
+  <target name="test" depends="property-check, init-test" >
+    <property name="tests.to.run" value=""/>
+    <exec executable="${harness.dir}/test_harness.pl" dir="${test.location}" failonerror="true">
+      <env key="HARNESS_ROOT" value="${harness.dir}"/>
+      <env key="DRIVER_ROOT" value="${basedir}/drivers"/>
+      <env key="TH_WORKING_DIR" value="${test.location}"/>
+      <env key="TH_INPDIR_LOCAL" value="${inpdir.local}"/>
+      <env key="TH_INPDIR_HDFS" value="${inpdir.hdfs}"/>
+      <env key="TH_OUT" value="."/>
+      <env key="TH_ROOT" value="."/>
+      <env key="WEBHDFS_URL" value="${harness.webhdfs.url}"/>
+      <env key="TEMPLETON_URL" value="${harness.templeton.url}"/>
+      <env key="USER_NAME" value="${test.user.name}"/>
+      <env key="HARNESS_CONF" value="${basedir}/conf/default.conf"/>
+      <env key="SECURE_MODE" value="${secure.mode}"/>
+      <arg line="${tests.to.run}"/>
+      <arg value="${basedir}/tests/serverstatus.conf"/>
+      <arg value="${basedir}/tests/ddl.conf"/>
+      <arg value="${basedir}/tests/jobsubmission.conf"/>
+    </exec>
+  </target>
+
+  <target name="test-hcat-authorization" depends="property-check, init-test" >
+    <property name="tests.to.run" value=""/>
+    <exec executable="${harness.dir}/test_harness.pl" dir="${test.location}" failonerror="true">
+      <env key="HARNESS_ROOT" value="${harness.dir}"/>
+      <env key="DRIVER_ROOT" value="${basedir}/drivers"/>
+      <env key="TH_WORKING_DIR" value="${test.location}"/>
+      <env key="TH_INPDIR_LOCAL" value="${inpdir.local}"/>
+      <env key="TH_INPDIR_HDFS" value="${inpdir.hdfs}"/>
+      <env key="TH_OUT" value="."/>
+      <env key="TH_ROOT" value="."/>
+      <env key="WEBHDFS_URL" value="${harness.webhdfs.url}"/>
+      <env key="TEMPLETON_URL" value="${harness.templeton.url}"/>
+      <env key="USER_NAME" value="${test.user.name}"/>
+      <env key="GROUP_NAME" value="${test.group.name}"/>
+      <env key="GROUP_USER_NAME" value="${test.group.user.name}"/>
+      <env key="OTHER_USER_NAME" value="${test.other.user.name}"/>
+      <env key="HARNESS_CONF" value="${basedir}/conf/default.conf"/>
+      <env key="SECURE_MODE" value="${secure.mode}"/>
+      <env key="KEYTAB_DIR" value="${keytab.dir}"/>
+      <arg line="${tests.to.run}"/>
+      <arg value="${basedir}/tests/hcatperms.conf"/>
+    </exec>
+  </target>
+
+
+
+</project>
+
+

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/conf/default.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/conf/default.conf?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/conf/default.conf (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/conf/default.conf Wed Jul 25 20:29:44 2012
@@ -0,0 +1,40 @@
+############################################################################           
+#  Licensed to the Apache Software Foundation (ASF) under one or more                  
+#  contributor license agreements.  See the NOTICE file distributed with               
+#  this work for additional information regarding copyright ownership.                 
+#  The ASF licenses this file to You under the Apache License, Version 2.0             
+#  (the "License"); you may not use this file except in compliance with                
+#  the License.  You may obtain a copy of the License at                               
+#                                                                                      
+#      http://www.apache.org/licenses/LICENSE-2.0                                      
+#                                                                                      
+#  Unless required by applicable law or agreed to in writing, software                 
+#  distributed under the License is distributed on an "AS IS" BASIS,                   
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.            
+#  See the License for the specific language governing permissions and                 
+#  limitations under the License.                                                      
+                                                                                       
+my $me = `whoami`;
+chomp $me;
+
+# The contents of this file can be rewritten to fit your installation.
+# Also, you can define the following environment variables and set things up as in the test setup
+# TH_ROOT           Root directory where test harness is installed
+# TH_OUT       	    Root directory where output data will be stored 
+
+$cfg = {
+    #HDFS
+      'outpathbase'    => '/tmp/templeton_test_out'
+    , 'localoutpathbase' => "$ENV{TH_WORKING_DIR}/"
+
+    #TEST
+    , 'tmpPath'          => '/tmp/pigtest'
+
+    , 'userhomePath' => "$ENV{HOME}"
+    ,'local.bin'     => '/usr/bin'
+ 
+    ,'logDir'                => "$ENV{TH_OUT}/log" 
+#    ,'propertiesFile'     => "./conf/testpropertiesfile.conf"
+    ,'harness.console.level' => 'ERROR'
+
+};

Added: incubator/hcatalog/trunk/src/test/e2e/templeton/deployAndTest.pl
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/templeton/deployAndTest.pl?rev=1365722&view=auto
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/templeton/deployAndTest.pl (added)
+++ incubator/hcatalog/trunk/src/test/e2e/templeton/deployAndTest.pl Wed Jul 25 20:29:44 2012
@@ -0,0 +1,158 @@
+#!/usr/local/bin/perl -w
+use strict;
+
+my $TESTDIR = '/tmp/templetontest/';
+my $TEST_INP_DIR = '/tmp/test_inpdir/'; #dir on hadoop
+my $TEST_USER = 'hortonth';
+my $WEBHDFS_URL = 'http://localhost:50070';
+my $TEMPLETON_URL = 'http://localhost:8080';
+my $CATALINA_HOME = $ENV{'CATALINA_HOME'};
+
+#use env variables if they have been set
+if(defined  $ENV{'TESTDIR'}){
+    $TESTDIR = $ENV{'TESTDIR'};
+}
+if(defined  $ENV{'TEST_INP_DIR'}){
+    $TEST_INP_DIR = $ENV{'TEST_INP_DIR'};
+}
+if(defined  $ENV{'TEST_USER'}){
+    $TEST_USER = $ENV{'TEST_USER'};
+}
+if(defined  $ENV{'WEBHDFS_URL'}){
+    $WEBHDFS_URL = $ENV{'WEBHDFS_URL'};
+}
+if(defined  $ENV{'TEMPLETON_URL'}){
+    $TEMPLETON_URL = $ENV{'TEMPLETON_URL'};
+}
+
+if(! defined $ENV{'HCAT_PREFIX'}){
+    $ENV{'HCAT_PREFIX'}='/usr/';
+}
+
+if(! defined $ENV{'HADOOP_PREFIX'}){
+    $ENV{'HADOOP_PREFIX'}='/usr/';
+}
+
+my $host = `hostname` ;
+chomp $host;
+
+if(! defined $ENV{'ZOOKEEPER_HOST'}){
+    $ENV{'ZOOKEEPER_HOST'} =  $host . ':2181';
+}
+
+if(! defined $ENV{'METASTORE_HOST'}){
+    $ENV{'METASTORE_HOST'} =  $host . ':9933';
+}
+
+print STDERR "##################################################################\n";
+print STDERR "Using the following settings for environment variables\n" .
+    " (Set them to override the default values) \n" .
+    "WEBHDFS_URL : $WEBHDFS_URL \n" .
+    "TEMPLETON_URL : $TEMPLETON_URL \n" .
+    'CATALINA_HOME :' . $ENV{'CATALINA_HOME'} . "\n" .
+    'HADOOP_PREFIX :' . $ENV{'HADOOP_PREFIX'} . "\n" .
+    'HCAT_PREFIX :' . $ENV{'HCAT_PREFIX'} . "\n" .
+    'ZOOKEEPER_HOST :' . $ENV{'ZOOKEEPER_HOST'} . "\n" .
+    'METASTORE_HOST :' . $ENV{'METASTORE_HOST'} . "\n" 
+;
+print STDERR "##################################################################\n";
+
+system("rm -rf $TESTDIR/");
+
+#restart tomcat with updated env variables
+my $templeton_src = "$TESTDIR/templeton_src";
+$ENV{'TEMPLETON_HOME'} = "$templeton_src/templeton";
+
+system ("$CATALINA_HOME/bin/shutdown.sh") == 0 or die "tomcat shutdown failed" ;
+sleep 3;
+
+
+
+#get templeton git repo, build and install
+system("mkdir -p $templeton_src") == 0 or die "could not create dir $templeton_src: $!";
+chdir  "$templeton_src"  or die "could not change directory to  $templeton_src  : $!";
+system ('git clone  git@github.com:hortonworks/templeton.git')  == 0 or die "could not clone templeton git repo";
+chdir 'templeton' or die 'could not change dir : $!';
+
+
+#put a templeton-site.xml in $TEMPLETON_HOME with zookeeper hostname
+writeTempletonSiteXml();
+system ('ant install-war') == 0 or die "templeton build failed";
+
+#tomcat should have shutdown by now, try starting it
+system ("$CATALINA_HOME/bin/startup.sh") == 0 or die 'tomcat startup failed';
+sleep 3;
+
+my $tdir = "$templeton_src/templeton/src/test/e2e/templeton";
+chdir $tdir or die "could not change dir $tdir : $!";
+
+#copy input files
+system("hadoop fs -rmr $TEST_INP_DIR");
+system("hadoop fs -copyFromLocal $tdir/inpdir $TEST_INP_DIR") == 0 or die "failed to copy input dir : $!";
+system("hadoop fs -chmod -R 777 $TEST_INP_DIR")  == 0 or die "failed to set input dir permissions : $!";
+
+#start tests
+my $cmd = "ant test -Dinpdir.hdfs=$TEST_INP_DIR  -Dtest.user.name=$TEST_USER" .
+    " -Dharness.webhdfs.url=$WEBHDFS_URL -Dharness.templeton.url=$TEMPLETON_URL ";
+
+system($cmd) == 0 or die "templeton tests failed";
+
+#############################
+sub writeTempletonSiteXml {
+    my $conf = $ENV{'TEMPLETON_HOME'} . "/templeton-site.xml";
+    open ( CFH,  ">$conf" ) or die $!;
+
+
+    print CFH '<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+  <property>
+    <name>templeton.zookeeper.hosts</name>
+    <value>'  . 
+ $ENV{'ZOOKEEPER_HOST'} .
+'</value>
+    <description>ZooKeeper servers, as comma separated host:port pairs</description>
+  </property>
+
+  <property>
+    <name>templeton.hive.properties</name>
+    <value>hive.metastore.local=false,hive.metastore.uris=thrift://' .
+    $ENV{'METASTORE_HOST'} .
+    ',hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true</value>
+    <description>Properties to set when running hive.</description>
+  </property>
+
+  <property>
+    <name>templeton.hive.archive</name>
+    <value>hdfs:///user/templeton/hcatalog-0.3.0.tar.gz</value>
+    <description>The path to the Hive archive.</description>
+  </property>
+
+  <property>
+    <name>templeton.hive.path</name>
+    <value>hcatalog-0.3.0.tar.gz/hcatalog-0.3.0/bin/hive</value>
+    <description>The path to the Hive executable.</description>
+  </property>
+
+  <property>
+    <name>templeton.pig.archive</name>
+    <value>hdfs:///user/templeton/pig-0.9.2.tar.gz</value>
+    <description>The path to the Pig archive.</description>
+  </property>
+
+  <property>
+    <name>templeton.pig.path</name>
+    <value>pig-0.9.2.tar.gz/pig-0.9.2/bin/pig</value>
+    <description>The path to the Pig executable.</description>
+  </property>
+
+</configuration>
+';
+    close CFH or die $!;
+
+;
+
+
+
+
+
+}

Propchange: incubator/hcatalog/trunk/src/test/e2e/templeton/deployAndTest.pl
------------------------------------------------------------------------------
    svn:executable = *



Mime
View raw message