hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject svn commit: r1536919 - in /hive/trunk/hcatalog: src/test/e2e/templeton/tests/jobstatus.conf webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
Date Tue, 29 Oct 2013 22:22:26 GMT
Author: thejas
Date: Tue Oct 29 22:22:26 2013
New Revision: 1536919

URL: http://svn.apache.org/r1536919
Log:
HIVE-5519 : Use paging mechanism for templeton get requests. (Hari Sankar Sivarama Subramaniyan
via Thejas Nair)

Modified:
    hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf
    hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java

Modified: hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf?rev=1536919&r1=1536918&r2=1536919&view=diff
==============================================================================
--- hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf (original)
+++ hive/trunk/hcatalog/src/test/e2e/templeton/tests/jobstatus.conf Tue Oct 29 22:22:26 2013
@@ -152,6 +152,36 @@ $cfg = 
      'json_path' => {'$[-1:]' => 'job_.*'},
      'status_code' => 200,
     },
+    {
+     # GET jobs?user.name=UNAME_OTHER&fields=*, get all the details of the oldest 2 jobs
whose
+     # id is greater than job_0
+     'num' => 9,
+     'depends_on' => 'JOBS_1,JOBS_2,JOBS_3',
+     'method' => 'GET',
+     'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME_OTHER:&fields=*&numrecords=2&jobid=job_0',
+     'format_header' => 'Content-Type: application/json',
+     'json_path' => {'$[-1:].id' => 'job_.*',
+                     '$[-1:].detail.status.jobId' => 'job_.*',
+                     '$[-1:].detail.status.runState' => '\\d+',
+                     '$[-1:].detail.status.jobId' => 'job_.*',
+                     '$[-1:].detail.status.jobComplete' => 'true',
+                     '$[-1:].detail.profile.user' => ':UNAME_OTHER:',
+                     '$[-1:].detail.profile.jobFile' => '^.+$',
+                     '$[-1:].detail.profile.url' => '^.+$',
+                     '$[-1:].detail.profile.queueName' => '^.+$',
+                     '$[-1:].detail.profile.jobID.id' => '\\d+',
+                     '$[-1:].detail.profile.jobID.jtIdentifier' => '\\d+',
+                     '$[-1:].detail.profile.jobId' => 'job_.*',
+                     '$[-1:].detail.id' => 'job_.*',
+                     '$[-1:].detail.parentId' => 'job_.*',
+                     '$[-1:].detail.percentComplete' => '100%',
+                     '$[-1:].detail.exitValue' => '0',
+                     '$[-1:].detail.user' => ':UNAME_OTHER:',
+                     '$[-1:].detail.callback' => '^.+$',
+                     '$[-1:].detail.completed' => 'done',
+                    },
+     'status_code' => 200,
+    },
 
    ]
   }

Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1536919&r1=1536918&r2=1536919&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
(original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java
Tue Oct 29 22:22:26 2013
@@ -855,15 +855,59 @@ public class Server {
   }
 
   /**
-   * Return all the known job ids for this user.
+   * Return all the known job ids for this user based on the optional filter conditions.
+   * <p>
+   * Example usages:
+   * 1. curl -s 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan'
+   * Return all the Job IDs submitted by hsubramaniyan
+   * 2. curl -s 
+   * 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&showall=true'
+   * Return all the Job IDs that are visible to hsubramaniyan
+   * 3. curl -s
+   * 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&jobid=job_201312091733_0003'
+   * Return all the Job IDs for hsubramaniyan after job_201312091733_0003.
+   * 4. curl -s 'http://localhost:50111/templeton/v1/jobs? 
+   * user.name=hsubramaniyan&jobid=job_201312091733_0003&numrecords=5'
+   * Return the first 5(atmost) Job IDs submitted by hsubramaniyan after job_201312091733_0003.
 
+   * 5.  curl -s 
+   * 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&numrecords=5'
+   * Return the first 5(atmost) Job IDs submitted by hsubramaniyan after sorting the Job
ID list 
+   * lexicographically.
+   * </p>
+   * <p>
+   * Supporting pagination using "jobid" and "numrecords" parameters:
+   * Step 1: Get the start "jobid" = job_xxx_000, "numrecords" = n
+   * Step 2: Issue a curl command by specifying the user-defined "numrecords" and "jobid"

+   * Step 3: If list obtained from Step 2 has size equal to "numrecords", retrieve the list's

+   * last record and get the Job Id of the last record as job_yyy_k, else quit.
+   * Step 4: set "jobid"=job_yyy_k and go to step 2.
+   * </p> 
+   * @param fields If "fields" set to "*", the request will return full details of the job.
+   * If "fields" is missing, will only return the job ID. Currently the value can only
+   * be "*", other values are not allowed and will throw exception.
+   * @param showall If "showall" is set to "true", the request will return all jobs the user
+   * has permission to view, not only the jobs belonging to the user.
+   * @param jobid If "jobid" is present, the records whose Job Id is lexicographically greater

+   * than "jobid" are only returned. For example, if "jobid" = "job_201312091733_0001", 
+   * the jobs whose Job ID is greater than "job_201312091733_0001" are returned. The number
of 
+   * records returned depends on the value of "numrecords".
+   * @param numrecords If the "jobid" and "numrecords" parameters are present, the top #numrecords

+   * records appearing after "jobid" will be returned after sorting the Job Id list 
+   * lexicographically. 
+   * If "jobid" parameter is missing and "numrecords" is present, the top #numrecords will

+   * be returned after lexicographically sorting the Job Id list. If "jobid" parameter is
present 
+   * and "numrecords" is missing, all the records whose Job Id is greater than "jobid" are
returned.
+   * @return list of job items based on the filter conditions specified by the user.
    */
   @GET
   @Path("jobs")
   @Produces({MediaType.APPLICATION_JSON})
   public List<JobItemBean> showJobList(@QueryParam("fields") String fields,
-                                       @QueryParam("showall") boolean showall)
+                                       @QueryParam("showall") boolean showall,
+                                       @QueryParam("jobid") String jobid,
+                                       @QueryParam("numrecords") String numrecords)
     throws NotAuthorizedException, BadParam, IOException, InterruptedException {
-
+    
     verifyUser();
 
     boolean showDetails = false;
@@ -877,7 +921,46 @@ public class Server {
     ListDelegator ld = new ListDelegator(appConf);
     List<String> list = ld.run(getDoAsUser(), showall);
     List<JobItemBean> detailList = new ArrayList<JobItemBean>();
+    int currRecord = 0;
+    int numRecords;
+
+    // Parse numrecords to an integer
+    try {
+      if (numrecords != null) {
+        numRecords = Integer.parseInt(numrecords);
+	if (numRecords <= 0) {
+	  throw new BadParam("numrecords should be an integer > 0");
+	}    
+      }
+      else {
+        numRecords = -1;
+      }
+    }
+    catch(Exception e) {
+      throw new BadParam("Invalid numrecords format: numrecords should be an integer >
0");
+    }
+
+    // Sort the list lexicographically            
+    Collections.sort(list);
+
     for (String job : list) {
+      // If numRecords = -1, fetch all records.
+      // Hence skip all the below checks when numRecords = -1.
+      if (numRecords != -1) {
+        // If currRecord >= numRecords, we have already fetched the top #numRecords  
                                                                                         
                 
+        if (currRecord >= numRecords) {
+          break;
+        } 
+        // If the current record needs to be returned based on the 
+        // filter conditions specified by the user, increment the counter
+        else if ((jobid != null && job.compareTo(jobid) > 0) || jobid == null)
{
+          currRecord++;
+        }
+        // The current record should not be included in the output detailList.
+        else {
+          continue;
+        }
+      }
       JobItemBean jobItem = new JobItemBean();
       jobItem.id = job;
       if (showDetails) {



Mime
View raw message