hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From athu...@apache.org
Subject svn commit: r758836 - in /hadoop/hive/trunk: ./ hwi/ hwi/src/java/org/apache/hadoop/hive/hwi/ hwi/src/test/org/apache/hadoop/hive/hwi/ hwi/web/
Date Thu, 26 Mar 2009 20:26:01 GMT
Author: athusoo
Date: Thu Mar 26 20:26:01 2009
New Revision: 758836

URL: http://svn.apache.org/viewvc?rev=758836&view=rev
Log:
HIVE-278. Add support for Hive History to 
Hive Web UI. (Edward Capriolo via athusoo)


Added:
    hadoop/hive/trunk/hwi/web/session_history.jsp
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/hwi/build.xml
    hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java
    hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionManager.java
    hadoop/hive/trunk/hwi/src/test/org/apache/hadoop/hive/hwi/TestHWISessionManager.java
    hadoop/hive/trunk/hwi/web/session_manage.jsp

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=758836&r1=758835&r2=758836&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Mar 26 20:26:01 2009
@@ -84,6 +84,9 @@
     HIVE-367. Null Pointer when some of the files in the table are
     empty. (Namit Jain via athusoo) 
 
+    HIVE-278. Add support for Hive History to 
+    Hive Web UI. (Edward Capriolo via athusoo)
+
 Release 0.2.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/hwi/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hwi/build.xml?rev=758836&r1=758835&r2=758836&view=diff
==============================================================================
--- hadoop/hive/trunk/hwi/build.xml (original)
+++ hadoop/hive/trunk/hwi/build.xml Thu Mar 26 20:26:01 2009
@@ -27,15 +27,11 @@
   </target>
 
  <path id="classpath-hwi">
-    <pathelement location="${classpath}"/>
      <fileset dir="${hadoop.root}/lib">
        <include name="**/*.jar" />
        <exclude name="**/excluded/" />
      </fileset>
-     <fileset dir="../build">
-       <include name="**/*.jar" />
-       <exclude name="**/excluded/" />
-     </fileset>
+    <path refid="classpath" />
   </path>
 
 </project>

Modified: hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java?rev=758836&r1=758835&r2=758836&view=diff
==============================================================================
--- hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java (original)
+++ hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java Thu Mar
26 20:26:01 2009
@@ -13,156 +13,130 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.exec.ExecDriver;
+import org.apache.hadoop.hive.ql.history.HiveHistoryViewer;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.conf.*;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+
 /**
- * HWISessionItem can be viewed as a wrapper for a Hive shell. With it the user has a  
- * session on the web server rather then in a console window. 
- *
+ * HWISessionItem can be viewed as a wrapper for a Hive shell. With it the user
+ * has a session on the web server rather then in a console window.
+ * 
  */
 public class HWISessionItem implements Runnable, Comparable<HWISessionItem> {
 
-	protected static final Log l4j = LogFactory.getLog( HWISessionItem.class.getName() );
+	protected static final Log l4j = LogFactory.getLog(HWISessionItem.class
+			.getName());
+
 	/**
 	 * Represents the state a session item can be in.
-	 *
+	 * 
 	 */
-	public enum WebSessionItemStatus{
-		 NEW,QUERY_SET,QUERY_RUNNING,QUERY_COMPLETE,DESTROY,KILL_QUERY
+	public enum WebSessionItemStatus {
+		NEW, QUERY_SET, QUERY_RUNNING, QUERY_COMPLETE, DESTROY, KILL_QUERY
 	};
-	
+
 	private String sessionName;
 	private HWISessionItem.WebSessionItemStatus status;
-	
-	
+
 	private CliSessionState ss;
 	private SetProcessor sp;
 	private Driver qp;
 
 	private String resultFile;
 	private String errorFile;
-	
+
 	private String query;
 	private int queryRet;
-	
+
 	HiveConf conf;
 	Thread runnable;
+	HWIAuth auth;
+	private String historyFile;
+
 	/**
-	 * Creates an instance of WebSessionItem, sets status
-	 * to NEW.
+	 * Creates an instance of WebSessionItem, sets status to NEW.
 	 */
-	protected HWISessionItem(){
+	protected HWISessionItem() {
 		l4j.debug("HWISessionItem created");
-		status=WebSessionItemStatus.NEW;
-		queryRet=-40;
+		status = WebSessionItemStatus.NEW;
+		queryRet = -40;
+		runnable = new Thread(this);
+		runnable.start();
 	}
-	
+
 	/**
 	 * This is the initialization process that is carried out for each
-	 * SessionItem this is a section of the code taken from the CLIDriver
+	 * SessionItem. The goal is to emulate the startup of CLIDriver.
 	 */
 	protected void itemInit() {
-		l4j.debug("HWISessionItem itemInit start "+this.getSessionName() );
+		l4j.debug("HWISessionItem itemInit start " + this.getSessionName());
 		OptionsProcessor oproc = new OptionsProcessor();
-		
-		if (System.getProperty("hwi-args")!=null){
-			String [] parts = System.getProperty("hwi-args").split("\\s+");
-		
+
+		if (System.getProperty("hwi-args") != null) {
+			String[] parts = System.getProperty("hwi-args").split("\\s+");
+
 			if (!oproc.process_stage1(parts)) {
 			}
 		}
-		
+
 		SessionState.initHiveLog4j();
-		conf=new HiveConf(SessionState.class);
+		conf = new HiveConf(SessionState.class);
 		ss = new CliSessionState(conf);
 		SessionState.start(ss);
-
 		sp = new SetProcessor();
 		qp = new Driver();
-		l4j.debug("HWISessionItem itemInit Complete "+this.getSessionName() );
+
+		runSetProcessorQuery("hadoop.job.ugi=" + auth.getUser() + ","
+				+ auth.getGroups()[0]);
+		runSetProcessorQuery("user.name=" + auth.getUser());
+		/*
+		 * HiveHistoryFileName will not be accessible outside this thread. We must
+		 * capture this now.
+		 */
+		this.historyFile = this.ss.get().getHiveHistory().getHistFileName();
+		l4j.debug("HWISessionItem itemInit Complete " + this.getSessionName());
 	}
 
 	/**
-	* Set processor queries block for only a short amount of time the
-	* client can issue these directly.
-	* @param query This is a query in the form of SET THIS=THAT
-	* @return chained call to setProcessor.run(String)
-	*/
-	public int runSetProcessorQuery(String query){
+	 * Set processor queries block for only a short amount of time. The client can
+	 * issue these directly.
+	 * 
+	 * @param query
+	 *          This is a query in the form of SET THIS=THAT
+	 * @return chained call to setProcessor.run(String)
+	 */
+	public int runSetProcessorQuery(String query) {
 		return sp.run(query);
 	}
-	
+
 	/**
-	* The client does not start the thread themselves. They set their status to
-	* QUERY_SET. 
-	* 
-	*/
+	 * HWISessionItem is a Runnable instance. Calling this method will change the
+	 * status to QUERY_SET and notify(). The run method detects this and then
+	 * continues processing.
+	 */
 	public void clientStart() throws HWIException {
-		if (ss == null) {
-			throw new HWIException("SessionState is null.");
-		}
-
-		if (query == null) {
-			throw new HWIException("No Query was specified.");
-		}
 
 		if (this.status == WebSessionItemStatus.QUERY_RUNNING) {
 			throw new HWIException("Query already running");
 		}
 		this.status = WebSessionItemStatus.QUERY_SET;
-		l4j.debug(this.getSessionName()+" Query is set to start");
+		synchronized (this.runnable) {
+			this.runnable.notifyAll();
+		}
+		l4j.debug(this.getSessionName() + " Query is set to start");
 	}
-	
+
 	public void clientKill() throws HWIException {
-		if (this.status != WebSessionItemStatus.QUERY_RUNNING){
+		if (this.status != WebSessionItemStatus.QUERY_RUNNING) {
 			throw new HWIException("Can not kill that which is not running.");
 		}
-		this.status=WebSessionItemStatus.KILL_QUERY;
-		l4j.debug(this.getSessionName()+" Query is set to KILL_QUERY");
+		this.status = WebSessionItemStatus.KILL_QUERY;
+		l4j.debug(this.getSessionName() + " Query is set to KILL_QUERY");
 	}
-	/**
-	 * This method returns A KILL command a non silent clisession would return 
-	 * when invoking a HQL query from the CLI.
-	 * @return a string URL showing the job in the job tracker
-	 */
-	public Collection<String> getKillCommand(){
-		return ExecDriver.runningJobKillURIs.values();
-		//Kill Command = /opt/hadoop/hadoop-0.19.0/bin/../bin/hadoop job  -Dmapred.job.tracker=hadoop1.jointhegrid.local:54311
-kill job_200812231617_0001
-		//StringBuffer sb = new StringBuffer();
-		//sb.append( conf.getVar(HiveConf.ConfVars.HADOOPBIN));
-		//sb.append( " job ");
-		//sb.append( " -Dmapred.job.tracker= ");
-		//sb.append( conf.getVar(HiveConf.ConfVars.HADOOPJT));
-		//sb.append( " -kill ");
-		//sb.append( conf.getVar(HiveConf.ConfVars.HADOOPJOBNAME));
-		//return sb.toString();
-	}
-	/**
-	 * This method returns the URL a non silent clisession would return 
-	 * when invoking a HQL query from the CLI.
-	 * @return a string URL showing the job in the job tracker
-	 */
-	public Vector<String> getJobTrackerURI(){
-		//is this my job?
-		Vector<String> results = new Vector<String>();
-		for (String id: ExecDriver.runningJobKillURIs.keySet() ){
-
-			StringBuffer sb = new StringBuffer();
-			sb.append("http://");
-			//sb.append( this.conf.getVar( HiveConf.ConfVars.HADOOPJT ) );
-			sb.append( conf.get("mapred.job.tracker.http.address") );
-			sb.append("/jobdetails.jsp?jobid=");
-			sb.append(id);
-			results.add( sb.toString() );
-		
-		}
-		return results;
-		//http://hadoop1.jointhegrid.local:50030/jobdetails.jsp?jobid=
-	}
-	
+
 	/** This method clears the private member variables. */
 	public void clientRenew() throws HWIException {
 		if (this.status == WebSessionItemStatus.QUERY_RUNNING) {
@@ -173,59 +147,55 @@
 		this.resultFile = null;
 		this.errorFile = null;
 		this.status = WebSessionItemStatus.NEW;
-		l4j.debug(this.getSessionName()+" Query is renewed to start");
+		this.resultFile = null;
+		this.conf = null;
+		this.ss = null;
+		this.qp = null;
+		this.sp = null;
+		l4j.debug(this.getSessionName() + " Query is renewed to start");
 	}
 
-	/** 
-	* This is a chained call to SessionState.setIsSilent(). Use this 
-	* if you do not want the result file to have information status
-	*/
-	public void setSSIsSilent(boolean silent) throws HWIException{
-		if (ss == null) 
+	/**
+	 * This is a chained call to SessionState.setIsSilent(). Use this if you do
+	 * not want the result file to have information status
+	 */
+	public void setSSIsSilent(boolean silent) throws HWIException {
+		if (ss == null)
 			throw new HWIException("Session State is null");
 		this.ss.setIsSilent(silent);
 	}
-	
-	/** 
-	* This is a chained call to SessionState.getIsSilent()
-	*/
+
+	/**
+	 * This is a chained call to SessionState.getIsSilent()
+	 */
 	public boolean getSSIsSilent() throws HWIException {
-		if (ss == null) 
+		if (ss == null)
 			throw new HWIException("Session State is null");
 		return ss.getIsSilent();
 	}
 
-	 /**
-	 * This is a callback style function used by the HiveSessionManager. The
-	 * HiveSessionManager notices this and changes the state to QUERY_RUNNING
-	 * then starts a thread.
-	 */
-	protected void startIt() {
-		this.status = WebSessionItemStatus.QUERY_RUNNING;
-		l4j.debug(this.getSessionName()+" Runnable is starting");
-		runnable = new Thread(this);
-		runnable.start();
-		l4j.debug(this.getSessionName()+" Runnable has started");
-	}
 	/**
 	 * This is a callback style function used by the HiveSessionManager. The
-	 * HiveSessionManager notices this and attempts to stop the query. 
+	 * HiveSessionManager notices this and attempts to stop the query.
 	 */
-	protected void killIt(){
-		l4j.debug(this.getSessionName()+" Attempting kill.");
-		if ( this.runnable != null){
+	protected void killIt() {
+		l4j.debug(this.getSessionName() + " Attempting kill.");
+		if (this.runnable != null) {
 			try {
 				this.runnable.join(1000);
-				l4j.debug(this.getSessionName()+" Thread join complete");
+				l4j.debug(this.getSessionName() + " Thread join complete");
 			} catch (InterruptedException e) {
-				l4j.error(this.getSessionName()+" killing session caused exception ",e );
+				l4j.error(this.getSessionName() + " killing session caused exception ",
+						e);
 			}
 		}
 	}
+
 	/**
 	 * Helper function to get configuration variables
 	 * 
-	 * @param wanted a ConfVar
+	 * @param wanted
+	 *          a ConfVar
 	 * @return
 	 */
 	public String getHiveConfVar(HiveConf.ConfVars wanted) throws HWIException {
@@ -238,15 +208,37 @@
 		return result;
 	}
 
-	
 	@Override
-	/**
-	* The run method will be called by HiveSessionManager. This method
-	* reads the results from query processor completely when the query is done.
-	* when completed this is set to HWISessionItemStatus.QUERY_COMPLETE.
-	*/
+	/*
+	 * HWISessionItem uses a wait() notify() system. If the thread detects conf to
+	 * be null, control is transfered to initItem().A status of QUERY_SET causes
+	 * control to transfer to the runQuery() method. DESTROY will cause the run
+	 * loop to end permanently.
+	 */
 	public void run() {
-		l4j.debug(this.getSessionName()+" state is now QUERY_COMPLETE.");
+		synchronized (this.runnable) {
+			while (this.status != HWISessionItem.WebSessionItemStatus.DESTROY) {
+				if (conf == null) {
+					this.itemInit();
+				}
+				if (this.status == WebSessionItemStatus.QUERY_SET) {
+					this.runQuery();
+				}
+				try {
+					this.runnable.wait();
+				} catch (InterruptedException e) {
+					l4j.error("in wait() state ", e);
+				}
+			}
+		}
+	}
+
+	/**
+	 * This method calls the qp.run() method, writes the output to the result
+	 * file, when finished the status will be QUERY_COMPLETE.
+	 */
+	public void runQuery() {
+
 		FileOutputStream fos = null;
 
 		if (this.getResultFile() != null) {
@@ -254,34 +246,39 @@
 				fos = new FileOutputStream(new File(this.resultFile));
 				ss.out = new PrintStream(fos, true, "UTF-8");
 			} catch (java.io.FileNotFoundException fex) {
-				l4j.error(this.getSessionName()+" opening resultfile "+this.resultFile, fex);
+				l4j.error(this.getSessionName() + " opening resultfile "
+						+ this.resultFile, fex);
 			} catch (java.io.UnsupportedEncodingException uex) {
-				l4j.error(this.getSessionName()+" opening resultfile "+this.resultFile, uex);
+				l4j.error(this.getSessionName() + " opening resultfile "
+						+ this.resultFile, uex);
 			}
 		} else {
-			l4j.debug(this.getSessionName()+" Output file was not specified");
+			l4j.debug(this.getSessionName() + " Output file was not specified");
 		}
-		
+
+		l4j.debug(this.getSessionName() + " state is now QUERY_RUNNING.");
+		this.status = WebSessionItemStatus.QUERY_RUNNING;
+
 		queryRet = qp.run(this.query);
 		Vector<String> res = new Vector<String>();
 		while (qp.getResults(res)) {
 			for (String row : res) {
-				if (ss.out != null){
+				if (ss.out != null) {
 					ss.out.println(row);
 				}
 			}
 			res.clear();
 		}
 		try {
-			if (fos!= null){
+			if (fos != null) {
 				fos.close();
 			}
 		} catch (IOException ex) {
-			l4j.error(this.getSessionName()+" closing result file "+this.getResultFile()+" caused
exception." ,ex );
+			l4j.error(this.getSessionName() + " closing result file "
+					+ this.getResultFile() + " caused exception.", ex);
 		}
 		this.status = WebSessionItemStatus.QUERY_COMPLETE;
-		l4j.debug(this.getSessionName()+" state is now QUERY_COMPLETE.");
-
+		l4j.debug(this.getSessionName() + " state is now QUERY_COMPLETE.");
 	}
 
 	public int compareTo(HWISessionItem other) {
@@ -291,42 +288,65 @@
 	}
 
 	/**
+	 * 
+	 * @return the HiveHistoryViewer for the session
+	 * @throws HWIException
+	 */
+	public HiveHistoryViewer getHistoryViewer() throws HWIException {
+		if (ss == null)
+			throw new HWIException("Session state was null");
+		/*
+		 * we can not call this.ss.get().getHiveHistory().getHistFileName() directly
+		 * as this call is made from a a Jetty thread and will return null
+		 */
+		HiveHistoryViewer hv = new HiveHistoryViewer(this.historyFile);
+		return hv;
+	}
+
+	/**
 	 * Uses the sessionName property to compare to sessions
+	 * 
 	 * @return true if sessionNames are equal false otherwise
 	 */
-	public boolean equals(Object other){
+	public boolean equals(Object other) {
 		if (other == null)
 			return false;
-		if (! (other instanceof HWISessionItem))
+		if (!(other instanceof HWISessionItem))
 			return false;
 		HWISessionItem o = (HWISessionItem) other;
-		if (this.getSessionName().equals(o.getSessionName()) ){
+		if (this.getSessionName().equals(o.getSessionName())) {
 			return true;
 		} else {
 			return false;
 		}
 	}
-	
+
 	protected void setQp(Driver qp) {
 		this.qp = qp;
 	}
 
 	/**
 	 * The query executed by Hive
+	 * 
 	 * @return The query that this is executing or will be executed
 	 */
 	public String getQuery() {
 		return query;
 	}
+
 	/**
 	 * Use this function to set the query that Hive will run.
-	 * @param query A query in Hive Query Language
+	 * 
+	 * @param query
+	 *          A query in Hive Query Language
 	 */
 	public void setQuery(String query) {
 		this.query = query;
 	}
+
 	/**
 	 * Used to determine the status of a query, possibly why it failed
+	 * 
 	 * @return The result from Hive queryProcessor
 	 */
 	public int getQueryRet() {
@@ -344,9 +364,11 @@
 	public void setResultFile(String resultFile) {
 		this.resultFile = resultFile;
 	}
+
 	/**
 	 * The session name is an identifier to recognize the session
-	 * @return the sessions name
+	 * 
+	 * @return the session's name
 	 */
 	public String getSessionName() {
 		return sessionName;
@@ -357,7 +379,7 @@
 	}
 
 	protected SetProcessor getSp() {
-		    return sp;
+		return sp;
 	}
 
 	protected void setSp(SetProcessor sp) {
@@ -371,9 +393,12 @@
 	protected void setSs(CliSessionState ss) {
 		this.ss = ss;
 	}
+
 	/**
-	 * Used to represent to the user and other components what state the HWISessionItem 
-	 * is in. Certain commands can only be run when the application is in certain states.
+	 * Used to represent to the user and other components what state the
+	 * HWISessionItem is in. Certain commands can only be run when the application
+	 * is in certain states.
+	 * 
 	 * @return the current status of the session
 	 */
 	public WebSessionItemStatus getStatus() {
@@ -382,6 +407,7 @@
 
 	/**
 	 * Currently unused
+	 * 
 	 * @return a String with the full path to the error file.
 	 */
 	public String getErrorFile() {
@@ -390,11 +416,26 @@
 
 	/**
 	 * Currently unused
+	 * 
 	 * @param errorFile
-	 *            the full path to the file for results.
+	 *          the full path to the file for results.
 	 */
 	public void setErrorFile(String errorFile) {
 		this.errorFile = errorFile;
 	}
 
+	/**
+	 * @return the auth
+	 */
+	public HWIAuth getAuth() {
+		return auth;
+	}
+
+	/**
+	 * @param auth the auth to set
+	 */
+	protected void setAuth(HWIAuth auth) {
+		this.auth = auth;
+	}
+
 }

Modified: hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionManager.java?rev=758836&r1=758835&r2=758836&view=diff
==============================================================================
--- hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionManager.java (original)
+++ hadoop/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionManager.java Thu Mar
26 20:26:01 2009
@@ -15,24 +15,25 @@
  * 
  */
 public class HWISessionManager implements Runnable {
-	
-	protected static final Log l4j = LogFactory.getLog( HWISessionManager.class.getName() );
-	
+
+	protected static final Log l4j = LogFactory.getLog(HWISessionManager.class
+			.getName());
+
 	private boolean goOn;
-	private TreeMap<HWIAuth,Set<HWISessionItem>> items;
+	private TreeMap<HWIAuth, Set<HWISessionItem>> items;
 
 	protected HWISessionManager() {
 		goOn = true;
-		items = new TreeMap<HWIAuth,Set<HWISessionItem>>();
+		items = new TreeMap<HWIAuth, Set<HWISessionItem>>();
 	}
 
 	/**
 	 * This method scans the SessionItem collection. If a SessionItem is in the
 	 * QUERY_SET state that signals that its thread should be started. If the
-	 * SessionItem is in the DESTROY state it should be cleaned up and
-	 * removed from the collection. Currently we are using a sleep. A
-	 * wait/notify could be implemented. Queries will run for a long time, a one
-	 * second wait on start will not be noticed.
+	 * SessionItem is in the DESTROY state it should be cleaned up and removed
+	 * from the collection. Currently we are using a sleep. A wait/notify could be
+	 * implemented. Queries will run for a long time, a one second wait on start
+	 * will not be noticed.
 	 * 
 	 */
 	public void run() {
@@ -40,25 +41,21 @@
 		while (goOn) {
 			l4j.debug("locking items");
 			synchronized (items) {
-				
-				for (HWIAuth a: items.keySet() ){
+
+				for (HWIAuth a : items.keySet()) {
 					for (HWISessionItem i : items.get(a)) {
-						if (i.getStatus() == HWISessionItem.WebSessionItemStatus.QUERY_SET) {
-							l4j.debug("Starting item: "+i.getSessionName());
-							i.startIt();
-						}
 						if (i.getStatus() == HWISessionItem.WebSessionItemStatus.DESTROY) {
 							items.get(a).remove(i);
 						}
-						if (i.getStatus() == HWISessionItem.WebSessionItemStatus.KILL_QUERY){
-							l4j.debug("Killing item: "+i.getSessionName());
+						if (i.getStatus() == HWISessionItem.WebSessionItemStatus.KILL_QUERY) {
+							l4j.debug("Killing item: " + i.getSessionName());
 							i.killIt();
-							l4j.debug("Killed item: "+i.getSessionName());
+							l4j.debug("Killed item: " + i.getSessionName());
 							items.get(a).remove(i);
 						}
 					}
 				}
-				
+
 			} // end sync
 			try {
 				Thread.sleep(100);
@@ -69,16 +66,16 @@
 		l4j.debug("goOn is false. Loop has ended.");
 		// Cleanup used here to stop all threads
 		synchronized (items) {
-			for (HWIAuth a: items.keySet() ){
+			for (HWIAuth a : items.keySet()) {
 				for (HWISessionItem i : items.get(a)) {
 					try {
-						if (i.getStatus()==HWISessionItem.WebSessionItemStatus.QUERY_RUNNING){
-							l4j.debug(i.getSessionName() +"Joining ");
+						if (i.getStatus() == HWISessionItem.WebSessionItemStatus.QUERY_RUNNING) {
+							l4j.debug(i.getSessionName() + "Joining ");
 							i.runnable.join(1000);
-							l4j.debug(i.getSessionName() +"Joined ");
+							l4j.debug(i.getSessionName() + "Joined ");
 						}
 					} catch (InterruptedException ex) {
-						l4j.error(i.getSessionName() +"while joining ",ex);
+						l4j.error(i.getSessionName() + "while joining ", ex);
 					}
 				}
 			}
@@ -93,101 +90,115 @@
 		this.goOn = goOn;
 	}
 
-	protected TreeMap<HWIAuth,Set<HWISessionItem>> getItems() {
+	protected TreeMap<HWIAuth, Set<HWISessionItem>> getItems() {
 		return items;
 	}
 
-	protected void setItems(TreeMap <HWIAuth,Set<HWISessionItem>> items) {
+	protected void setItems(TreeMap<HWIAuth, Set<HWISessionItem>> items) {
 		this.items = items;
 	}
 
 	// client methods called from JSP
 	/**
-	 * Rather then return the actual items we return a list copies. This
-	 * enforces our HWISessionManager by preventing the ability of the
-	 * client(jsp) to create SessionItems.
+	 * Rather then return the actual items we return a list copies. This enforces
+	 * our HWISessionManager by preventing the ability of the client(jsp) to
+	 * create SessionItems.
 	 * 
 	 * @return A set of SessionItems this framework manages
 	 */
 	public Vector<HWISessionItem> findAllSessionItems() {
 		Vector<HWISessionItem> otherItems = new Vector<HWISessionItem>();
-		for (HWIAuth a:items.keySet()){
+		for (HWIAuth a : items.keySet()) {
 			otherItems.addAll(items.get(a));
 		}
 		return otherItems;
 	}
+
+	/**
+	 * Here we handle creating the SessionItem, we do this for the JSP client
+	 * because we need to set parameters the client is not aware of. One such
+	 * parameter is the command line arguments the server was started with.
+	 * 
+	 * @param sessionname
+	 *          Represents the session name
+	 * @param sessionpass
+	 *          the session password, can be ""
+	 * @return a new SessionItem or null if a session with that name already
+	 *         exists
+	 */
+	public HWISessionItem createSession(HWIAuth a, String sessionName) {
+
+		l4j.debug("Creating session: " + sessionName);
+
+		HWISessionItem si = null;
+
+		synchronized (this.items) {
+			if (findSessionItemByName(a, sessionName) == null) {
+				l4j.debug("Initializing session: " + sessionName + " a for "
+						+ a.getUser());
+				si = new HWISessionItem();
+				si.setSessionName(sessionName);
+				si.setAuth(a);
+
+				if (!items.containsKey(a)) {
+					l4j.debug("SessionList is empty " + a.getUser());
+					TreeSet<HWISessionItem> list = new TreeSet<HWISessionItem>();
+					list.add(si);
+					items.put(a, list);
+					l4j.debug("Item added " + si.getSessionName() + " for user "
+							+ a.getUser());
+				} else {
+					items.get(a).add(si);
+					l4j.debug("Item added " + si.getSessionName() + " for user "
+							+ a.getUser());
+				}
+
+			} else {
+				l4j.debug("Creating session: " + sessionName + " already exists "
+						+ a.getUser());
+			}
+		}
+		return si;
+	}
+
 	/**
-	   * Here we handle creating the SessionItem, we do this for the JSP client
-	   * because we need to set parameters the client is not aware of. One such 
-	   * parameter is the command line arguments the server was started with.
-	   * @param sessionname Represents the session name
-	   * @param sessionpass the session password, can be ""
-	   * @return a new SessionItem or null if a session with that name already exists
-	   */
-	  public HWISessionItem createSession( HWIAuth a, String sessionName ){
-
-		l4j.debug("Creating session: "+sessionName);
-		
-	    HWISessionItem si=null;
-	    
-	    synchronized ( this.items) {
-	      if ( findSessionItemByName(a,sessionName) == null){
-	    	l4j.debug("Initializing session: "+sessionName+" already exists for "+a.getUser());
-	        si = new HWISessionItem();
-	        si.setSessionName(sessionName);
-	        si.itemInit();
-	        si.runSetProcessorQuery("hadoop.job.ugi="+a.getUser()+","+a.getGroups()[0]);
-	        si.runSetProcessorQuery("user.name="+a.getUser() );
-	        if (! items.containsKey(a)){
-	        	l4j.debug("SessionList is empty " +a.getUser());
-	        	TreeSet<HWISessionItem> list=  new TreeSet<HWISessionItem>();
-	        	list.add(si);
-	        	items.put(a,list );
-	        	l4j.debug("Item added "+si.getSessionName()+" for user " +a.getUser());
-	        } else{
-	        	items.get(a).add(si);
-	        	l4j.debug("Item added "+si.getSessionName()+" for user " +a.getUser());
-	        }
-	        
-	      } else {
-	    	  l4j.debug("Creating session: "+sessionName+" already exists " +a.getUser());
-	      }
-	    }
-	    return si;
-	  }
-	  
-	  /**
-	   * Helper method useful when you know the session name you wish to reference.
-	   * @param sessionname
-	   * @return A SessionItem matching the sessionname or null if it does not exists
-	   */
-	  public HWISessionItem findSessionItemByName(HWIAuth auth,String sessionname){
-	    Collection<HWISessionItem> sessForUser=  this.items.get(auth);
-	    if (sessForUser==null){
-	    	return null;
-	    }
-		for (HWISessionItem si : sessForUser ){
-	      if (si.getSessionName().equals(sessionname)){
-	        return si;
-	      }
-	    }
-	    return null;
-	  }
-
-	  /**
-	   * Used to list all users that have at least one session
-	   * @return keySet of items all users that have any sessions
-	   */
-	  public Set <HWIAuth> findAllUsersWithSessions() {
-		  return items.keySet();
-	  }
-	  /**
-	   * Used to list all the sessions of a user
-	   * @param auth the user being enquired about
-	   * @return all the sessions of that user
-	   */
-	  public Set<HWISessionItem> findAllSessionsForUser(HWIAuth auth) {
-		  return this.items.get(auth);
-	  }
-	  
+	 * Helper method useful when you know the session name you wish to reference.
+	 * 
+	 * @param sessionname
+	 * @return A SessionItem matching the sessionname or null if it does not
+	 *         exists
+	 */
+	public HWISessionItem findSessionItemByName(HWIAuth auth, String sessionname) {
+		Collection<HWISessionItem> sessForUser = this.items.get(auth);
+		if (sessForUser == null) {
+			return null;
+		}
+		for (HWISessionItem si : sessForUser) {
+			if (si.getSessionName().equals(sessionname)) {
+				return si;
+			}
+		}
+		return null;
+	}
+
+	/**
+	 * Used to list all users that have at least one session
+	 * 
+	 * @return keySet of items all users that have any sessions
+	 */
+	public Set<HWIAuth> findAllUsersWithSessions() {
+		return items.keySet();
+	}
+
+	/**
+	 * Used to list all the sessions of a user
+	 * 
+	 * @param auth
+	 *          the user being enquired about
+	 * @return all the sessions of that user
+	 */
+	public Set<HWISessionItem> findAllSessionsForUser(HWIAuth auth) {
+		return this.items.get(auth);
+	}
+
 }
\ No newline at end of file

Modified: hadoop/hive/trunk/hwi/src/test/org/apache/hadoop/hive/hwi/TestHWISessionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hwi/src/test/org/apache/hadoop/hive/hwi/TestHWISessionManager.java?rev=758836&r1=758835&r2=758836&view=diff
==============================================================================
--- hadoop/hive/trunk/hwi/src/test/org/apache/hadoop/hive/hwi/TestHWISessionManager.java (original)
+++ hadoop/hive/trunk/hwi/src/test/org/apache/hadoop/hive/hwi/TestHWISessionManager.java Thu
Mar 26 20:26:01 2009
@@ -1,4 +1,8 @@
 package org.apache.hadoop.hive.hwi;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
 import junit.framework.TestCase;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.StringUtils;
@@ -6,94 +10,171 @@
 import org.apache.hadoop.hive.hwi.HWIAuth;
 import org.apache.hadoop.hive.hwi.HWISessionItem;
 import org.apache.hadoop.hive.hwi.HWISessionManager;
+import org.apache.hadoop.hive.ql.history.HiveHistoryViewer;
 
 public class TestHWISessionManager extends TestCase {
 
-  private static String tableName="test_hwi_table";
-  
-  private HiveConf conf;
-  private Path dataFilePath;
-  private HWISessionManager hsm; 
-
-  public TestHWISessionManager(String name){
-    super(name);
-    conf = new HiveConf(TestHWISessionManager.class);
-    String dataFileDir = conf.get("test.data.files").replace('\\', '/').replace("c:", "");
-    dataFilePath = new Path(dataFileDir, "kv1.txt");
-  }
-
-  protected void setUp() throws Exception {
-    super.setUp();
-    hsm = new HWISessionManager();
-    Thread t = new Thread(hsm);
-    t.start();
-  }
-  
-  protected void tearDown() throws Exception {
-    super.tearDown();
-	hsm.setGoOn(false);
-  }
-  
-  public final void testHiveDriver() throws Exception {
-	  //create a user
-	  HWIAuth user1= new HWIAuth();
-	  user1.setUser("hadoop");
-	  user1.setGroups(new String[]{ "hadoop"});
-	  
-	  //create two sessions for user
-	  HWISessionItem user1_item1 = hsm.createSession(user1, "session1");
-	  HWISessionItem user1_item2 = hsm.createSession(user1, "session2");
-	  
-	  //create second user
-	  HWIAuth user2= new HWIAuth();
-	  user2.setUser("user2");
-	  user2.setGroups(new String[]{ "user2"});
-	  
-	  //create one session for this user
-	  HWISessionItem user2_item1 =hsm.createSession(user2, "session1");
-	  
-	  assertEquals( hsm.findAllSessionsForUser(user1).size(),2 );
-	  assertEquals( hsm.findAllSessionsForUser(user2).size(),1 );
-	  assertEquals( hsm.findAllSessionItems().size(),3);
-	
-	  HWISessionItem searchItem =hsm.findSessionItemByName(user1, "session1");
-	  assertEquals( searchItem, user1_item1);
-	  
-	  searchItem.setQuery("create table " + tableName + " (key int, value string)");
-	  searchItem.clientStart();
-	  
-	  //wait for the session manager to make the table it is non blocking api
-	  while (searchItem.getStatus() != HWISessionItem.WebSessionItemStatus.QUERY_COMPLETE){
-		  Thread.sleep(1);
-	  }
-	  assertEquals(searchItem.getQueryRet(),0 );
-	  
-	  //start two queries simultaniously
-	  user1_item2.setQuery("select distinct(test_hwi_table.key) from "+tableName);
-	  user2_item1.setQuery("select distinct(test_hwi_table.key) from "+tableName);
-	  
-	  user1_item2.clientStart();
-	  user2_item1.clientStart();
-	  
-	  
-	  while (user1_item2.getStatus() != HWISessionItem.WebSessionItemStatus.QUERY_COMPLETE ){

-		  Thread.sleep(1);
-	  }
-	  while (user2_item1.getStatus() != HWISessionItem.WebSessionItemStatus.QUERY_COMPLETE )
{
-		  Thread.sleep(1);
-	  }
-	  
-	  assertEquals(user1_item2.getQueryRet(),0);
-	  assertEquals(user2_item1.getQueryRet(),0);
-
-          HWISessionItem cleanup = hsm.createSession(user1, "cleanup");
-          cleanup.setQuery("drop table " + tableName);
-          cleanup.clientStart();
-
-          while (cleanup.getStatus() != HWISessionItem.WebSessionItemStatus.QUERY_COMPLETE
){
-                  Thread.sleep(1);
-          }
+	private static String tableName = "test_hwi_table";
 
-	  assertEquals(cleanup.getQueryRet(),0);
-  }
+	private HiveConf conf;
+	private Path dataFilePath;
+	private HWISessionManager hsm;
+
+	public TestHWISessionManager(String name) {
+		super(name);
+		conf = new HiveConf(TestHWISessionManager.class);
+		String dataFileDir = conf.get("test.data.files").replace('\\', '/')
+		.replace("c:", "");
+		dataFilePath = new Path(dataFileDir, "kv1.txt");
+	}
+
+	protected void setUp() throws Exception {
+		super.setUp();
+		hsm = new HWISessionManager();
+		Thread t = new Thread(hsm);
+		t.start();
+	}
+
+	protected void tearDown() throws Exception {
+		super.tearDown();
+		hsm.setGoOn(false);
+	}
+
+	public final void testHiveDriver() throws Exception {
+		// create a user
+		HWIAuth user1 = new HWIAuth();
+		user1.setUser("hadoop");
+		user1.setGroups(new String[] { "hadoop" });
+
+		// create two sessions for user
+		HWISessionItem user1_item1 = hsm.createSession(user1, "session1");
+		HWISessionItem user1_item2 = hsm.createSession(user1, "session2");
+
+		// create second user
+		HWIAuth user2 = new HWIAuth();
+		user2.setUser("user2");
+		user2.setGroups(new String[] { "user2" });
+
+		// create one session for this user
+		HWISessionItem user2_item1 = hsm.createSession(user2, "session1");
+
+		// testing storage of sessions in HWISessionManager
+		assertEquals(hsm.findAllSessionsForUser(user1).size(), 2);
+		assertEquals(hsm.findAllSessionsForUser(user2).size(), 1);
+		assertEquals(hsm.findAllSessionItems().size(), 3);
+
+		HWISessionItem searchItem = hsm.findSessionItemByName(user1, "session1");
+		assertEquals(searchItem, user1_item1);
+
+		searchItem.setQuery("create table " + tableName
+				+ " (key int, value string)");
+		searchItem.clientStart();
+
+		// wait for the session manager to make the table. It is non blocking API.
+		while (searchItem.getStatus() != HWISessionItem.WebSessionItemStatus.QUERY_COMPLETE) {
+			Thread.sleep(1);
+		}
+		assertEquals(searchItem.getQueryRet(), 0);
+
+		// load data into table
+		searchItem.clientRenew();
+		searchItem.setQuery(("load data local inpath '" + dataFilePath.toString()
+				+ "' into table " + tableName));
+		searchItem.clientStart();
+		while (searchItem.getStatus() != HWISessionItem.WebSessionItemStatus.QUERY_COMPLETE) {
+			Thread.sleep(1);
+		}
+		assertEquals(searchItem.getQueryRet(), 0);
+
+		// start two queries simultaniously
+		user1_item2.setQuery("select distinct(test_hwi_table.key) from "
+				+ tableName);
+		user2_item1.setQuery("select distinct(test_hwi_table.key) from "
+				+ tableName);
+
+		// set result files to compare results
+		File tmpdir = new File("/tmp/" + System.getProperty("user.name") + "/");
+		if (tmpdir.exists() && !tmpdir.isDirectory()) {
+			throw new RuntimeException(tmpdir + " exists but is not a directory");
+		}
+
+		if (!tmpdir.exists()) {
+			if (!tmpdir.mkdirs()) {
+				throw new RuntimeException("Could not make scratch directory " + tmpdir);
+			}
+		}
+
+		File result1 = new File(tmpdir, "user1_item2");
+		File result2 = new File(tmpdir, "user2_item1");
+		user1_item2.setResultFile(result1.toString());
+		user2_item1.setResultFile(result2.toString());
+		user1_item2.setSSIsSilent(true);
+		user2_item1.setSSIsSilent(true);
+
+		user1_item2.clientStart();
+		user2_item1.clientStart();
+
+		while (user1_item2.getStatus() != HWISessionItem.WebSessionItemStatus.QUERY_COMPLETE) {
+			Thread.sleep(1);
+		}
+		while (user2_item1.getStatus() != HWISessionItem.WebSessionItemStatus.QUERY_COMPLETE) {
+			Thread.sleep(1);
+		}
+
+		assertEquals(user1_item2.getQueryRet(), 0);
+		assertEquals(user2_item1.getQueryRet(), 0);
+		assertEquals(isFileContentEqual(result1, result2), true);
+
+		// clean up the files
+		result1.delete();
+		result2.delete();
+
+		// test a session renew/refresh
+		user2_item1.clientRenew();
+		user2_item1.setQuery("select distinct(test_hwi_table.key) from "
+				+ tableName);
+		user2_item1.clientStart();
+		while (user2_item1.getStatus() != HWISessionItem.WebSessionItemStatus.QUERY_COMPLETE) {
+			Thread.sleep(1);
+		}
+
+		// cleanup
+		HWISessionItem cleanup = hsm.createSession(user1, "cleanup");
+		cleanup.setQuery("drop table " + tableName);
+		cleanup.clientStart();
+
+		while (cleanup.getStatus() != HWISessionItem.WebSessionItemStatus.QUERY_COMPLETE) {
+			Thread.sleep(1);
+		}
+
+		// test the history is non null object.
+		HiveHistoryViewer hhv = cleanup.getHistoryViewer();
+		assertNotNull(hhv);
+		assertEquals(cleanup.getQueryRet(), 0);
+	}
+
+	public boolean isFileContentEqual(File one, File two) throws Exception {
+		if (one.exists() && two.exists()) {
+			if (one.isFile() && two.isFile()) {
+				if (one.length() == two.length()) {
+					BufferedReader br1 = new BufferedReader(new FileReader(one));
+					BufferedReader br2 = new BufferedReader(new FileReader(one));
+					String line1 = null;
+					String line2 = null;
+					while ((line1 = br1.readLine()) != null) {
+						line2 = br2.readLine();
+						if (!line1.equals(line2)) {
+							br1.close();
+							br2.close();
+							return false;
+						}
+					}
+					br1.close();
+					br2.close();
+					return true;
+				}
+			}
+		}
+		return false;
+	}
 }

Added: hadoop/hive/trunk/hwi/web/session_history.jsp
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hwi/web/session_history.jsp?rev=758836&view=auto
==============================================================================
--- hadoop/hive/trunk/hwi/web/session_history.jsp (added)
+++ hadoop/hive/trunk/hwi/web/session_history.jsp Thu Mar 26 20:26:01 2009
@@ -0,0 +1,57 @@
+<%@page contentType="text/html" pageEncoding="UTF-8"%>
+<%@page errorPage="error_page.jsp"%>
+<%@page import="org.apache.hadoop.hive.hwi.*"%>
+<%@page import="org.apache.hadoop.hive.conf.*"%>
+<%@page import="org.apache.hadoop.hive.ql.history.*"%>
+<%@page import="org.apache.hadoop.hive.ql.history.HiveHistory.*"%>
+<%@page import="java.util.*"%>
+
+<% HWISessionManager hs = (HWISessionManager) application.getAttribute("hs"); %>
+<% if (hs == null) { %>
+  <jsp:forward page="error.jsp">
+	<jsp:param name="message" value="Hive Session Manager Not Found" />
+  </jsp:forward>
+<% } %>
+
+<% HWIAuth auth = (HWIAuth) session.getAttribute("auth"); %>
+<% if (auth==null) { %>
+  <jsp:forward page="/authorize.jsp" />
+<% } %>
+<% String sessionName = request.getParameter("sessionName"); %>
+<% HWISessionItem si = hs.findSessionItemByName(auth,sessionName); %>
+<% HiveHistoryViewer hv = si.getHistoryViewer(); %>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <title>Session History</title>
+  </head>
+  <body>
+    <table>
+	  <tr>
+		<td valign="top"><jsp:include page="left_navigation.jsp" /></td>
+		<td valign="top">
+		<h2>Hive History</h2>
+		SessionID: <%= hv.getSessionId() %><br>
+		<% for (String jobKey: hv.getJobInfoMap().keySet() ){ %> <%=jobKey%><br>
+		<ul>
+			<% QueryInfo qi = hv.getJobInfoMap().get(jobKey); %>
+			<% for (String qiKey: qi.hm.keySet() ){ %>
+			<li><%=qiKey%> <%=qi.hm.get(qiKey) %></li>
+			<% } %>
+		</ul>
+		<% } %>
+		<br>
+		<br>
+		<% for (String taskKey: hv.getTaskInfoMap().keySet() ){ %> <%=taskKey%><br>
+		<ul>
+			<% TaskInfo ti = hv.getTaskInfoMap().get(taskKey); %>
+			<% for (String tiKey: ti.hm.keySet() ) { %>
+			<li><%=tiKey%> <%=ti.hm.get(tiKey)%></li>
+			<% } %>
+		</ul>
+		<% } %>
+		</td>
+	  </tr>
+    </table>
+  </body>
+</html>
\ No newline at end of file

Modified: hadoop/hive/trunk/hwi/web/session_manage.jsp
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hwi/web/session_manage.jsp?rev=758836&r1=758835&r2=758836&view=diff
==============================================================================
--- hadoop/hive/trunk/hwi/web/session_manage.jsp (original)
+++ hadoop/hive/trunk/hwi/web/session_manage.jsp Thu Mar 26 20:26:01 2009
@@ -70,7 +70,7 @@
           	--%>
           <% } %>
           
-         
+          Session History:  <a href="/hwi/session_history.jsp?sessionName=<%=sessionName%>"><%=sessionName%></a><br>

           Session Diagnostics: <a href="/hwi/session_diagnostics.jsp?sessionName=<%=sessionName%>"><%=sessionName%></a><br>
 		  Set Processor: <a href="/hwi/set_processor.jsp?sessionName=<%=sessionName%>"><%=sessionName%></a><br>

           Session Remove: <a href="/hwi/session_remove.jsp?sessionName=<%=sessionName%>"><%=sessionName%></a><br>

@@ -147,4 +147,4 @@
       </tr>
     </table>
   </body>
-</html>
\ No newline at end of file
+</html>



Mime
View raw message