hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zs...@apache.org
Subject svn commit: r724613 - in /hadoop/hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/parse/
Date Tue, 09 Dec 2008 05:46:23 GMT
Author: zshao
Date: Mon Dec  8 21:46:23 2008
New Revision: 724613

URL: http://svn.apache.org/viewvc?rev=724613&view=rev
Log:
HIVE-140. Event Based Infrastructure for Syntax Trees in the compiler.
(Ashish Thusoo through zshao)

Added:
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTEventDispatcher.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTProcessor.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
Modified:
    hadoop/hive/trunk/CHANGES.txt

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=724613&r1=724612&r2=724613&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Mon Dec  8 21:46:23 2008
@@ -6,6 +6,9 @@
 
   NEW FEATURES
 
+    HIVE-140. Event Based Infrastructure for Syntax Trees in the compiler.
+    (Ashish Thusoo through zshao)
+
     HIVE-98. Dependency management with hadoop core using ivy.
     (Ashish Thusoo through zshao)
 

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java?rev=724613&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java Mon Dec  8
21:46:23 2008
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+/**
+ * Enumeration that encapsulates the various event types that are seen
+ * while processing the parse tree (in an implementation of the ParseTreeProcessor).
+ * These event types are used to register the different event processors with
+ * the parse tree processor.
+ *
+ */
+public enum ASTEvent {
+	
+	/**
+	 * Query event
+	 */
+	QUERY("QUERY"),
+	
+	/**
+	 * Union
+	 */
+	UNION("UNION"),
+	
+	/**
+	 * Source Table (table in the from clause)
+	 */
+	SRC_TABLE("SRC_TABLE"),
+
+	/**
+	 * Any type of Destination (this fires for hdfs directory, local directory and table)
+	 */
+	DESTINATION("DESTINATION"),
+	
+	/**
+	 * Select clause
+	 */
+	SELECT_CLAUSE("SELECT_CLAUSE"),
+	
+	/**
+	 * Join clause
+	 */
+	JOIN_CLAUSE("JOIN_CLAUSE"),
+	
+	/**
+	 * Where clause
+	 */
+	WHERE_CLAUSE("WHERE_CLAUSE"),
+	
+	/**
+	 * CLusterby clause
+	 */
+	CLUSTERBY_CLAUSE("CLUSTERBY_CLAUSE"),
+	
+	/**
+	 * Group by clause
+	 */
+	GROUPBY_CLAUSE("GROUPBY_CLAUSE"),
+	
+	/**
+	 * Limit clause
+	 */
+	LIMIT_CLAUSE("LIMIT_CLAUSE"),
+	
+	/**
+	 * Subquery
+	 */
+	SUBQUERY("SUBQUERY");
+	
+	/**
+	 * The name of the event (string representation of the event)
+	 */
+	private final String name;
+	
+	/**
+	 * Constructs the event
+	 * 
+	 * @param name The name(String representation of the event)
+	 */
+	ASTEvent(String name) {
+      this.name = name;
+	}
+	
+	/**
+	 * String representation of the event
+	 * 
+	 * @return String
+	 */
+	@Override
+	public String toString() {
+		return name;
+	}
+}

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java?rev=724613&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java
(added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java
Mon Dec  8 21:46:23 2008
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import org.antlr.runtime.tree.CommonTree;
+
+/**
+ * Dispatches ParseTreeEvent to the appropriate ParseTreeEventProcessor
+ */
+public interface ASTEventDispatcher {
+	
+	/**
+	 * Registers the event processor with the event
+	 * 
+	 * @param evt The parse tree event
+	 * @param evt_p The associated parse tree event processor
+	 */
+	void register(ASTEvent evt, ASTEventProcessor evt_p);
+	
+	/**
+	 * Dispatches the parse tree event to a registered event processor
+	 * 
+	 * @param evt The parse tree event to dispatch
+	 * @param pt The parse subtree to dispatch to the event processor
+	 */
+	void dispatch(ASTEvent evt, CommonTree pt);
+}

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java?rev=724613&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java Mon
Dec  8 21:46:23 2008
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import org.antlr.runtime.tree.CommonTree;
+
+/**
+ * Interface that a parse tree event processor needs to implement. Classes implementing
+ * this interface and registered with the ParseTreeProcessor are called by the later
+ * when associated ParseTreeEvents are processed
+ *
+ */
+public interface ASTEventProcessor {
+
+  /**
+   * Processes the parse subtree corresponding to the event
+   * 
+   * @param pt The parse subtree to process
+   */
+  public void process(CommonTree pt);
+}

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java?rev=724613&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java Mon Dec
 8 21:46:23 2008
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import org.antlr.runtime.tree.CommonTree;
+
+/**
+ * Interface that a parse tree processor needs to implement
+ */
+public interface ASTProcessor {
+  
+	/**
+	 * Sets the event dispatcher for the processors
+	 * 
+	 * @param dispatcher The parse tree event dispatcher
+	 */
+	void setDispatcher(ASTEventDispatcher dispatcher);
+	
+	/**
+	 * Processes the parse tree and calls the registered event processors
+	 * for the associated parse tree events
+	 * 
+	 * @param pt The parse tree to process
+	 */
+	void process(CommonTree pt);
+}
+

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTEventDispatcher.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTEventDispatcher.java?rev=724613&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTEventDispatcher.java
(added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTEventDispatcher.java
Mon Dec  8 21:46:23 2008
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.util.HashMap;
+import java.util.ArrayList;
+
+import org.antlr.runtime.tree.CommonTree;
+
+/**
+ * Implementation of a default ParseTreeEventDispatcher. This dispatcher calls
+ * the associated ParseTreeEventProcessors in the order in which they were
+ * registered for the event
+ *
+ */
+public class DefaultASTEventDispatcher implements
+		ASTEventDispatcher {
+	
+	/**
+	 * Stores the mapping from the ParseTreeEvent to the list of ParseTreeEventProcessors.
+	 * The later are stored in the order that they were registered.
+	 */
+	private HashMap<ASTEvent, ArrayList<ASTEventProcessor>> dispatchMap;
+
+	/**
+	 * Constructs the default event dispatcher
+	 */
+	DefaultASTEventDispatcher() {
+		dispatchMap = new HashMap<ASTEvent, ArrayList<ASTEventProcessor>>();
+	}
+	
+	/**
+	 * Dispatches the parse subtree to all the event processors registered for the
+	 * event in the order that they were registered.
+	 * 
+	 * @see org.apache.hadoop.hive.ql.parse.ASTEventDispatcher#dispatch(org.apache.hadoop.hive.ql.parse.ASTEvent,
org.antlr.runtime.tree.CommonTree)
+	 */
+	@Override
+	public void dispatch(ASTEvent evt, CommonTree pt) {
+		
+		ArrayList<ASTEventProcessor> evtp_l = dispatchMap.get(evt);
+		if (evtp_l == null) {
+			return;
+		}
+
+		for(ASTEventProcessor evt_p: evtp_l) {
+			// Do the actual dispatch
+			evt_p.process(pt);
+		}
+	}
+
+	/**
+	 * Registers the event processor for the event.
+	 * 
+	 * @see org.apache.hadoop.hive.ql.parse.ASTEventDispatcher#register(org.apache.hadoop.hive.ql.parse.ASTEvent,
org.apache.hadoop.hive.ql.parse.ASTEventProcessor)
+	 */
+	@Override
+	public void register(ASTEvent evt, ASTEventProcessor evt_p) {
+
+		ArrayList<ASTEventProcessor> evtp_l = dispatchMap.get(evt);
+		if (evtp_l == null) {
+			evtp_l = new ArrayList<ASTEventProcessor>();
+			dispatchMap.put(evt, evtp_l);
+		}
+
+		evtp_l.add(evt_p);
+	}
+}

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTProcessor.java?rev=724613&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTProcessor.java
(added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTProcessor.java
Mon Dec  8 21:46:23 2008
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import org.antlr.runtime.tree.CommonTree;
+
+/**
+ * Implementation of a parse tree processor. This processor does a depth first walk
+ * of the parse tree and calls the associated event processors
+ */
+public class DefaultASTProcessor implements ASTProcessor {
+
+	/**
+	 * The dispatcher used to dispatch ParseTreeEvents to the ParseTreeEventProcessors
+	 */
+	private ASTEventDispatcher dispatcher;
+	
+	/**
+	 * Processes the parse tree
+	 * 
+	 * @see org.apache.hadoop.hive.ql.parse.ASTProcessor#process(org.antlr.runtime.tree.CommonTree)
+	 */
+	@Override
+	public void process(CommonTree ast) {
+
+		// Base case
+		if (ast.getToken() == null) {
+			return;
+		}
+
+		switch (ast.getToken().getType()) {
+		case HiveParser.TOK_SELECTDI:
+		case HiveParser.TOK_SELECT:
+			dispatcher.dispatch(ASTEvent.SELECT_CLAUSE, ast);
+			break;
+
+		case HiveParser.TOK_WHERE:
+			dispatcher.dispatch(ASTEvent.WHERE_CLAUSE, ast);
+			break;
+
+		case HiveParser.TOK_DESTINATION:
+			dispatcher.dispatch(ASTEvent.DESTINATION, ast);
+			break;
+
+		case HiveParser.TOK_FROM:
+
+			// Check if this is a subquery
+			CommonTree frm = (CommonTree) ast.getChild(0);
+			if (frm.getToken().getType() == HiveParser.TOK_TABREF) {
+				dispatcher.dispatch(ASTEvent.SRC_TABLE, ast);
+			} else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) {
+				dispatcher.dispatch(ASTEvent.SUBQUERY, ast);
+			} else if (ParseUtils.isJoinToken(frm)) {
+				dispatcher.dispatch(ASTEvent.JOIN_CLAUSE, ast);
+			}
+			break;
+
+		case HiveParser.TOK_CLUSTERBY:
+			dispatcher.dispatch(ASTEvent.CLUSTERBY_CLAUSE, ast);
+			break;
+
+		case HiveParser.TOK_GROUPBY:
+			dispatcher.dispatch(ASTEvent.GROUPBY_CLAUSE, ast);
+			break;
+
+		case HiveParser.TOK_LIMIT:
+			dispatcher.dispatch(ASTEvent.LIMIT_CLAUSE, ast);
+			break;
+		default:
+			break;
+		}
+
+		// Iterate over the rest of the children
+		int child_count = ast.getChildCount();
+		for (int child_pos = 0; child_pos < child_count; ++child_pos) {
+			// Recurse
+			process((CommonTree) ast.getChild(child_pos));
+		}
+	}
+
+	/**
+	 * Sets the dispatcher for the parse tree processor
+	 * 
+	 * @see org.apache.hadoop.hive.ql.parse.ASTProcessor#register(org.apache.hadoop.hive.ql.parse.ASTEvent,
org.apache.hadoop.hive.ql.parse.ParseTreeEventProcessor)
+	 */
+	@Override
+	public void setDispatcher(ASTEventDispatcher dispatcher) {
+		
+		this.dispatcher = dispatcher;
+	}
+
+}

Added: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java?rev=724613&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java (added)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java Mon Dec
 8 21:46:23 2008
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import org.antlr.runtime.tree.CommonTree;
+
+/**
+ * Library of utility functions used in the parse code
+ *
+ */
+public class ParseUtils {
+	
+	/**
+	 * Tests whether the parse tree node is a join token
+	 * 
+	 * @param node The parse tree node
+	 * @return boolean
+	 */
+	public static boolean isJoinToken(CommonTree node) {
+		if ((node.getToken().getType() == HiveParser.TOK_JOIN)
+				|| (node.getToken().getType() == HiveParser.TOK_LEFTOUTERJOIN)
+				|| (node.getToken().getType() == HiveParser.TOK_RIGHTOUTERJOIN)
+				|| (node.getToken().getType() == HiveParser.TOK_FULLOUTERJOIN))
+			return true;
+
+		return false;
+	}
+}



Mime
View raw message