hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r654308 - in /hadoop/core/trunk: ./ src/java/org/apache/hadoop/mapred/join/ src/test/org/apache/hadoop/mapred/join/
Date Wed, 07 May 2008 22:36:14 GMT
Author: omalley
Date: Wed May  7 15:36:13 2008
New Revision: 654308

URL: http://svn.apache.org/viewvc?rev=654308&view=rev
Log:
HADOOP-3299. CompositeInputFormat should configure the sub-input
formats. Contributed by Chris Douglas.

Added:
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/join/ConfigurableInputFormat.java
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/mapred/join/CompositeInputFormat.java
    hadoop/core/trunk/src/java/org/apache/hadoop/mapred/join/Parser.java
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/join/TestDatamerge.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=654308&r1=654307&r2=654308&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed May  7 15:36:13 2008
@@ -194,6 +194,9 @@
     HADOOP-3085. Catch Exception in metrics util classes to ensure that
     misconfigured metrics don't prevent others from updating. (cdouglas)
 
+    HADOOP-3299. CompositeInputFormat should configure the sub-input
+    formats. (cdouglas via omalley)
+
 Release 0.17.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/join/CompositeInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/join/CompositeInputFormat.java?rev=654308&r1=654307&r2=654308&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/join/CompositeInputFormat.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/join/CompositeInputFormat.java Wed
May  7 15:36:13 2008
@@ -72,9 +72,7 @@
   public void setFormat(JobConf job) throws IOException {
     addDefaults();
     addUserIdentifiers(job);
-    Class<? extends WritableComparator> cmpcl =
-      job.getClass("mapred.join.keycomparator", null, WritableComparator.class);
-    root = Parser.parse(job.get("mapred.join.expr", null), cmpcl);
+    root = Parser.parse(job.get("mapred.join.expr", null), job);
   }
 
   /**

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/join/Parser.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/join/Parser.java?rev=654308&r1=654307&r2=654308&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/join/Parser.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/join/Parser.java Wed May  7 15:36:13
2008
@@ -233,7 +233,7 @@
     protected void setKeyComparator(Class<? extends WritableComparator> cmpcl) {
       this.cmpcl = cmpcl;
     }
-    abstract void parse(List<Token> args) throws IOException;
+    abstract void parse(List<Token> args, JobConf job) throws IOException;
   }
 
   /**
@@ -260,7 +260,7 @@
      * Let the first actual define the InputFormat and the second define
      * the <tt>mapred.input.dir</tt> property.
      */
-    public void parse(List<Token> ll) throws IOException {
+    public void parse(List<Token> ll, JobConf job) throws IOException {
       StringBuilder sb = new StringBuilder();
       Iterator<Token> i = ll.iterator();
       while (i.hasNext()) {
@@ -269,7 +269,7 @@
           try {
             inf = (InputFormat)ReflectionUtils.newInstance(
                 Class.forName(sb.toString()).asSubclass(InputFormat.class),
-                null);
+                job);
           } catch (ClassNotFoundException e) {
             throw (IOException)new IOException().initCause(e);
           } catch (IllegalArgumentException e) {
@@ -424,7 +424,7 @@
     /**
      * Parse a list of comma-separated nodes.
      */
-    public void parse(List<Token> args) throws IOException {
+    public void parse(List<Token> args, JobConf job) throws IOException {
       ListIterator<Token> i = args.listIterator();
       while (i.hasNext()) {
         Token t = i.next();
@@ -447,7 +447,7 @@
     }
   }
 
-  private static Token reduce(Stack<Token> st) throws IOException {
+  private static Token reduce(Stack<Token> st, JobConf job) throws IOException {
     LinkedList<Token> args = new LinkedList<Token>();
     while (!st.isEmpty() && !TType.LPAREN.equals(st.peek().getType())) {
       args.addFirst(st.pop());
@@ -460,7 +460,7 @@
       throw new IOException("Identifier expected");
     }
     Node n = Node.forIdent(st.pop().getStr());
-    n.parse(args);
+    n.parse(args, job);
     return new NodeToken(n);
   }
 
@@ -468,17 +468,18 @@
    * Given an expression and an optional comparator, build a tree of
    * InputFormats using the comparator to sort keys.
    */
-  static Node parse(String expr, Class<? extends WritableComparator> cmpcl)
-      throws IOException {
+  static Node parse(String expr, JobConf job) throws IOException {
     if (null == expr) {
       throw new IOException("Expression is null");
     }
+    Class<? extends WritableComparator> cmpcl =
+      job.getClass("mapred.join.keycomparator", null, WritableComparator.class);
     Lexer lex = new Lexer(expr);
     Stack<Token> st = new Stack<Token>();
     Token tok;
     while ((tok = lex.next()) != null) {
       if (TType.RPAREN.equals(tok.getType())) {
-        st.push(reduce(st));
+        st.push(reduce(st, job));
       } else {
         st.push(tok);
       }

Added: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/join/ConfigurableInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/join/ConfigurableInputFormat.java?rev=654308&view=auto
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/join/ConfigurableInputFormat.java
(added)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/join/ConfigurableInputFormat.java
Wed May  7 15:36:13 2008
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred.join;
+
+import java.io.IOException;
+
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobConfigurable;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+
+public class ConfigurableInputFormat implements InputFormat, JobConfigurable {
+  boolean configured = false;
+
+  public ConfigurableInputFormat() { }
+
+  public void configure(JobConf job) {
+    configured = true;
+  }
+  public void validateInput(JobConf job) throws IOException {
+    if (!configured)
+      throw new IOException("Failed to configure child InputFormat");
+  }
+
+  public InputSplit[] getSplits(JobConf job, int numSplits)
+      throws IOException {
+    return null;
+  }
+
+  public RecordReader getRecordReader(
+      InputSplit split, JobConf job, Reporter reporter) throws IOException {
+    return null;
+  }
+}

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/join/TestDatamerge.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/join/TestDatamerge.java?rev=654308&r1=654307&r2=654308&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/join/TestDatamerge.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/join/TestDatamerge.java Wed May  7
15:36:13 2008
@@ -243,4 +243,12 @@
   public void testSimpleOverride() throws Exception {
     joinAs("override", OverrideChecker.class);
   }
+
+  public void testConfiguredInputFormat() throws Exception {
+    JobConf conf = new JobConf();
+    conf.set("mapred.join.expr", CompositeInputFormat.compose(
+          ConfigurableInputFormat.class, "/dingos"));
+    CompositeInputFormat cif = new CompositeInputFormat();
+    cif.validateInput(conf);
+  }
 }



Mime
View raw message