hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r771590 - in /hadoop/core/trunk: CHANGES.txt src/mapred/org/apache/hadoop/mapred/lib/db/DBConfiguration.java src/mapred/org/apache/hadoop/mapred/lib/db/DBOutputFormat.java src/test/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java
Date Tue, 05 May 2009 06:47:28 GMT
Author: cdouglas
Date: Tue May  5 06:47:27 2009
New Revision: 771590

URL: http://svn.apache.org/viewvc?rev=771590&view=rev
Log:
HADOOP-5500. Where field names are absent permit the number of fields to be
sufficient to construct the select query. Contributed by Enis Soztutar

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBConfiguration.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBOutputFormat.java
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=771590&r1=771589&r2=771590&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue May  5 06:47:27 2009
@@ -308,6 +308,10 @@
 
     HADOOP-5727. Simplify hashcode for ID types. (Shevek via cdouglas)
 
+    HADOOP-5500. In DBOutputFormat, where field names are absent permit the
+    number of fields to be sufficient to construct the select query. (Enis
+    Soztutar via cdouglas)
+
   OPTIMIZATIONS
 
     HADOOP-5595. NameNode does not need to run a replicator to choose a

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBConfiguration.java?rev=771590&r1=771589&r2=771590&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBConfiguration.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBConfiguration.java Tue
May  5 06:47:27 2009
@@ -80,6 +80,9 @@
   /** Field names in the Output table */
   public static final String OUTPUT_FIELD_NAMES_PROPERTY = "mapred.jdbc.output.field.names";
 
 
+  /** Number of fields in the Output table */
+  public static final String OUTPUT_FIELD_COUNT_PROPERTY = "mapred.jdbc.output.field.count";
 
+  
   /**
    * Sets the DB access related fields in the JobConf.  
    * @param job the job
@@ -212,5 +215,13 @@
     job.setStrings(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY, fieldNames);
   }
 
+  void setOutputFieldCount(int fieldCount) {
+    job.setInt(DBConfiguration.OUTPUT_FIELD_COUNT_PROPERTY, fieldCount);
+  }
+  
+  int getOutputFieldCount() {
+    return job.getInt(OUTPUT_FIELD_COUNT_PROPERTY, 0);
+  }
+  
 }
 

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBOutputFormat.java?rev=771590&r1=771589&r2=771590&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBOutputFormat.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBOutputFormat.java Tue May
 5 06:47:27 2009
@@ -151,6 +151,10 @@
     String tableName = dbConf.getOutputTableName();
     String[] fieldNames = dbConf.getOutputFieldNames();
     
+    if(fieldNames == null) {
+      fieldNames = new String[dbConf.getOutputFieldCount()];
+    }
+    
     try {
       Connection connection = dbConf.getConnection();
       PreparedStatement statement = null;
@@ -166,21 +170,42 @@
   /**
    * Initializes the reduce-part of the job with the appropriate output settings
    * 
-   * @param job
-   *          The job
-   * @param tableName
-   *          The table to insert data into
-   * @param fieldNames
-   *          The field names in the table. If unknown, supply the appropriate
-   *          number of nulls.
+   * @param job The job
+   * @param tableName The table to insert data into
+   * @param fieldNames The field names in the table.
    */
   public static void setOutput(JobConf job, String tableName, String... fieldNames) {
+    if(fieldNames.length > 0 && fieldNames[0] != null) {
+      DBConfiguration dbConf = setOutput(job, tableName);
+      dbConf.setOutputFieldNames(fieldNames);
+    } else {
+      if(fieldNames.length > 0)
+        setOutput(job, tableName, fieldNames.length);
+      else 
+        throw new IllegalArgumentException("Field names must be greater than 0");
+    }
+  }
+  
+  /**
+   * Initializes the reduce-part of the job with the appropriate output settings
+   * 
+   * @param job The job
+   * @param tableName The table to insert data into
+   * @param fieldCount the number of fields in the table.
+   */
+  public static void setOutput(JobConf job, String tableName, int fieldCount) {
+    DBConfiguration dbConf = setOutput(job, tableName);
+    dbConf.setOutputFieldCount(fieldCount);
+  }
+  
+  private static DBConfiguration setOutput(JobConf job, String tableName) {
     job.setOutputFormat(DBOutputFormat.class);
     job.setReduceSpeculativeExecution(false);
 
     DBConfiguration dbConf = new DBConfiguration(job);
     
     dbConf.setOutputTableName(tableName);
-    dbConf.setOutputFieldNames(fieldNames);
+    return dbConf;
   }
+  
 }

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java?rev=771590&r1=771589&r2=771590&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java Tue
May  5 06:47:27 2009
@@ -1,19 +1,50 @@
 package org.apache.hadoop.mapred.lib.db;
 
-import org.apache.hadoop.io.NullWritable;
+import java.io.IOException;
 
 import junit.framework.TestCase;
 
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapred.JobConf;
+
 public class TestConstructQuery extends TestCase {
-  public void testConstructQuery() {
-    DBOutputFormat<DBWritable, NullWritable> format = new DBOutputFormat<DBWritable,
NullWritable>();
-    String expected = "INSERT INTO hadoop_output (id,name,value) VALUES (?,?,?);";
-    String[] fieldNames = new String[] { "id", "name", "value" };
+  
+  private String[] fieldNames = new String[] { "id", "name", "value" };
+  private String[] nullFieldNames = new String[] { null, null, null };
+  private String expected = "INSERT INTO hadoop_output (id,name,value) VALUES (?,?,?);";
+  private String nullExpected = "INSERT INTO hadoop_output VALUES (?,?,?);"; 
+  
+  private DBOutputFormat<DBWritable, NullWritable> format 
+    = new DBOutputFormat<DBWritable, NullWritable>();
+  
+  public void testConstructQuery() {  
     String actual = format.constructQuery("hadoop_output", fieldNames);
     assertEquals(expected, actual);
-    expected = "INSERT INTO hadoop_output VALUES (?,?,?);";
-    fieldNames = new String[] { null, null, null };
-    actual = format.constructQuery("hadoop_output", fieldNames);
+    
+    actual = format.constructQuery("hadoop_output", nullFieldNames);
+    assertEquals(nullExpected, actual);
+  }
+  
+  public void testSetOutput() throws IOException {
+    JobConf job = new JobConf();
+    DBOutputFormat.setOutput(job, "hadoop_output", fieldNames);
+    
+    DBConfiguration dbConf = new DBConfiguration(job);
+    String actual = format.constructQuery(dbConf.getOutputTableName()
+        , dbConf.getOutputFieldNames());
+    
     assertEquals(expected, actual);
+    
+    job = new JobConf();
+    dbConf = new DBConfiguration(job);
+    DBOutputFormat.setOutput(job, "hadoop_output", nullFieldNames.length);
+    assertNull(dbConf.getOutputFieldNames());
+    assertEquals(nullFieldNames.length, dbConf.getOutputFieldCount());
+    
+    actual = format.constructQuery(dbConf.getOutputTableName()
+        , new String[dbConf.getOutputFieldCount()]);
+    
+    assertEquals(nullExpected, actual);
   }
+  
 }



Mime
View raw message