hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From junping...@apache.org
Subject hadoop git commit: MAPREDUCE-6246. DBOutputFormat.java appending extra semicolon to query which is incompatible with DB2. Contributed by ramtin and Gergely Novák.
Date Fri, 07 Jul 2017 21:19:37 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 d657c0517 -> 5ad710bc7


MAPREDUCE-6246. DBOutputFormat.java appending extra semicolon to query which is incompatible
with DB2. Contributed by ramtin and Gergely Novák.

(cherry picked from commit f484a6ff602d48413556a1d046670e2003c71c2e)
(cherry picked from commit f823f9fd784fa4944178247a82df24fba03c2051)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ad710bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ad710bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ad710bc

Branch: refs/heads/branch-2.8
Commit: 5ad710bc7aec08b19ce1900c089ae6384fe42d6a
Parents: d657c05
Author: Junping Du <junping_du@apache.org>
Authored: Fri Jul 7 13:23:43 2017 -0700
Committer: Junping Du <junping_du@apache.org>
Committed: Fri Jul 7 13:32:21 2017 -0700

----------------------------------------------------------------------
 .../hadoop/mapreduce/lib/db/DBOutputFormat.java | 15 ++++++-
 .../mapreduce/lib/db/TestDBOutputFormat.java    | 45 ++++++++++++++++++++
 2 files changed, 58 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ad710bc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java
index 2e3a9d8..c222bf5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.lib.db;
 
 import java.io.IOException;
 import java.sql.Connection;
+import java.sql.DatabaseMetaData;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
 
@@ -51,6 +52,8 @@ public class DBOutputFormat<K  extends DBWritable, V>
 extends OutputFormat<K,V> {
 
   private static final Log LOG = LogFactory.getLog(DBOutputFormat.class);
+  public String dbProductName = "DEFAULT";
+
   public void checkOutputSpecs(JobContext context) 
       throws IOException, InterruptedException {}
 
@@ -158,7 +161,12 @@ extends OutputFormat<K,V> {
         query.append(",");
       }
     }
-    query.append(");");
+
+    if (dbProductName.startsWith("DB2") || dbProductName.startsWith("ORACLE")) {
+      query.append(")");
+    } else {
+      query.append(");");
+    }
 
     return query.toString();
   }
@@ -177,7 +185,10 @@ extends OutputFormat<K,V> {
     try {
       Connection connection = dbConf.getConnection();
       PreparedStatement statement = null;
-  
+
+      DatabaseMetaData dbMeta = connection.getMetaData();
+      this.dbProductName = dbMeta.getDatabaseProductName().toUpperCase();
+
       statement = connection.prepareStatement(
                     constructQuery(tableName, fieldNames));
       return new DBRecordWriter(connection, statement);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ad710bc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java
index 014855f..e547c8a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java
@@ -18,7 +18,9 @@
 package org.apache.hadoop.mapreduce.lib.db;
 
 import java.io.IOException;
+import java.lang.reflect.Field;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.mapreduce.Job;
@@ -26,6 +28,7 @@ import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
 
 public class TestDBOutputFormat {
   private String[] fieldNames = new String[] { "id", "name", "value" };
@@ -47,6 +50,48 @@ public class TestDBOutputFormat {
   }
 
   @Test
+  public void testDB2ConstructQuery() {
+    String db2expected = StringUtils.removeEnd(expected, ";");
+    String db2nullExpected = StringUtils.removeEnd(nullExpected, ";");
+
+    try {
+      Class<?> clazz = this.format.getClass();
+      Field field = clazz.getDeclaredField("dbProductName");
+      field.setAccessible(true);
+      field.set(format, "DB2");
+    } catch (IllegalAccessException | NoSuchFieldException e) {
+      fail(e.getMessage());
+    }
+
+    String actual = format.constructQuery("hadoop_output", fieldNames);
+    assertEquals(db2expected, actual);
+
+    actual = format.constructQuery("hadoop_output", nullFieldNames);
+    assertEquals(db2nullExpected, actual);
+  }
+
+  @Test
+  public void testORACLEConstructQuery() {
+    String oracleExpected = StringUtils.removeEnd(expected, ";");
+    String oracleNullExpected = StringUtils.removeEnd(nullExpected, ";");
+
+    try {
+      Class<?> clazz = this.format.getClass();
+      Field field = clazz.getDeclaredField("dbProductName");
+      field.setAccessible(true);
+      field.set(format, "ORACLE");
+    } catch (IllegalAccessException | NoSuchFieldException e) {
+      fail(e.getMessage());
+    }
+
+    String actual = format.constructQuery("hadoop_output", fieldNames);
+    assertEquals(oracleExpected, actual);
+
+    actual = format.constructQuery("hadoop_output", nullFieldNames);
+    assertEquals(oracleNullExpected, actual);
+  }
+
+  @Test
   public void testSetOutput() throws IOException {
     Job job = Job.getInstance(new Configuration());
     DBOutputFormat.setOutput(job, "hadoop_output", fieldNames);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message