hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r925040 - in /hadoop/mapreduce/trunk: ./ src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/ src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/orm/ src/contrib/sqoop/src/test/...
Date Thu, 18 Mar 2010 23:17:51 GMT
Author: tomwhite
Date: Thu Mar 18 23:17:51 2010
New Revision: 925040

URL: http://svn.apache.org/viewvc?rev=925040&view=rev
Log:
MAPREDUCE-1446. Sqoop should support CLOB and BLOB datatypes. Contributed by Aaron Kimball.

Added:
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/BlobRef.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/ClobRef.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/LobSerializer.java
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/FieldFormatter.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/JdbcWritableBridge.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/SqlManager.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/orm/ClassWriter.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/TestColumnTypes.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLCompatTest.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/OracleCompatTest.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/testutil/ManagerCompatTestCase.java
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/lib/db/DBRecordReader.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=925040&r1=925039&r2=925040&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Mar 18 23:17:51 2010
@@ -68,6 +68,9 @@ Trunk (unreleased changes)
     MAPREDUCE-1455. Introduces job-level authorization for mapreduce servlets.
     (Ravi Gummadi via vinodkv)
 
+    MAPREDUCE-1446. Sqoop should support CLOB and BLOB datatypes.
+    (Aaron Kimball via tomwhite)
+
   IMPROVEMENTS
 
     MAPREDUCE-1198. Alternatively schedule different types of tasks in

Added: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/BlobRef.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/BlobRef.java?rev=925040&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/BlobRef.java
(added)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/BlobRef.java
Thu Mar 18 23:17:51 2010
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.lib;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * BlobRef is a wrapper that holds a Blob either directly, or a
+ * reference to a file that holds the blob data.
+ */
+public class BlobRef implements Writable {
+
+  public BlobRef(byte [] bytes) {
+    this.blobFileNum = 0;
+    this.data = new BytesWritable(bytes);
+  }
+
+  public BlobRef() {
+    this.blobFileNum = 0;
+    this.data = null;
+  }
+
+  // If the data is 'small', it's held directly, here.
+  private BytesWritable data;
+
+  // If there data is too large, it's written into a file
+  // and the file is numbered; this number is recorded here.
+  // This takes precedence if this value is positive.
+  private long blobFileNum;
+
+  public byte [] getData() {
+    if (blobFileNum > 0) {
+      // We have a numbered file.
+      // TODO: Implement this.
+      throw new RuntimeException("Unsupported: Indirect BLOBs are not supported");
+    }
+
+    return data.getBytes();
+  }
+
+  @Override
+  public String toString() {
+    if (blobFileNum > 0) {
+      return "indirectBlob(" + blobFileNum + ")";
+    } else {
+      return data.toString();
+    }
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    // The serialization format for this object is:
+    // boolean isIndirect
+    // if true, the next field is a Long containing blobFileNum
+    // if false, the next field is String data.
+
+    boolean isIndirect = in.readBoolean();
+    if (isIndirect) {
+      this.data = null;
+      this.blobFileNum = in.readLong();
+    } else {
+      if (null == this.data) {
+        this.data = new BytesWritable();
+      }
+      this.data.readFields(in);
+      this.blobFileNum = 0;
+    }
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    boolean isIndirect = blobFileNum > 0;
+    out.writeBoolean(isIndirect);
+    if (isIndirect) {
+      out.writeLong(blobFileNum);
+    } else {
+      data.write(out);
+    }
+  }
+}
+

Added: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/ClobRef.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/ClobRef.java?rev=925040&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/ClobRef.java
(added)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/ClobRef.java
Thu Mar 18 23:17:51 2010
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.lib;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * ClobRef is a wrapper that holds a Clob either directly, or a
+ * reference to a file that holds the clob data.
+ */
+public class ClobRef implements Writable {
+
+  public ClobRef(String chars) {
+    this.clobFileNum = 0;
+    this.data = chars;
+  }
+
+  public ClobRef() {
+    this.clobFileNum = 0;
+    this.data = null;
+  }
+
+  // If the data is 'small', it's held directly, here.
+  private String data;
+
+  // If there data is too large, it's written into a file
+  // and the file is numbered; this number is recorded here.
+  // This takes precedence if this value is positive.
+  private long clobFileNum;
+
+  public String getData() {
+    if (clobFileNum > 0) {
+      // We have a numbered file.
+      // TODO: Implement this.
+      throw new RuntimeException("Unsupported: Indirect CLOBs are not supported");
+    }
+
+    return data;
+  }
+
+  @Override
+  public String toString() {
+    if (clobFileNum > 0) {
+      return "indirectClob(" + clobFileNum + ")";
+    } else {
+      return data;
+    }
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    // The serialization format for this object is:
+    // boolean isIndirect
+    // if true, the next field is a Long containing clobFileNum
+    // if false, the next field is String data.
+
+    boolean isIndirect = in.readBoolean();
+    if (isIndirect) {
+      this.data = null;
+      this.clobFileNum = in.readLong();
+    } else {
+      this.data = Text.readString(in);
+      this.clobFileNum = 0;
+    }
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+    boolean isIndirect = clobFileNum > 0;
+    out.writeBoolean(isIndirect);
+    if (isIndirect) {
+      out.writeLong(clobFileNum);
+    } else {
+      Text.writeString(out, data);
+    }
+  }
+}
+

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/FieldFormatter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/FieldFormatter.java?rev=925040&r1=925039&r2=925040&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/FieldFormatter.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/FieldFormatter.java
Thu Mar 18 23:17:51 2010
@@ -20,9 +20,7 @@ package org.apache.hadoop.sqoop.lib;
 
 
 /**
- * Abstract base class for all DBWritable types generated by Sqoop.
- * Contains methods required by all such types, to help with parsing,
- * stringification, etc.
+ * Static helper class that will help format data with quotes and escape chars.
  */
 public final class FieldFormatter {
 

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/JdbcWritableBridge.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/JdbcWritableBridge.java?rev=925040&r1=925039&r2=925040&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/JdbcWritableBridge.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/JdbcWritableBridge.java
Thu Mar 18 23:17:51 2010
@@ -19,6 +19,8 @@
 package org.apache.hadoop.sqoop.lib;
 
 import java.math.BigDecimal;
+import java.sql.Blob;
+import java.sql.Clob;
 import java.sql.Date;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
@@ -31,12 +33,13 @@ import java.sql.Timestamp;
  * Java types, and do serialization of these types to/from DataInput/DataOutput
  * for use with Hadoop's Writable implementation. This supports null values
  * for all types.
- *
- * 
- *
  */
 public final class JdbcWritableBridge {
 
+  // Currently, cap BLOB/CLOB objects at 16 MB until we can use external storage.
+  public final static long MAX_BLOB_LENGTH = 16 * 1024 * 1024;
+  public final static long MAX_CLOB_LENGTH = 16 * 1024 * 1024;
+
   private JdbcWritableBridge() {
   }
 
@@ -110,6 +113,36 @@ public final class JdbcWritableBridge {
     return r.getBigDecimal(colNum);
   }
 
+  public static BlobRef readBlobRef(int colNum, ResultSet r)
+      throws SQLException {
+    Blob b = r.getBlob(colNum);
+    if (null == b) {
+      return null;
+    } else if (b.length() > MAX_BLOB_LENGTH) {
+      // TODO: Deserialize very large BLOBs into separate files.
+      throw new UnsupportedOperationException("BLOB size exceeds max: "
+          + MAX_BLOB_LENGTH);
+    } else {
+      // This is a 1-based array.
+      return new BlobRef(b.getBytes(1, (int) b.length()));
+    }
+  }
+
+  public static ClobRef readClobRef(int colNum, ResultSet r)
+      throws SQLException {
+    Clob c = r.getClob(colNum);
+    if (null == c) {
+      return null;
+    } else if (c.length() > MAX_CLOB_LENGTH) {
+      // TODO: Deserialize very large CLOBs into separate files.
+      throw new UnsupportedOperationException("CLOB size exceeds max: "
+          + MAX_CLOB_LENGTH);
+    } else {
+      // This is a 1-based array.
+      return new ClobRef(c.getSubString(1, (int) c.length()));
+    }
+  }
+
   public static void writeInteger(Integer val, int paramIdx, int sqlType, PreparedStatement
s)
       throws SQLException {
     if (null == val) {
@@ -200,4 +233,15 @@ public final class JdbcWritableBridge {
     }
   }
 
+  public static void writeBlobRef(BlobRef val, int paramIdx,
+      int sqlType, PreparedStatement s) throws SQLException {
+    // TODO: support this.
+    throw new RuntimeException("Unsupported: Cannot export BLOB data");
+  }
+
+  public static void writeClobRef(ClobRef val, int paramIdx,
+      int sqlType, PreparedStatement s) throws SQLException {
+    // TODO: support this.
+    throw new RuntimeException("Unsupported: Cannot export CLOB data");
+  }
 }

Added: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/LobSerializer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/LobSerializer.java?rev=925040&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/LobSerializer.java
(added)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/lib/LobSerializer.java
Thu Mar 18 23:17:51 2010
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.sqoop.lib;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * Serialize LOB classes to/from DataInput and DataOutput objects.
+ */
+public final class LobSerializer {
+
+  private LobSerializer() { }
+
+  public static void writeClob(ClobRef clob, DataOutput out) throws IOException {
+    clob.write(out);
+  }
+
+  public static void writeBlob(BlobRef blob, DataOutput out) throws IOException {
+    blob.write(out);
+  }
+
+  public static ClobRef readClobFields(DataInput in) throws IOException {
+    ClobRef clob = new ClobRef();
+    clob.readFields(in);
+    return clob;
+  }
+
+  public static BlobRef readBlobFields(DataInput in) throws IOException {
+    BlobRef blob = new BlobRef();
+    blob.readFields(in);
+    return blob;
+  }
+}

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/SqlManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/SqlManager.java?rev=925040&r1=925039&r2=925040&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/SqlManager.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/SqlManager.java
Thu Mar 18 23:17:51 2010
@@ -20,6 +20,8 @@ package org.apache.hadoop.sqoop.manager;
 
 import org.apache.hadoop.sqoop.SqoopOptions;
 import org.apache.hadoop.sqoop.hive.HiveTypes;
+import org.apache.hadoop.sqoop.lib.BlobRef;
+import org.apache.hadoop.sqoop.lib.ClobRef;
 import org.apache.hadoop.sqoop.mapreduce.DataDrivenImportJob;
 import org.apache.hadoop.sqoop.mapreduce.ExportJob;
 import org.apache.hadoop.sqoop.util.ExportException;
@@ -359,10 +361,17 @@ public abstract class SqlManager extends
       return "java.sql.Time";
     } else if (sqlType == Types.TIMESTAMP) {
       return "java.sql.Timestamp";
+    } else if (sqlType == Types.CLOB) {
+      return ClobRef.class.getName();
+    } else if (sqlType == Types.BLOB
+        || sqlType == Types.LONGVARBINARY
+        || sqlType == Types.VARBINARY
+        || sqlType == Types.BINARY) {
+      return BlobRef.class.getName();
     } else {
-      // TODO(aaron): Support BINARY, VARBINARY, LONGVARBINARY, DISTINCT, CLOB, BLOB, ARRAY,
-      // STRUCT, REF, JAVA_OBJECT.
-      // return database specific java data type
+      // TODO(aaron): Support DISTINCT, ARRAY, STRUCT, REF, JAVA_OBJECT.
+      // Return null indicating database-specific manager should return a
+      // java data type if it can find one for any nonstandard type.
       return null;
     }
   }

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/orm/ClassWriter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/orm/ClassWriter.java?rev=925040&r1=925039&r2=925040&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/orm/ClassWriter.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/orm/ClassWriter.java
Thu Mar 18 23:17:51 2010
@@ -22,9 +22,12 @@ import org.apache.hadoop.sqoop.SqoopOpti
 import org.apache.hadoop.sqoop.manager.ConnManager;
 import org.apache.hadoop.sqoop.manager.SqlManager;
 import org.apache.hadoop.sqoop.lib.BigDecimalSerializer;
-import org.apache.hadoop.sqoop.lib.JdbcWritableBridge;
 import org.apache.hadoop.sqoop.lib.FieldFormatter;
+import org.apache.hadoop.sqoop.lib.JdbcWritableBridge;
+import org.apache.hadoop.sqoop.lib.LobSerializer;
 import org.apache.hadoop.sqoop.lib.RecordParser;
+import org.apache.hadoop.sqoop.lib.BlobRef;
+import org.apache.hadoop.sqoop.lib.ClobRef;
 import org.apache.hadoop.sqoop.lib.SqoopRecord;
 
 import java.io.File;
@@ -291,6 +294,12 @@ public class ClassWriter {
     } else if (javaType.equals("java.math.BigDecimal")) {
       return "    this." + colName + " = " + BigDecimalSerializer.class.getCanonicalName()
           + ".readFields(" + inputObj + ");\n";
+    } else if (javaType.equals(ClobRef.class.getName())) {
+      return "    this." + colName + " = " + LobSerializer.class.getCanonicalName()
+          + ".readClobFields(" + inputObj + ");\n";
+    } else if (javaType.equals(BlobRef.class.getName())) {
+      return "    this." + colName + " = " + LobSerializer.class.getCanonicalName()
+          + ".readBlobFields(" + inputObj + ");\n";
     } else {
       LOG.error("No ResultSet method for Java type " + javaType);
       return null;
@@ -342,6 +351,12 @@ public class ClassWriter {
     } else if (javaType.equals("java.math.BigDecimal")) {
       return "    " + BigDecimalSerializer.class.getCanonicalName()
           + ".write(this." + colName + ", " + outputObj + ");\n";
+    } else if (javaType.equals(ClobRef.class.getName())) {
+      return "    " + LobSerializer.class.getCanonicalName()
+          + ".writeClob(this." + colName + ", " + outputObj + ");\n";
+    } else if (javaType.equals(BlobRef.class.getName())) {
+      return "    " + LobSerializer.class.getCanonicalName()
+          + ".writeBlob(this." + colName + ", " + outputObj + ");\n";
     } else {
       LOG.error("No ResultSet method for Java type " + javaType);
       return null;
@@ -622,6 +637,13 @@ public class ClassWriter {
       sb.append("      this." + colName + " = java.sql.Timestamp.valueOf(__cur_str);\n");
     } else if (javaType.equals("java.math.BigDecimal")) {
       sb.append("      this." + colName + " = new java.math.BigDecimal(__cur_str);\n");
+    } else if (javaType.equals(ClobRef.class.getName())) {
+      sb.append("      this." + colName + " = new ClobRef(__cur_str);\n");
+    } else if (javaType.equals(BlobRef.class.getName())) {
+      // We don't support parsing BLOB data.
+      // Users must store this in SequenceFiles.
+      LOG.warn("BLOB data cannot be reparsed from text files");
+      sb.append("      this." + colName + " = new BlobRef();\n");
     } else {
       LOG.error("No parser available for Java type " + javaType);
     }
@@ -820,6 +842,8 @@ public class ClassWriter {
     sb.append("import " + JdbcWritableBridge.class.getCanonicalName() + ";\n");
     sb.append("import " + FieldFormatter.class.getCanonicalName() + ";\n");
     sb.append("import " + RecordParser.class.getCanonicalName() + ";\n");
+    sb.append("import " + BlobRef.class.getCanonicalName() + ";\n");
+    sb.append("import " + ClobRef.class.getCanonicalName() + ";\n");
     sb.append("import " + SqoopRecord.class.getCanonicalName() + ";\n");
     sb.append("import java.sql.PreparedStatement;\n");
     sb.append("import java.sql.ResultSet;\n");

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/TestColumnTypes.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/TestColumnTypes.java?rev=925040&r1=925039&r2=925040&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/TestColumnTypes.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/TestColumnTypes.java
Thu Mar 18 23:17:51 2010
@@ -58,5 +58,17 @@ public class TestColumnTypes extends Man
   }
 
   // Don't need to override getConnectString() because the default uses hsqldb.
+
+  // HSQLdb does not support these types over JDBC.
+
+  @Override
+  protected boolean supportsClob() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsBlob() {
+    return false;
+  }
 }
 

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLCompatTest.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLCompatTest.java?rev=925040&r1=925039&r2=925040&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLCompatTest.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/MySQLCompatTest.java
Thu Mar 18 23:17:51 2010
@@ -83,6 +83,16 @@ public class MySQLCompatTest extends Man
   }
 
   @Override
+  protected String getClobType() {
+    return "MEDIUMTEXT";
+  }
+
+  @Override
+  protected String getBlobType() {
+    return "MEDIUMBLOB";
+  }
+
+  @Override
   protected String getTrueBoolDbOutput() {
     return "1";
   }

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/OracleCompatTest.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/OracleCompatTest.java?rev=925040&r1=925039&r2=925040&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/OracleCompatTest.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/manager/OracleCompatTest.java
Thu Mar 18 23:17:51 2010
@@ -18,10 +18,12 @@
 
 package org.apache.hadoop.sqoop.manager;
 
+import java.io.UnsupportedEncodingException;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.Formatter;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -192,9 +194,31 @@ public class OracleCompatTest extends Ma
     return doubleAsInserted;
   }
 
+  @Override
+  protected String getBlobInsertStr(String blobData) {
+    // Oracle wants blob data encoded as hex (e.g. '01fca3b5').
+
+    StringBuilder sb = new StringBuilder();
+    sb.append("'");
+
+    Formatter fmt = new Formatter(sb);
+    try {
+      for (byte b : blobData.getBytes("UTF-8")) {
+        fmt.format("%02X", b);
+      }
+    } catch (UnsupportedEncodingException uee) {
+      // Should not happen; Java always supports UTF-8.
+      fail("Could not get utf-8 bytes for blob string");
+      return null;
+    }
+    sb.append("'");
+    return sb.toString();
+  }
+
   // Disable this test since Oracle isn't ANSI compliant.
   @Override
   public void testEmptyStringCol() {
+    this.skipped = true;
     LOG.info(
         "Oracle treats empty strings as null (non-ANSI compliant). Skipping.");
   }

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/testutil/ManagerCompatTestCase.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/testutil/ManagerCompatTestCase.java?rev=925040&r1=925039&r2=925040&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/testutil/ManagerCompatTestCase.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/test/org/apache/hadoop/sqoop/testutil/ManagerCompatTestCase.java
Thu Mar 18 23:17:51 2010
@@ -20,12 +20,16 @@ package org.apache.hadoop.sqoop.testutil
 
 import java.io.File;
 import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.sql.Blob;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.util.ToolRunner;
 
 import org.apache.hadoop.sqoop.SqoopOptions;
@@ -146,6 +150,16 @@ public abstract class ManagerCompatTestC
     return true;
   }
 
+  /** @return true if the database under test supports CLOB types */
+  protected boolean supportsClob() {
+    return true;
+  }
+
+  /** @return true if the database under test supports BLOB types */
+  protected boolean supportsBlob() {
+    return true;
+  }
+
   //////// These methods indicate how to define various datatypes.
 
   /**
@@ -215,6 +229,20 @@ public abstract class ManagerCompatTestC
     return "TIMESTAMP";
   }
 
+  /**
+   * Define a CLOB column that can contain up to 16 MB of data.
+   */
+  protected String getClobType() {
+    return "CLOB";
+  }
+  
+  /**
+   * Define a BLOB column that can contain up to 16 MB of data.
+   */
+  protected String getBlobType() {
+    return "BLOB";
+  }
+
   //////// These methods indicate how databases respond to various datatypes.
   //////// Since our comparisons are all string-based, these return strings.
 
@@ -425,6 +453,38 @@ public abstract class ManagerCompatTestC
     return asInserted;
   }
 
+  /**
+   * Encode a string to be inserted in a BLOB field
+   * @param blobData the raw text (Without quote marks) to insert for a BLOB.
+   * @return 'blobData' in a String form ready for insertion
+   */
+  protected String getBlobInsertStr(String blobData) {
+    return "'" + blobData + "'";
+  }
+
+  /**
+   * @return A byte array declaring how an inserted BLOB will be returned to
+   * us via the database.
+   */
+  protected byte [] getBlobDbOutput(String asInserted) {
+    // The database will give us back a byte array; we need to create
+    // an identical byte array.
+    try {
+      return asInserted.getBytes("UTF-8");
+    } catch (UnsupportedEncodingException uee) {
+      fail("Could not get utf8 bytes"); // Java should always support UTF-8.
+      return null;
+    }
+  }
+
+  /**
+   * @return A String declaring how an inserted BLOB will be returned to
+   * us via the sequencefile.
+   */
+  protected String getBlobSeqOutput(String asInserted) {
+    return new BytesWritable(getBlobDbOutput(asInserted)).toString();
+  }
+
   //////// The actual tests occur below here. ////////
 
   /**
@@ -794,5 +854,87 @@ public abstract class ManagerCompatTestC
         "this is a long varchar");
   }
 
+
+  protected void verifyClob(String insertVal, String returnVal, String seqFileVal) {
+    String [] types = { "INTEGER NOT NULL", getClobType() };
+    String [] vals = { "1", insertVal };
+    String [] checkCol = { "DATA_COL0", "DATA_COL1" };
+
+    createTableWithColTypes(types, vals);
+    verifyReadback(2, returnVal);
+    verifyImport("1," + seqFileVal, checkCol);
+  }
+
+  protected void verifyBlob(String insertVal, byte [] returnVal, String seqFileVal) {
+    String [] types = { "INTEGER NOT NULL", getBlobType() };
+    String [] vals = { "1", insertVal };
+    String [] checkCols = { "DATA_COL0", "DATA_COL1" };
+
+    createTableWithColTypes(types, vals);
+
+    // Verify readback of the data.
+    ResultSet results = null;
+    try {
+      results = getManager().readTable(getTableName(), getColNames());
+      assertNotNull("Null results from readTable()!", results);
+      assertTrue("Expected at least one row returned", results.next());
+      Blob blob = results.getBlob(2);
+      byte [] databaseBytes = blob.getBytes(1, (int) blob.length());
+      LOG.info("Verifying readback of bytes from " + getTableName());
+
+      assertEquals("byte arrays differ in size", returnVal.length,
+          databaseBytes.length);
+      for (int i = 0; i < returnVal.length; i++) {
+        assertEquals("bytes differ at position " + i + ". Expected "
+            + returnVal[i] + "; got " + databaseBytes[i],
+            returnVal[i],
+            databaseBytes[i]);
+      }
+
+      assertFalse("Expected at most one row returned", results.next());
+    } catch (SQLException sqlE) {
+      fail("Got SQLException: " + sqlE.toString());
+    } finally {
+      if (null != results) {
+        try {
+          results.close();
+        } catch (SQLException sqlE) {
+          fail("Got SQLException in resultset.close(): " + sqlE.toString());
+        }
+      }
+
+      // Free internal resources after the readTable.
+      getManager().release();
+    }
+
+    // Now verify that we can use the Sqoop import mechanism on this data.
+    verifyImport("1," + seqFileVal, checkCols);
+  }
+
+
+  @Test
+  public void testClob1() {
+    if (!supportsClob()) {
+      LOG.info("Skipping CLOB test; database does not support CLOB");
+      return;
+    }
+
+    verifyClob("'This is short CLOB data'",
+        "This is short CLOB data",
+        "This is short CLOB data");
+  }
+
+  @Test
+  public void testBlob1() {
+    if (!supportsBlob()) {
+      LOG.info("Skipping BLOB test; database does not support BLOB");
+      return;
+    }
+
+    verifyBlob(getBlobInsertStr("This is short BLOB data"),
+        getBlobDbOutput("This is short BLOB data"),
+        getBlobSeqOutput("This is short BLOB data"));
+  }
+
 }
 

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/lib/db/DBRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/lib/db/DBRecordReader.java?rev=925040&r1=925039&r2=925040&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/lib/db/DBRecordReader.java
(original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/lib/db/DBRecordReader.java
Thu Mar 18 23:17:51 2010
@@ -232,7 +232,7 @@ public class DBRecordReader<T extends DB
 
       pos ++;
     } catch (SQLException e) {
-      throw new IOException(e.getMessage());
+      throw new IOException("SQLException in nextKeyValue", e);
     }
     return true;
   }



Mime
View raw message