hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r910763 - in /hadoop/mapreduce/trunk: ./ src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/ src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/
Date Tue, 16 Feb 2010 23:26:58 GMT
Author: tomwhite
Date: Tue Feb 16 23:26:57 2010
New Revision: 910763

URL: http://svn.apache.org/viewvc?rev=910763&view=rev
Log:
MAPREDUCE-1444. Sqoop ConnManager instances can leak Statement objects. Contributed by Aaron
Kimball.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/SqoopOptions.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ConnManager.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/OracleManager.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/PostgresqlManager.java
    hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/SqlManager.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=910763&r1=910762&r2=910763&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue Feb 16 23:26:57 2010
@@ -354,6 +354,9 @@
     MAPREDUCE-1398. Fix TaskLauncher to stop waiting for slots on a TIP that
     is killed / failed.
     (Amareshwari Sriramadasu via yhemanth)
+
+    MAPREDUCE-1444. Sqoop ConnManager instances can leak Statement objects.
+    (Aaron Kimball via tomwhite)
  
 Release 0.21.0 - Unreleased
 

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java?rev=910763&r1=910762&r2=910763&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java (original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/Sqoop.java Tue
Feb 16 23:26:57 2010
@@ -63,16 +63,21 @@
   private List<String> generatedJarFiles;
 
   public Sqoop() {
-    init();
+    this((Configuration) null);
   }
 
   public Sqoop(Configuration conf) {
-    init();
-    setConf(conf);
+    this(conf, new SqoopOptions());
   }
 
-  private void init() {
+  public Sqoop(Configuration conf, SqoopOptions opts) {
     generatedJarFiles = new ArrayList<String>();
+    if (null != conf) {
+      setConf(conf);
+    }
+
+    this.options = opts;
+    this.options.setConf(getConf());
   }
 
   public SqoopOptions getOptions() {
@@ -146,8 +151,13 @@
    * Actual main entry-point for the program
    */
   public int run(String [] args) {
-    options = new SqoopOptions();
-    options.setConf(getConf());
+    if (options.getConf() == null) {
+      // Configuration wasn't initialized until after the ToolRunner
+      // got us to this point. ToolRunner gave Sqoop itself a Conf
+      // though.
+      options.setConf(getConf());
+    }
+
     try {
       options.parse(args);
       options.validate();

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/SqoopOptions.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/SqoopOptions.java?rev=910763&r1=910762&r2=910763&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/SqoopOptions.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/SqoopOptions.java
Tue Feb 16 23:26:57 2010
@@ -723,6 +723,10 @@
     return tableName;
   }
 
+  public void setTableName(String tableName) {
+    this.tableName = tableName;
+  }
+
   public String getExportDir() {
     return exportDir;
   }

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ConnManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ConnManager.java?rev=910763&r1=910762&r2=910763&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ConnManager.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/ConnManager.java
Tue Feb 16 23:26:57 2010
@@ -83,7 +83,8 @@
    * If columns is null, all columns from the table are read. This is a direct
    * (non-parallelized) read of the table back to the current client.
    * The client is responsible for calling ResultSet.close() when done with the
-   * returned ResultSet object.
+   * returned ResultSet object, and for calling release() after that to free
+   * internal state.
    */
   public abstract ResultSet readTable(String tableName, String [] columns) throws SQLException;
 
@@ -144,5 +145,16 @@
       throws IOException, ExportException {
     throw new ExportException("This database does not support exports");
   }
+
+  /**
+   * If a method of this ConnManager has returned a ResultSet to you,
+   * you are responsible for calling release() after you close the
+   * ResultSet object, to free internal resources. ConnManager
+   * implementations do not guarantee the ability to have multiple
+   * returned ResultSets available concurrently. Requesting a new
+   * ResultSet from a ConnManager may cause other open ResulSets
+   * to close.
+   */
+  public abstract void release();
 }
 

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java?rev=910763&r1=910762&r2=910763&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/MySQLManager.java
Tue Feb 16 23:26:57 2010
@@ -23,6 +23,7 @@
 import java.net.URISyntaxException;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
+import java.sql.Statement;
 import java.sql.SQLException;
 import java.util.ArrayList;
 
@@ -46,6 +47,8 @@
   // set to true after we warn the user that we can use direct fastpath.
   private static boolean warningPrinted = false;
 
+  private Statement lastStatement;
+
   public MySQLManager(final SqoopOptions opts) {
     super(DRIVER_CLASS, opts);
   }
@@ -70,6 +73,7 @@
       results = execute("SHOW DATABASES");
     } catch (SQLException sqlE) {
       LOG.error("Error executing statement: " + sqlE.toString());
+      release();
       return null;
     }
 
@@ -90,6 +94,8 @@
       } catch (SQLException sqlE) {
         LOG.warn("Exception closing ResultSet: " + sqlE.toString());
       }
+
+      release();
     }
   }
 
@@ -182,6 +188,9 @@
    * @return A ResultSet encapsulating the results or null on error
    */
   protected ResultSet execute(String stmt, Object... args) throws SQLException {
+    // Free any previous resources.
+    release();
+
     PreparedStatement statement = null;
     statement = this.getConnection().prepareStatement(stmt,
         ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
@@ -193,9 +202,22 @@
     }
 
     LOG.info("Executing SQL statement: " + stmt);
+    this.lastStatement = statement;
     return statement.executeQuery();
   }
 
+  public void release() {
+    if (null != this.lastStatement) {
+      try {
+        this.lastStatement.close();
+      } catch (SQLException e) {
+        LOG.warn("Exception closing executed Statement: " + e);
+      }
+
+      this.lastStatement = null;
+    }
+  }
+
   /**
    * When using a column name in a generated SQL query, how (if at all)
    * should we escape that column name? e.g., a column named "table"

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/OracleManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/OracleManager.java?rev=910763&r1=910762&r2=910763&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/OracleManager.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/OracleManager.java
Tue Feb 16 23:26:57 2010
@@ -157,6 +157,30 @@
     importer.runImport(tableName, jarFile, splitCol, options.getConf());
   }
 
+  @Override
+  public ResultSet readTable(String tableName, String[] columns) throws SQLException {
+    if (columns == null) {
+      columns = getColumnNames(tableName);
+    }
+
+    StringBuilder sb = new StringBuilder();
+    sb.append("SELECT ");
+    boolean first = true;
+    for (String col : columns) {
+      if (!first) {
+        sb.append(", ");
+      }
+      sb.append(escapeColName(col));
+      first = false;
+    }
+    sb.append(" FROM ");
+    sb.append(escapeTableName(tableName));
+
+    String sqlCmd = sb.toString();
+    LOG.debug("Reading table with command: " + sqlCmd);
+    return execute(sqlCmd);
+  }
+
   /**
    * Resolve a database-specific type to the Java type that should contain it.
    * @param sqlType

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/PostgresqlManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/PostgresqlManager.java?rev=910763&r1=910762&r2=910763&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/PostgresqlManager.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/PostgresqlManager.java
Tue Feb 16 23:26:57 2010
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
+import java.sql.Statement;
 import java.sql.SQLException;
 import java.util.ArrayList;
 
@@ -46,6 +47,8 @@
   // set to true after we warn the user that we can use direct fastpath.
   private static boolean warningPrinted = false;
 
+  private Statement lastStatement;
+
   public PostgresqlManager(final SqoopOptions opts) {
     super(DRIVER_CLASS, opts);
   }
@@ -108,6 +111,9 @@
    * @return A ResultSet encapsulating the results or null on error
    */
   protected ResultSet execute(String stmt, Object... args) throws SQLException {
+    // Free any previous resources used.
+    release();
+
     PreparedStatement statement = null;
     statement = this.getConnection().prepareStatement(stmt,
         ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
@@ -119,7 +125,20 @@
     }
 
     LOG.info("Executing SQL statement: " + stmt);
+    this.lastStatement = statement;
     return statement.executeQuery();
   }
+
+  public void release() {
+    if (null != this.lastStatement) {
+      try {
+        this.lastStatement.close();
+      } catch (SQLException e) {
+        LOG.warn("Exception closing executed Statement: " + e);
+      }
+
+      this.lastStatement = null;
+    }
+  }
 }
 

Modified: hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/SqlManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/SqlManager.java?rev=910763&r1=910762&r2=910763&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/SqlManager.java
(original)
+++ hadoop/mapreduce/trunk/src/contrib/sqoop/src/java/org/apache/hadoop/sqoop/manager/SqlManager.java
Tue Feb 16 23:26:57 2010
@@ -33,6 +33,7 @@
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.ResultSetMetaData;
+import java.sql.Statement;
 import java.sql.SQLException;
 import java.sql.Types;
 import java.util.ArrayList;
@@ -53,6 +54,7 @@
   public static final Log LOG = LogFactory.getLog(SqlManager.class.getName());
 
   protected SqoopOptions options;
+  private Statement lastStatement;
 
   /**
    * Constructs the SqlManager
@@ -81,6 +83,7 @@
       results = execute(stmt);
     } catch (SQLException sqlE) {
       LOG.error("Error executing statement: " + sqlE.toString());
+      release();
       return null;
     }
 
@@ -106,6 +109,8 @@
       } catch (SQLException sqlE) {
         LOG.warn("SQLException closing ResultSet: " + sqlE.toString());
       }
+
+      release();
     }
   }
 
@@ -126,6 +131,7 @@
       results = execute(stmt);
     } catch (SQLException sqlE) {
       LOG.error("Error executing statement: " + sqlE.toString());
+      release();
       return null;
     }
 
@@ -152,9 +158,11 @@
       try {
         results.close();
         getConnection().commit();
-      } catch (SQLException sqlE) { 
+      } catch (SQLException sqlE) {
         LOG.warn("SQLException closing ResultSet: " + sqlE.toString());
       }
+
+      release();
     }
   }
 
@@ -179,7 +187,9 @@
     sb.append(" AS ");   // needed for hsqldb; doesn't hurt anyone else.
     sb.append(escapeTableName(tableName));
 
-    return execute(sb.toString());
+    String sqlCmd = sb.toString();
+    LOG.debug("Reading table with command: " + sqlCmd);
+    return execute(sqlCmd);
   }
 
   @Override
@@ -291,6 +301,9 @@
    * @return A ResultSet encapsulating the results or null on error
    */
   protected ResultSet execute(String stmt, Object... args) throws SQLException {
+    // Release any previously-open statement.
+    release();
+
     PreparedStatement statement = null;
     statement = this.getConnection().prepareStatement(stmt,
         ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
@@ -301,6 +314,7 @@
     }
 
     LOG.info("Executing SQL statement: " + stmt);
+    this.lastStatement = statement;
     return statement.executeQuery();
   }
 
@@ -363,6 +377,7 @@
   }
 
   public void close() throws SQLException {
+    release();
   }
 
   /**
@@ -376,6 +391,7 @@
       results = execute(s);
     } catch (SQLException sqlE) {
       LOG.error("Error executing statement: " + sqlE.toString());
+      release();
       return;
     }
 
@@ -404,6 +420,8 @@
       } catch (SQLException sqlE) {
         LOG.warn("SQLException closing ResultSet: " + sqlE.toString());
       }
+
+      release();
     }
   }
 
@@ -446,4 +464,16 @@
     ExportJob exportJob = new ExportJob(context);
     exportJob.runExport();
   }
+
+  public void release() {
+    if (null != this.lastStatement) {
+      try {
+        this.lastStatement.close();
+      } catch (SQLException e) {
+        LOG.warn("Exception closing executed Statement: " + e);
+      }
+
+      this.lastStatement = null;
+    }
+  }
 }



Mime
View raw message