hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject svn commit: r1148921 - in /hive/trunk: cli/src/java/org/apache/hadoop/hive/cli/ contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/io/
Date Wed, 20 Jul 2011 19:54:15 GMT
Author: jvs
Date: Wed Jul 20 19:54:14 2011
New Revision: 1148921

URL: http://svn.apache.org/viewvc?rev=1148921&view=rev
Log:
HIVE-1884. Potential risk of resource leaks in Hive
(Chinna Rao Lalam via jvs)


Modified:
    hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
    hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesWritableInput.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileInputFormat.java

Modified: hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java?rev=1148921&r1=1148920&r2=1148921&view=diff
==============================================================================
--- hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (original)
+++ hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java Wed Jul 20 19:54:14
2011
@@ -61,6 +61,7 @@ import org.apache.hadoop.hive.ql.session
 import org.apache.hadoop.hive.service.HiveClient;
 import org.apache.hadoop.hive.service.HiveServerException;
 import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.thrift.TException;
 
 import sun.misc.Signal;
@@ -390,14 +391,18 @@ public class CliDriver {
 
   public int processFile(String fileName) throws IOException {
     FileReader fileReader = null;
+    BufferedReader bufferReader = null;
+    int rc = 0;
     try {
       fileReader = new FileReader(fileName);
-      return processReader(new BufferedReader(fileReader));
+      bufferReader = new BufferedReader(fileReader);
+      rc = processReader(bufferReader);
+      bufferReader.close();
+      bufferReader = null;
     } finally {
-      if (fileReader != null) {
-        fileReader.close();
-      }
+      IOUtils.closeStream(bufferReader);
     }
+    return rc;
   }
 
   public void processInitFiles(CliSessionState ss) throws IOException {

Modified: hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesWritableInput.java
URL: http://svn.apache.org/viewvc/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesWritableInput.java?rev=1148921&r1=1148920&r2=1148921&view=diff
==============================================================================
--- hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesWritableInput.java
(original)
+++ hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesWritableInput.java
Wed Jul 20 19:54:14 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.io.ArrayWritabl
 import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.MapWritable;
@@ -380,22 +381,29 @@ public class TypedBytesWritableInput imp
   }
 
   public Writable readWritable(Writable writable) throws IOException {
-    ByteArrayInputStream bais = new ByteArrayInputStream(in.readBytes());
-    DataInputStream dis = new DataInputStream(bais);
-    String className = WritableUtils.readString(dis);
-    if (writable == null) {
-      try {
-        Class<? extends Writable> cls = conf.getClassByName(className)
-            .asSubclass(Writable.class);
-        writable = (Writable) ReflectionUtils.newInstance(cls, conf);
-      } catch (ClassNotFoundException e) {
-        throw new IOException(e);
+    DataInputStream dis = null;
+    try {
+      ByteArrayInputStream bais = new ByteArrayInputStream(in.readBytes());
+      dis = new DataInputStream(bais);
+      String className = WritableUtils.readString(dis);
+      if (writable == null) {
+        try {
+          Class<? extends Writable> cls = conf.getClassByName(className)
+              .asSubclass(Writable.class);
+          writable = (Writable) ReflectionUtils.newInstance(cls, conf);
+        } catch (ClassNotFoundException e) {
+          throw new IOException(e);
+        }
+      } else if (!writable.getClass().getName().equals(className)) {
+        throw new IOException("wrong Writable class given");
       }
-    } else if (!writable.getClass().getName().equals(className)) {
-      throw new IOException("wrong Writable class given");
+      writable.readFields(dis);
+      dis.close();
+      dis = null;
+      return writable;
+    } finally {
+      IOUtils.closeStream(dis);
     }
-    writable.readFields(dis);
-    return writable;
   }
 
   public Writable readWritable() throws IOException {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=1148921&r1=1148920&r2=1148921&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Wed Jul 20 19:54:14
2011
@@ -37,10 +37,10 @@ import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.Map.Entry;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -96,7 +96,6 @@ import org.apache.hadoop.hive.ql.plan.Ad
 import org.apache.hadoop.hive.ql.plan.AlterDatabaseDesc;
 import org.apache.hadoop.hive.ql.plan.AlterIndexDesc;
 import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
-import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
 import org.apache.hadoop.hive.ql.plan.AlterTableSimpleDesc;
 import org.apache.hadoop.hive.ql.plan.CreateDatabaseDesc;
 import org.apache.hadoop.hive.ql.plan.CreateIndexDesc;
@@ -129,6 +128,7 @@ import org.apache.hadoop.hive.ql.plan.Sh
 import org.apache.hadoop.hive.ql.plan.ShowTablesDesc;
 import org.apache.hadoop.hive.ql.plan.SwitchDatabaseDesc;
 import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
+import org.apache.hadoop.hive.ql.plan.AlterTableDesc.AlterTableTypes;
 import org.apache.hadoop.hive.ql.plan.api.StageType;
 import org.apache.hadoop.hive.ql.security.authorization.Privilege;
 import org.apache.hadoop.hive.serde.Constants;
@@ -141,6 +141,7 @@ import org.apache.hadoop.hive.serde2.dyn
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
 import org.apache.hadoop.hive.shims.HadoopShims;
 import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.ToolRunner;
 
 /**
@@ -435,10 +436,11 @@ public class DDLTask extends Task<DDLWor
   }
 
   private int showGrants(ShowGrantDesc showGrantDesc) throws HiveException {
+    DataOutput outStream = null;
     try {
       Path resFile = new Path(showGrantDesc.getResFile());
       FileSystem fs = resFile.getFileSystem(conf);
-      DataOutput outStream = fs.create(resFile);
+      outStream = fs.create(resFile);
       PrincipalDesc principalDesc = showGrantDesc.getPrincipalDesc();
       PrivilegeObjectDesc hiveObjectDesc = showGrantDesc.getHiveObj();
       String principalName = principalDesc.getName();
@@ -581,6 +583,7 @@ public class DDLTask extends Task<DDLWor
         }
       }
       ((FSDataOutputStream) outStream).close();
+      outStream = null;
     } catch (FileNotFoundException e) {
       LOG.info("show table status: " + stringifyException(e));
       return 1;
@@ -590,6 +593,8 @@ public class DDLTask extends Task<DDLWor
     } catch (Exception e) {
       e.printStackTrace();
       throw new HiveException(e);
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
     return 0;
   }
@@ -732,6 +737,7 @@ public class DDLTask extends Task<DDLWor
 
   private int roleDDL(RoleDDLDesc roleDDLDesc) {
     RoleDDLDesc.RoleOperation operation = roleDDLDesc.getOperation();
+    DataOutput outStream = null;
     try {
       if (operation.equals(RoleDDLDesc.RoleOperation.CREATE_ROLE)) {
         db.createRole(roleDDLDesc.getName(), roleDDLDesc.getRoleOwnerName());
@@ -743,12 +749,13 @@ public class DDLTask extends Task<DDLWor
         if (roles != null && roles.size() > 0) {
           Path resFile = new Path(roleDDLDesc.getResFile());
           FileSystem fs = resFile.getFileSystem(conf);
-          DataOutput outStream = fs.create(resFile);
+          outStream = fs.create(resFile);
           for (Role role : roles) {
             outStream.writeBytes("role name:" + role.getRoleName());
             outStream.write(terminator);
           }
           ((FSDataOutputStream) outStream).close();
+          outStream = null;
         }
       } else {
         throw new HiveException("Unkown role operation "
@@ -762,6 +769,8 @@ public class DDLTask extends Task<DDLWor
     } catch (IOException e) {
       LOG.info("role ddl exception: " + stringifyException(e));
       return 1;
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
 
     return 0;
@@ -1620,10 +1629,11 @@ public class DDLTask extends Task<DDLWor
     }
 
     // write the results in the file
+    DataOutput outStream = null;
     try {
       Path resFile = new Path(showParts.getResFile());
       FileSystem fs = resFile.getFileSystem(conf);
-      DataOutput outStream = fs.create(resFile);
+      outStream = fs.create(resFile);
       Iterator<String> iterParts = parts.iterator();
 
       while (iterParts.hasNext()) {
@@ -1632,6 +1642,7 @@ public class DDLTask extends Task<DDLWor
         outStream.write(terminator);
       }
       ((FSDataOutputStream) outStream).close();
+      outStream = null;
     } catch (FileNotFoundException e) {
       LOG.info("show partitions: " + stringifyException(e));
       throw new HiveException(e.toString());
@@ -1640,6 +1651,8 @@ public class DDLTask extends Task<DDLWor
       throw new HiveException(e.toString());
     } catch (Exception e) {
       throw new HiveException(e.toString());
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
 
     return 0;
@@ -1667,10 +1680,11 @@ public class DDLTask extends Task<DDLWor
     indexes = db.getIndexes(tbl.getDbName(), tbl.getTableName(), (short) -1);
 
     // write the results in the file
+    DataOutput outStream = null;
     try {
       Path resFile = new Path(showIndexes.getResFile());
       FileSystem fs = resFile.getFileSystem(conf);
-      DataOutput outStream = fs.create(resFile);
+      outStream = fs.create(resFile);
 
       if (showIndexes.isFormatted()) {
         // column headers
@@ -1685,6 +1699,7 @@ public class DDLTask extends Task<DDLWor
       }
 
       ((FSDataOutputStream) outStream).close();
+      outStream = null;
 
     } catch (FileNotFoundException e) {
       LOG.info("show indexes: " + stringifyException(e));
@@ -1694,6 +1709,8 @@ public class DDLTask extends Task<DDLWor
       throw new HiveException(e.toString());
     } catch (Exception e) {
       throw new HiveException(e.toString());
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
 
     return 0;
@@ -1720,10 +1737,11 @@ public class DDLTask extends Task<DDLWor
     LOG.info("results : " + databases.size());
 
     // write the results in the file
+    DataOutput outStream = null;
     try {
       Path resFile = new Path(showDatabasesDesc.getResFile());
       FileSystem fs = resFile.getFileSystem(conf);
-      DataOutput outStream = fs.create(resFile);
+      outStream = fs.create(resFile);
 
       for (String database : databases) {
         // create a row per database name
@@ -1731,6 +1749,7 @@ public class DDLTask extends Task<DDLWor
         outStream.write(terminator);
       }
       ((FSDataOutputStream) outStream).close();
+      outStream = null;
     } catch (FileNotFoundException e) {
       LOG.warn("show databases: " + stringifyException(e));
       return 1;
@@ -1739,6 +1758,8 @@ public class DDLTask extends Task<DDLWor
       return 1;
     } catch (Exception e) {
       throw new HiveException(e.toString());
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
     return 0;
   }
@@ -1772,10 +1793,11 @@ public class DDLTask extends Task<DDLWor
     }
 
     // write the results in the file
+    DataOutput outStream = null;
     try {
       Path resFile = new Path(showTbls.getResFile());
       FileSystem fs = resFile.getFileSystem(conf);
-      DataOutput outStream = fs.create(resFile);
+      outStream = fs.create(resFile);
       SortedSet<String> sortedTbls = new TreeSet<String>(tbls);
       Iterator<String> iterTbls = sortedTbls.iterator();
 
@@ -1785,6 +1807,7 @@ public class DDLTask extends Task<DDLWor
         outStream.write(terminator);
       }
       ((FSDataOutputStream) outStream).close();
+      outStream = null;
     } catch (FileNotFoundException e) {
       LOG.warn("show table: " + stringifyException(e));
       return 1;
@@ -1793,6 +1816,8 @@ public class DDLTask extends Task<DDLWor
       return 1;
     } catch (Exception e) {
       throw new HiveException(e.toString());
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
     return 0;
   }
@@ -1818,10 +1843,11 @@ public class DDLTask extends Task<DDLWor
     }
 
     // write the results in the file
+    DataOutput outStream = null;
     try {
       Path resFile = new Path(showFuncs.getResFile());
       FileSystem fs = resFile.getFileSystem(conf);
-      DataOutput outStream = fs.create(resFile);
+      outStream = fs.create(resFile);
       SortedSet<String> sortedFuncs = new TreeSet<String>(funcs);
       Iterator<String> iterFuncs = sortedFuncs.iterator();
 
@@ -1831,6 +1857,7 @@ public class DDLTask extends Task<DDLWor
         outStream.write(terminator);
       }
       ((FSDataOutputStream) outStream).close();
+      outStream = null;
     } catch (FileNotFoundException e) {
       LOG.warn("show function: " + stringifyException(e));
       return 1;
@@ -1839,6 +1866,8 @@ public class DDLTask extends Task<DDLWor
       return 1;
     } catch (Exception e) {
       throw new HiveException(e.toString());
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
     return 0;
   }
@@ -1861,10 +1890,11 @@ public class DDLTask extends Task<DDLWor
     }
 
     // write the results in the file
+    DataOutput outStream = null;
     try {
       Path resFile = new Path(showLocks.getResFile());
       FileSystem fs = resFile.getFileSystem(conf);
-      DataOutput outStream = fs.create(resFile);
+      outStream = fs.create(resFile);
       List<HiveLock> locks = null;
 
       if (showLocks.getTableName() == null) {
@@ -1915,6 +1945,7 @@ public class DDLTask extends Task<DDLWor
         outStream.write(terminator);
       }
       ((FSDataOutputStream) outStream).close();
+      outStream = null;
     } catch (FileNotFoundException e) {
       LOG.warn("show function: " + stringifyException(e));
       return 1;
@@ -1923,6 +1954,8 @@ public class DDLTask extends Task<DDLWor
       return 1;
     } catch (Exception e) {
       throw new HiveException(e.toString());
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
     return 0;
   }
@@ -2040,10 +2073,11 @@ public class DDLTask extends Task<DDLWor
     String funcName = descFunc.getName();
 
     // write the results in the file
+    DataOutput outStream = null;
     try {
       Path resFile = new Path(descFunc.getResFile());
       FileSystem fs = resFile.getFileSystem(conf);
-      DataOutput outStream = fs.create(resFile);
+      outStream = fs.create(resFile);
 
       // get the function documentation
       Description desc = null;
@@ -2079,6 +2113,7 @@ public class DDLTask extends Task<DDLWor
       outStream.write(terminator);
 
       ((FSDataOutputStream) outStream).close();
+      outStream = null;
     } catch (FileNotFoundException e) {
       LOG.warn("describe function: " + stringifyException(e));
       return 1;
@@ -2087,15 +2122,18 @@ public class DDLTask extends Task<DDLWor
       return 1;
     } catch (Exception e) {
       throw new HiveException(e.toString());
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
     return 0;
   }
 
   private int descDatabase(DescDatabaseDesc descDatabase) throws HiveException {
+    DataOutput outStream = null;
     try {
       Path resFile = new Path(descDatabase.getResFile());
       FileSystem fs = resFile.getFileSystem(conf);
-      DataOutput outStream = fs.create(resFile);
+      outStream = fs.create(resFile);
 
       Database database = db.getDatabase(descDatabase.getDatabaseName());
 
@@ -2123,6 +2161,7 @@ public class DDLTask extends Task<DDLWor
       outStream.write(terminator);
 
       ((FSDataOutputStream) outStream).close();
+      outStream = null;
 
     } catch (FileNotFoundException e) {
       LOG.warn("describe database: " + stringifyException(e));
@@ -2132,6 +2171,8 @@ public class DDLTask extends Task<DDLWor
       return 1;
     } catch (Exception e) {
       throw new HiveException(e.toString());
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
     return 0;
   }
@@ -2174,10 +2215,11 @@ public class DDLTask extends Task<DDLWor
     }
 
     // write the results in the file
+    DataOutput outStream = null;
     try {
       Path resFile = new Path(showTblStatus.getResFile());
       FileSystem fs = resFile.getFileSystem(conf);
-      DataOutput outStream = fs.create(resFile);
+      outStream = fs.create(resFile);
 
       Iterator<Table> iterTables = tbls.iterator();
       while (iterTables.hasNext()) {
@@ -2256,6 +2298,7 @@ public class DDLTask extends Task<DDLWor
         outStream.write(terminator);
       }
       ((FSDataOutputStream) outStream).close();
+      outStream = null;
     } catch (FileNotFoundException e) {
       LOG.info("show table status: " + stringifyException(e));
       return 1;
@@ -2264,6 +2307,8 @@ public class DDLTask extends Task<DDLWor
       return 1;
     } catch (Exception e) {
       throw new HiveException(e);
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
     return 0;
   }
@@ -2287,25 +2332,28 @@ public class DDLTask extends Task<DDLWor
     // describe the table - populate the output stream
     Table tbl = db.getTable(tableName, false);
     Partition part = null;
+    DataOutput outStream = null;
     try {
       Path resFile = new Path(descTbl.getResFile());
       if (tbl == null) {
         FileSystem fs = resFile.getFileSystem(conf);
-        DataOutput outStream = (DataOutput) fs.open(resFile);
+        outStream = (DataOutput) fs.open(resFile);
         String errMsg = "Table " + tableName + " does not exist";
         outStream.write(errMsg.getBytes("UTF-8"));
         ((FSDataOutputStream) outStream).close();
+        outStream = null;
         return 0;
       }
       if (descTbl.getPartSpec() != null) {
         part = db.getPartition(tbl, descTbl.getPartSpec(), false);
         if (part == null) {
           FileSystem fs = resFile.getFileSystem(conf);
-          DataOutput outStream = (DataOutput) fs.open(resFile);
+          outStream = (DataOutput) fs.open(resFile);
           String errMsg = "Partition " + descTbl.getPartSpec() + " for table "
               + tableName + " does not exist";
           outStream.write(errMsg.getBytes("UTF-8"));
           ((FSDataOutputStream) outStream).close();
+          outStream = null;
           return 0;
         }
         tbl = part.getTable();
@@ -2316,6 +2364,8 @@ public class DDLTask extends Task<DDLWor
     } catch (IOException e) {
       LOG.info("describe table: " + stringifyException(e));
       return 1;
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
 
     try {
@@ -2324,7 +2374,7 @@ public class DDLTask extends Task<DDLWor
 
       Path resFile = new Path(descTbl.getResFile());
       FileSystem fs = resFile.getFileSystem(conf);
-      DataOutput outStream = fs.create(resFile);
+      outStream = fs.create(resFile);
 
       if (colPath.equals(tableName)) {
         if (!descTbl.isFormatted()) {
@@ -2380,6 +2430,7 @@ public class DDLTask extends Task<DDLWor
 
       LOG.info("DDLTask: written data for " + tbl.getTableName());
       ((FSDataOutputStream) outStream).close();
+      outStream = null;
 
     } catch (FileNotFoundException e) {
       LOG.info("describe table: " + stringifyException(e));
@@ -2389,6 +2440,8 @@ public class DDLTask extends Task<DDLWor
       return 1;
     } catch (Exception e) {
       throw new HiveException(e);
+    } finally {
+      IOUtils.closeStream((FSDataOutputStream) outStream);
     }
 
     return 0;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileInputFormat.java?rev=1148921&r1=1148920&r2=1148921&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileInputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileInputFormat.java Wed Jul 20
19:54:14 2011
@@ -64,12 +64,18 @@ public class RCFileInputFormat<K extends
       return false;
     }
     for (int fileId = 0; fileId < files.size(); fileId++) {
+      RCFile.Reader reader = null;
       try {
-        RCFile.Reader reader = new RCFile.Reader(fs, files.get(fileId)
+        reader = new RCFile.Reader(fs, files.get(fileId)
             .getPath(), conf);
         reader.close();
+        reader = null;
       } catch (IOException e) {
         return false;
+      } finally {
+        if (null != reader) {
+          reader.close();
+        }
       }
     }
     return true;



Mime
View raw message