hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject svn commit: r1590142 - in /hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore: MetaStoreDirectSql.java ObjectStore.java Warehouse.java
Date Fri, 25 Apr 2014 21:00:01 GMT
Author: sershe
Date: Fri Apr 25 21:00:01 2014
New Revision: 1590142

URL: http://svn.apache.org/r1590142
Log:
HIVE-6945 : issues with dropping partitions on Oracle (Sergey Shelukhin, reviewed by Ashutosh
Chauhan)

Modified:
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java?rev=1590142&r1=1590141&r2=1590142&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
(original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
Fri Apr 25 21:00:01 2014
@@ -309,7 +309,7 @@ class MetaStoreDirectSql {
     StringBuilder partSb = new StringBuilder(sbCapacity);
     // Assume db and table names are the same for all partition, that's what we're selecting
for.
     for (Object partitionId : sqlResult) {
-      partSb.append((Long)partitionId).append(",");
+      partSb.append(extractSqlLong(partitionId)).append(",");
     }
     String partIds = trimCommaList(partSb);
     timingTrace(doTrace, queryText, start, queryTime);
@@ -346,10 +346,10 @@ class MetaStoreDirectSql {
     dbName = dbName.toLowerCase();
     for (Object[] fields : sqlResult2) {
       // Here comes the ugly part...
-      long partitionId = (Long)fields[0];
-      Long sdId = (Long)fields[1];
-      Long colId = (Long)fields[2];
-      Long serdeId = (Long)fields[3];
+      long partitionId = extractSqlLong(fields[0]);
+      Long sdId = extractSqlLong(fields[1]);
+      Long colId = extractSqlLong(fields[2]);
+      Long serdeId = extractSqlLong(fields[3]);
       // A partition must have either everything set, or nothing set if it's a view.
       if (sdId == null || colId == null || serdeId == null) {
         if (isView == null) {
@@ -502,7 +502,7 @@ class MetaStoreDirectSql {
       loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() {
         private Long currentListId;
         private List<String> currentList;
-        public void apply(StorageDescriptor t, Object[] fields) {
+        public void apply(StorageDescriptor t, Object[] fields) throws MetaException {
           if (!t.isSetSkewedInfo()) t.setSkewedInfo(new SkewedInfo());
           // Note that this is not a typical list accumulator - there's no call to finalize
           // the last list. Instead we add list to SD first, as well as locally to add elements.
@@ -511,7 +511,7 @@ class MetaStoreDirectSql {
             currentListId = null;
             t.getSkewedInfo().addToSkewedColValues(new ArrayList<String>());
           } else {
-            long fieldsListId = (Long)fields[1];
+            long fieldsListId = extractSqlLong(fields[1]);
             if (currentListId == null || fieldsListId != currentListId) {
               currentList = new ArrayList<String>();
               currentListId = fieldsListId;
@@ -539,7 +539,7 @@ class MetaStoreDirectSql {
       loopJoinOrderedResult(sds, queryText, 0, new ApplyFunc<StorageDescriptor>() {
         private Long currentListId;
         private List<String> currentList;
-        public void apply(StorageDescriptor t, Object[] fields) {
+        public void apply(StorageDescriptor t, Object[] fields) throws MetaException {
           if (!t.isSetSkewedInfo()) {
             SkewedInfo skewedInfo = new SkewedInfo();
             skewedInfo.setSkewedColValueLocationMaps(new HashMap<List<String>, String>());
@@ -552,7 +552,7 @@ class MetaStoreDirectSql {
             currentList = new ArrayList<String>(); // left outer join produced a list
with no values
             currentListId = null;
           } else {
-            long fieldsListId = (Long)fields[1];
+            long fieldsListId = extractSqlLong(fields[1]);
             if (currentListId == null || fieldsListId != currentListId) {
               currentList = new ArrayList<String>();
               currentListId = fieldsListId;
@@ -589,6 +589,14 @@ class MetaStoreDirectSql {
     return orderedResult;
   }
 
+  private Long extractSqlLong(Object obj) throws MetaException {
+    if (obj == null) return null;
+    if (!(obj instanceof Number)) {
+      throw new MetaException("Expected numeric type but got " + obj.getClass().getName());
+    }
+    return ((Number)obj).longValue();
+  }
+
   private void timingTrace(boolean doTrace, String queryText, long start, long queryTime)
{
     if (!doTrace) return;
     LOG.debug("Direct SQL query in " + (queryTime - start) / 1000000.0 + "ms + " +
@@ -604,9 +612,10 @@ class MetaStoreDirectSql {
     if (value instanceof String && ((String)value).length() == 1) {
       c = ((String)value).charAt(0);
     }
+    if (c == null) return null;
     if (c == 'Y') return true;
     if (c == 'N') return false;
-    throw new MetaException("Cannot extrace boolean from column value " + value);
+    throw new MetaException("Cannot extract boolean from column value " + value);
   }
 
   private int extractSqlInt(Object field) {
@@ -621,7 +630,7 @@ class MetaStoreDirectSql {
   }
 
   private abstract class ApplyFunc<Target> {
-    public abstract void apply(Target t, Object[] fields);
+    public abstract void apply(Target t, Object[] fields) throws MetaException;
   }
 
   /**
@@ -655,7 +664,7 @@ class MetaStoreDirectSql {
         if (fields == null) {
           fields = iter.next();
         }
-        long nestedId = (Long)fields[keyIndex];
+        long nestedId = extractSqlLong(fields[keyIndex]);
         if (nestedId < id) throw new MetaException("Found entries for unknown ID " + nestedId);
         if (nestedId > id) break; // fields belong to one of the next entries
         func.apply(entry.getValue(), fields);
@@ -943,7 +952,7 @@ class MetaStoreDirectSql {
     + "\"MAX_COL_LEN\", \"NUM_TRUES\", \"NUM_FALSES\", \"LAST_ANALYZED\" ";
 
   private ColumnStatistics makeColumnStats(
-      List<Object[]> list, ColumnStatisticsDesc csd, int offset) {
+      List<Object[]> list, ColumnStatisticsDesc csd, int offset) throws MetaException
{
     ColumnStatistics result = new ColumnStatistics();
     result.setStatsDesc(csd);
     List<ColumnStatisticsObj> csos = new ArrayList<ColumnStatisticsObj>(list.size());
@@ -951,8 +960,8 @@ class MetaStoreDirectSql {
       // LastAnalyzed is stored per column but thrift has it per several;
       // get the lowest for now as nobody actually uses this field.
       Object laObj = row[offset + 14];
-      if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() >
(Long)laObj)) {
-        csd.setLastAnalyzed((Long)laObj);
+      if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() >
extractSqlLong(laObj))) {
+        csd.setLastAnalyzed(extractSqlLong(laObj));
       }
       ColumnStatisticsData data = new ColumnStatisticsData();
       // see STATS_COLLIST

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1590142&r1=1590141&r2=1590142&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Fri Apr
25 21:00:01 2014
@@ -133,6 +133,7 @@ import org.apache.thrift.TException;
 import org.datanucleus.store.rdbms.exceptions.MissingTableException;
 
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 
 
 /**
@@ -946,6 +947,16 @@ public class ObjectStore implements RawS
     return tables;
   }
 
+  /** Makes shallow copy of a list to avoid DataNucleus mucking with our objects. */
+  private <T> List<T> convertList(List<T> dnList) {
+    return (dnList == null) ? null : Lists.newArrayList(dnList);
+  }
+
+  /** Makes shallow copy of a map to avoid DataNucleus mucking with our objects. */
+  private <K, V> Map<K, V> convertMap(Map<K, V> dnMap) {
+    return (dnMap == null) ? null : Maps.newHashMap(dnMap);
+  }
+
   private Table convertToTable(MTable mtbl) throws MetaException {
     if (mtbl == null) {
       return null;
@@ -964,9 +975,8 @@ public class ObjectStore implements RawS
     return new Table(mtbl.getTableName(), mtbl.getDatabase().getName(), mtbl
         .getOwner(), mtbl.getCreateTime(), mtbl.getLastAccessTime(), mtbl
         .getRetention(), convertToStorageDescriptor(mtbl.getSd()),
-        convertToFieldSchemas(mtbl.getPartitionKeys()), mtbl.getParameters(),
-        mtbl.getViewOriginalText(), mtbl.getViewExpandedText(),
-        tableType);
+        convertToFieldSchemas(mtbl.getPartitionKeys()), convertMap(mtbl.getParameters()),
+        mtbl.getViewOriginalText(), mtbl.getViewExpandedText(), tableType);
   }
 
   private MTable convertToMTable(Table tbl) throws InvalidObjectException,
@@ -1057,8 +1067,7 @@ public class ObjectStore implements RawS
     if (ms == null) {
       throw new MetaException("Invalid SerDeInfo object");
     }
-    return new SerDeInfo(ms.getName(), ms.getSerializationLib(), ms
-        .getParameters());
+    return new SerDeInfo(ms.getName(), ms.getSerializationLib(), convertMap(ms.getParameters()));
   }
 
   private MSerDeInfo converToMSerDeInfo(SerDeInfo ms) throws MetaException {
@@ -1094,9 +1103,9 @@ public class ObjectStore implements RawS
     StorageDescriptor sd = new StorageDescriptor(noFS ? null : convertToFieldSchemas(mFieldSchemas),
         msd.getLocation(), msd.getInputFormat(), msd.getOutputFormat(), msd
         .isCompressed(), msd.getNumBuckets(), converToSerDeInfo(msd
-        .getSerDeInfo()), msd.getBucketCols(), convertToOrders(msd
-        .getSortCols()), msd.getParameters());
-    SkewedInfo skewedInfo = new SkewedInfo(msd.getSkewedColNames(),
+        .getSerDeInfo()), convertList(msd.getBucketCols()), convertToOrders(msd
+        .getSortCols()), convertMap(msd.getParameters()));
+    SkewedInfo skewedInfo = new SkewedInfo(convertList(msd.getSkewedColNames()),
         convertToSkewedValues(msd.getSkewedColValues()),
         covertToSkewedMap(msd.getSkewedColValueLocationMaps()));
     sd.setSkewedInfo(skewedInfo);
@@ -1415,10 +1424,10 @@ public class ObjectStore implements RawS
     if (mpart == null) {
       return null;
     }
-    return new Partition(mpart.getValues(), mpart.getTable().getDatabase()
+    return new Partition(convertList(mpart.getValues()), mpart.getTable().getDatabase()
         .getName(), mpart.getTable().getTableName(), mpart.getCreateTime(),
         mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd()),
-        mpart.getParameters());
+        convertMap(mpart.getParameters()));
   }
 
   private Partition convertToPart(String dbName, String tblName, MPartition mpart)
@@ -1426,9 +1435,9 @@ public class ObjectStore implements RawS
     if (mpart == null) {
       return null;
     }
-    return new Partition(mpart.getValues(), dbName, tblName, mpart.getCreateTime(),
-        mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd(), false),
-        mpart.getParameters());
+    return new Partition(convertList(mpart.getValues()), dbName, tblName,
+        mpart.getCreateTime(), mpart.getLastAccessTime(),
+        convertToStorageDescriptor(mpart.getSd(), false), convertMap(mpart.getParameters()));
   }
 
   @Override

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java?rev=1590142&r1=1590141&r2=1590142&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/Warehouse.java Fri Apr
25 21:00:01 2014
@@ -556,7 +556,15 @@ public class Warehouse {
   public static String makePartName(List<FieldSchema> partCols,
       List<String> vals, String defaultStr) throws MetaException {
     if ((partCols.size() != vals.size()) || (partCols.size() == 0)) {
-      throw new MetaException("Invalid partition key & values");
+      String errorStr = "Invalid partition key & values; keys [";
+      for (FieldSchema fs : partCols) {
+        errorStr += (fs.getName() + ", ");
+      }
+      errorStr += "], values [";
+      for (String val : vals) {
+        errorStr += (val + ", ");
+      }
+      throw new MetaException(errorStr + "]");
     }
     List<String> colNames = new ArrayList<String>();
     for (FieldSchema col: partCols) {



Mime
View raw message