Return-Path: Delivered-To: apmail-hive-commits-archive@www.apache.org Received: (qmail 66546 invoked from network); 4 Mar 2011 14:51:02 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 4 Mar 2011 14:51:02 -0000 Received: (qmail 66717 invoked by uid 500); 4 Mar 2011 14:51:02 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 66669 invoked by uid 500); 4 Mar 2011 14:51:02 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Delivered-To: moderator for commits@hive.apache.org Received: (qmail 56242 invoked by uid 99); 4 Mar 2011 06:43:36 -0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1077829 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/HiveConf.java metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Date: Fri, 04 Mar 2011 06:43:10 -0000 To: commits@hive.apache.org From: cws@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110304064310.46E3C2388A2C@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cws Date: Fri Mar 4 06:43:09 2011 New Revision: 1077829 URL: http://svn.apache.org/viewvc?rev=1077829&view=rev Log: HIVE-2025 Fix TestEmbeddedHiveMetaStore and TestRemoteHiveMetaStore broken by HIVE-2022 (Ning Zhang via cws) Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1077829&r1=1077828&r2=1077829&view=diff ============================================================================== --- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original) +++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Mar 4 06:43:09 2011 @@ -150,6 +150,7 @@ public class HiveConf extends Configurat METASTOREPWD("javax.jdo.option.ConnectionPassword", ""), // Class name of JDO connection url hook METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook", ""), + METASTOREMULTITHREADED("javax.jdo.option.Multithreaded", "true"), // Name of the connection url in the configuration METASTORECONNECTURLKEY("javax.jdo.option.ConnectionURL", ""), // Number of attempts to retry connecting after there is a JDO datastore err Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1077829&r1=1077828&r2=1077829&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Fri Mar 4 06:43:09 2011 @@ -26,9 +26,9 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.Map.Entry; import java.util.Properties; +import java.util.Set; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -69,26 +69,26 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; +import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MDatabase; import org.apache.hadoop.hive.metastore.model.MFieldSchema; +import org.apache.hadoop.hive.metastore.model.MGlobalPrivilege; import org.apache.hadoop.hive.metastore.model.MIndex; import org.apache.hadoop.hive.metastore.model.MOrder; import org.apache.hadoop.hive.metastore.model.MPartition; -import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege; import org.apache.hadoop.hive.metastore.model.MPartitionPrivilege; import org.apache.hadoop.hive.metastore.model.MRole; -import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege; -import org.apache.hadoop.hive.metastore.model.MGlobalPrivilege; import org.apache.hadoop.hive.metastore.model.MRoleMap; import org.apache.hadoop.hive.metastore.model.MSerDeInfo; import org.apache.hadoop.hive.metastore.model.MStorageDescriptor; import org.apache.hadoop.hive.metastore.model.MTable; +import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege; import org.apache.hadoop.hive.metastore.model.MTablePrivilege; import org.apache.hadoop.hive.metastore.model.MType; +import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream; import org.apache.hadoop.hive.metastore.parser.FilterLexer; import org.apache.hadoop.hive.metastore.parser.FilterParser; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree.ANTLRNoCaseStringStream; import org.apache.hadoop.util.StringUtils; /** @@ -405,7 +405,7 @@ public class ObjectStore implements RawS db.setParameters(mdb.getParameters()); return db; } - + /** * Alter the database object in metastore. Currently only the parameters * of the database can be changed. @@ -576,7 +576,6 @@ public class ObjectStore implements RawS public boolean dropType(String typeName) { boolean success = false; - boolean commited = false; try { openTransaction(); Query query = pm.newQuery(MType.class, "name == typeName"); @@ -584,14 +583,15 @@ public class ObjectStore implements RawS query.setUnique(true); MType type = (MType) query.execute(typeName.trim()); pm.retrieve(type); - pm.deletePersistent(type); - commited = commitTransaction(); - success = true; + if (type != null) { + pm.deletePersistent(type); + } + success = commitTransaction(); } catch (JDOObjectNotFoundException e) { - commited = commitTransaction(); + success = commitTransaction(); LOG.debug("type not found " + typeName, e); } finally { - if (!commited) { + if (!success) { rollbackTransaction(); } } @@ -614,7 +614,7 @@ public class ObjectStore implements RawS Map> groupPrivs = principalPrivs.getGroupPrivileges(); putPersistentPrivObjects(mtbl, toPersistPrivObjs, now, groupPrivs, PrincipalType.GROUP); - + Map> rolePrivs = principalPrivs.getRolePrivileges(); putPersistentPrivObjects(mtbl, toPersistPrivObjs, now, rolePrivs, PrincipalType.ROLE); } @@ -631,7 +631,7 @@ public class ObjectStore implements RawS * Convert PrivilegeGrantInfo from privMap to MTablePrivilege, and add all of * them to the toPersistPrivObjs. These privilege objects will be persisted as * part of createTable. - * + * * @param mtbl * @param toPersistPrivObjs * @param now @@ -682,7 +682,7 @@ public class ObjectStore implements RawS if (partGrants != null && partGrants.size() > 0) { pm.deletePersistentAll(partGrants); } - + List partColGrants = listTableAllPartitionColumnGrants(dbName, tableName); if (partColGrants != null && partColGrants.size() > 0) { @@ -700,7 +700,7 @@ public class ObjectStore implements RawS } return success; } - + public Table getTable(String dbName, String tableName) throws MetaException { boolean commited = false; Table tbl = null; @@ -960,7 +960,7 @@ public class ObjectStore implements RawS toPersist.add(partGrant); } } - + if (tabColumnGrants != null) { for (MTableColumnPrivilege col : tabColumnGrants) { MPartitionColumnPrivilege partColumn = new MPartitionColumnPrivilege(col @@ -969,7 +969,7 @@ public class ObjectStore implements RawS .getGrantorType(), col.getGrantOption()); toPersist.add(partColumn); } - + if (toPersist.size() > 0) { pm.makePersistentAll(toPersist); } @@ -996,7 +996,7 @@ public class ObjectStore implements RawS } return part; } - + private MPartition getMPartition(String dbName, String tableName, List part_vals) throws MetaException { MPartition mpart = null; @@ -1068,7 +1068,7 @@ public class ObjectStore implements RawS colNames.add(col.getName()); } String partName = FileUtils.makePartName(colNames, part_vals); - + List partGrants = listPartitionGrants( dbName, tableName, partName); @@ -1134,7 +1134,7 @@ public class ObjectStore implements RawS } } } - + @Override public Partition getPartitionWithAuth(String dbName, String tblName, List partVals, String user_name, List group_names) @@ -1778,7 +1778,7 @@ public class ObjectStore implements RawS } return success; } - + private MRoleMap getMSecurityUserRoleMap(String userName, PrincipalType principalType, String roleName) { MRoleMap mRoleMember = null; @@ -1861,7 +1861,7 @@ public class ObjectStore implements RawS } return success; } - + private List listRoles(String userName, List groupNames) { List ret = new ArrayList(); @@ -1875,7 +1875,7 @@ public class ObjectStore implements RawS } return ret; } - + @SuppressWarnings("unchecked") @Override public List listRoles(String principalName, @@ -1942,7 +1942,7 @@ public class ObjectStore implements RawS .getOwnerName()); return ret; } - + private MRole getMRole(String roleName) { MRole mrole = null; boolean commited = false; @@ -1961,7 +1961,7 @@ public class ObjectStore implements RawS } return mrole; } - + public List listRoleNames() { boolean success = false; try { @@ -1982,7 +1982,7 @@ public class ObjectStore implements RawS } } } - + @Override public PrincipalPrivilegeSet getUserPrivilegeSet(String userName, List groupNames) throws InvalidObjectException, MetaException { @@ -2030,7 +2030,7 @@ public class ObjectStore implements RawS } return ret; } - + public List getDBPrivilege(String dbName, String principalName, PrincipalType principalType) throws InvalidObjectException, MetaException { @@ -2054,7 +2054,7 @@ public class ObjectStore implements RawS return new ArrayList(0); } - + @Override public PrincipalPrivilegeSet getDBPrivilegeSet(String dbName, String userName, List groupNames) throws InvalidObjectException, @@ -2185,7 +2185,7 @@ public class ObjectStore implements RawS } return ret; } - + @Override public PrincipalPrivilegeSet getColumnPrivilegeSet(String dbName, String tableName, String partitionName, String columnName, @@ -2231,7 +2231,7 @@ public class ObjectStore implements RawS } return ret; } - + private List getPartitionPrivilege(String dbName, String tableName, String partName, String principalName, PrincipalType principalType) { @@ -2262,7 +2262,7 @@ public class ObjectStore implements RawS private PrincipalType getPrincipalTypeFromStr(String str) { return str == null ? null : PrincipalType.valueOf(str); } - + private List getTablePrivilege(String dbName, String tableName, String principalName, PrincipalType principalType) { tableName = tableName.toLowerCase().trim(); @@ -2286,11 +2286,11 @@ public class ObjectStore implements RawS } return new ArrayList(0); } - + private List getColumnPrivilege(String dbName, String tableName, String columnName, String partitionName, String principalName, PrincipalType principalType) { - + tableName = tableName.toLowerCase().trim(); dbName = dbName.toLowerCase().trim(); columnName = columnName.toLowerCase().trim(); @@ -2472,7 +2472,7 @@ public class ObjectStore implements RawS userName, principalType, hiveObject.getDbName(), hiveObject .getObjectName(), partObj.getPartitionName(), hiveObject.getColumnName()); - + if (colPrivs != null) { for (MPartitionColumnPrivilege priv : colPrivs) { if (priv.getGrantor().equalsIgnoreCase(grantor)) { @@ -2495,13 +2495,13 @@ public class ObjectStore implements RawS grantOption); persistentObjs.add(mCol); } - + } else { List colPrivs = null; colPrivs = this.listPrincipalTableColumnGrants( userName, principalType, hiveObject.getDbName(), hiveObject .getObjectName(), hiveObject.getColumnName()); - + if (colPrivs != null) { for (MTableColumnPrivilege priv : colPrivs) { if (priv.getGrantor().equalsIgnoreCase(grantor)) { @@ -2539,7 +2539,7 @@ public class ObjectStore implements RawS } return committed; } - + @Override public boolean revokePrivileges(PrivilegeBag privileges) throws InvalidObjectException, MetaException, NoSuchObjectException { @@ -2547,13 +2547,13 @@ public class ObjectStore implements RawS try { openTransaction(); List persistentObjs = new ArrayList(); - + List privilegeList = privileges.getPrivileges(); - + if (privilegeList != null && privilegeList.size() > 0) { Iterator privIter = privilegeList.iterator(); - + while (privIter.hasNext()) { HiveObjectPrivilege privDef = privIter.next(); HiveObjectRef hiveObject = privDef.getHiveObject(); @@ -2585,7 +2585,7 @@ public class ObjectStore implements RawS } } } - + } else if (hiveObject.getObjectType() == HiveObjectType.DATABASE) { MDatabase dbObj = getMDatabase(hiveObject.getDbName()); if (dbObj != null) { @@ -2630,7 +2630,7 @@ public class ObjectStore implements RawS } } } else if (hiveObject.getObjectType() == HiveObjectType.PARTITION) { - + boolean found = false; Table tabObj = this.getTable(hiveObject.getDbName(), hiveObject.getObjectName()); String partName = null; @@ -2664,7 +2664,7 @@ public class ObjectStore implements RawS partName = Warehouse.makePartName(tabObj.getPartitionKeys(), hiveObject.getPartValues()); } - + if (partName != null) { List mSecCol = listPrincipalPartitionColumnGrants( userName, principalType, hiveObject.getDbName(), hiveObject @@ -2718,7 +2718,7 @@ public class ObjectStore implements RawS } } } - + if (persistentObjs.size() > 0) { pm.deletePersistentAll(persistentObjs); } @@ -2730,7 +2730,7 @@ public class ObjectStore implements RawS } return committed; } - + @SuppressWarnings("unchecked") private List listRoleMembers( MRole mRol) { @@ -2756,7 +2756,7 @@ public class ObjectStore implements RawS } return mRoleMemeberList; } - + @SuppressWarnings("unchecked") @Override public List listPrincipalGlobalGrants(String principalName, PrincipalType principalType) { @@ -2809,7 +2809,7 @@ public class ObjectStore implements RawS } return mSecurityDBList; } - + @SuppressWarnings("unchecked") private List listPrincipalAllDBGrant( String principalName, PrincipalType principalType) { @@ -2866,7 +2866,7 @@ public class ObjectStore implements RawS } return mSecurityTabList; } - + @SuppressWarnings("unchecked") public List listTableAllPartitionGrants(String dbName, String tableName) { @@ -2896,7 +2896,7 @@ public class ObjectStore implements RawS } return mSecurityTabPartList; } - + @SuppressWarnings("unchecked") public List listTableAllColumnGrants(String dbName, String tableName) { @@ -2924,7 +2924,7 @@ public class ObjectStore implements RawS } return mTblColPrivilegeList; } - + @SuppressWarnings("unchecked") public List listTableAllPartitionColumnGrants(String dbName, String tableName) { @@ -2952,7 +2952,7 @@ public class ObjectStore implements RawS } return mSecurityColList; } - + @SuppressWarnings("unchecked") public List listPartitionAllColumnGrants(String dbName, String tableName, String partName) { @@ -3006,7 +3006,7 @@ public class ObjectStore implements RawS } } } - + @SuppressWarnings("unchecked") private List listPartitionGrants(String dbName, String tableName, String partName) { @@ -3091,7 +3091,7 @@ public class ObjectStore implements RawS mSecurityTabPartList = (List) query .executeWithArray(principalName, principalType.toString(), tableName, dbName, partName); LOG.debug("Done executing query for listMSecurityPrincipalPartitionGrant"); - + pm.retrieveAll(mSecurityTabPartList); success = commitTransaction(); LOG.debug("Done retrieving all objects for listMSecurityPrincipalPartitionGrant"); @@ -3136,7 +3136,7 @@ public class ObjectStore implements RawS } return mSecurityColList; } - + @SuppressWarnings("unchecked") public List listPrincipalPartitionColumnGrants( String principalName, PrincipalType principalType, String dbName, @@ -3175,7 +3175,7 @@ public class ObjectStore implements RawS } return mSecurityColList; } - + @SuppressWarnings("unchecked") private List listPrincipalAllTableGrants( String principalName, PrincipalType principalType) { @@ -3229,7 +3229,7 @@ public class ObjectStore implements RawS } return mSecurityTabPartList; } - + @SuppressWarnings("unchecked") private List listPrincipalAllTableColumnGrants( String principalName, PrincipalType principalType) { @@ -3255,7 +3255,7 @@ public class ObjectStore implements RawS } return mSecurityColumnList; } - + @SuppressWarnings("unchecked") private List listPrincipalAllPartitionColumnGrants( String principalName, PrincipalType principalType) {