accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject svn commit: r1438335 - in /accumulo/trunk/server/src/main/java/org/apache/accumulo/server: Accumulo.java client/ClientServiceHandler.java master/Master.java master/balancer/TableLoadBalancer.java
Date Fri, 25 Jan 2013 06:17:25 GMT
Author: ctubbsii
Date: Fri Jan 25 06:17:25 2013
New Revision: 1438335

URL: http://svn.apache.org/viewvc?rev=1438335&view=rev
Log:
ACCUMULO-928 Use the ServerConfiguration object where available, instead of directly calling
HdfsZooInstance
ACCUMULO-923 Made error messages more descriptive

Modified:
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/Accumulo.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
    accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/Accumulo.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/Accumulo.java?rev=1438335&r1=1438334&r2=1438335&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/Accumulo.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/Accumulo.java Fri Jan 25
06:17:25 2013
@@ -112,7 +112,7 @@ public class Accumulo {
     DOMConfigurator.configureAndWatch(logConfig, 5000);
     
     log.info(application + " starting");
-    log.info("Instance " + HdfsZooInstance.getInstance().getInstanceID());
+    log.info("Instance " + config.getInstance().getInstanceID());
     int dataVersion = Accumulo.getAccumuloPersistentVersion(fs);
     log.info("Data Version " + dataVersion);
     Accumulo.waitForZookeeperAndHdfs(fs);
@@ -167,7 +167,7 @@ public class Accumulo {
       }
     }, 1000, 10 * 1000);
   }
-
+  
   public static String getLocalAddress(String[] args) throws UnknownHostException {
     InetAddress result = InetAddress.getLocalHost();
     for (int i = 0; i < args.length - 1; i++) {

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java?rev=1438335&r1=1438334&r2=1438335&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/client/ClientServiceHandler.java
Fri Jan 25 06:17:25 2013
@@ -51,7 +51,6 @@ import org.apache.accumulo.start.classlo
 import org.apache.log4j.Logger;
 import org.apache.thrift.TException;
 
-
 public class ClientServiceHandler implements ClientService.Iface {
   private static final Logger log = Logger.getLogger(ClientServiceHandler.class);
   private static Authenticator authenticator = ZKAuthenticator.getInstance();
@@ -64,11 +63,11 @@ public class ClientServiceHandler implem
   }
   
   protected String checkTableId(String tableName, TableOperation operation) throws ThriftTableOperationException
{
-    String tableId = Tables.getNameToIdMap(HdfsZooInstance.getInstance()).get(tableName);
+    String tableId = Tables.getNameToIdMap(instance).get(tableName);
     if (tableId == null) {
       // maybe the table exist, but the cache was not updated yet... so try to clear the
cache and check again
-      Tables.clearCache(HdfsZooInstance.getInstance());
-      tableId = Tables.getNameToIdMap(HdfsZooInstance.getInstance()).get(tableName);
+      Tables.clearCache(instance);
+      tableId = Tables.getNameToIdMap(instance).get(tableName);
       if (tableId == null)
         throw new ThriftTableOperationException(null, tableName, operation, TableOperationExceptionType.NOTFOUND,
null);
     }
@@ -77,12 +76,12 @@ public class ClientServiceHandler implem
   
   @Override
   public String getInstanceId() {
-    return HdfsZooInstance.getInstance().getInstanceID();
+    return instance.getInstanceID();
   }
   
   @Override
   public String getRootTabletLocation() {
-    return HdfsZooInstance.getInstance().getRootTabletLocation();
+    return instance.getRootTabletLocation();
   }
   
   @Override
@@ -255,9 +254,9 @@ public class ClientServiceHandler implem
       if (!authenticator.hasSystemPermission(credentials, credentials.getUser(), SystemPermission.SYSTEM))
         throw new AccumuloSecurityException(credentials.getUser(), SecurityErrorCode.PERMISSION_DENIED);
       return transactionWatcher.run(Constants.BULK_ARBITRATOR_TYPE, tid, new Callable<List<String>>()
{
+        @Override
         public List<String> call() throws Exception {
-          return BulkImporter.bulkLoad(new ServerConfiguration(instance).getConfiguration(),
instance, credentials, tid, tableId, files, errorDir,
-              setTime);
+          return BulkImporter.bulkLoad(new ServerConfiguration(instance).getConfiguration(),
instance, credentials, tid, tableId, files, errorDir, setTime);
         }
       });
     } catch (AccumuloSecurityException e) {

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1438335&r1=1438334&r2=1438335&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java (original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/Master.java Fri
Jan 25 06:17:25 2013
@@ -176,7 +176,6 @@ import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.data.ACL;
 import org.apache.zookeeper.data.Stat;
 
-
 /**
  * The Master is responsible for assigning and balancing tablets to tablet servers.
  * 
@@ -221,7 +220,7 @@ public class Master implements LiveTServ
       .unmodifiableSortedMap(new TreeMap<TServerInstance,TabletServerStatus>());
   
   private final Set<String> recoveriesInProgress = Collections.synchronizedSet(new
HashSet<String>());
-
+  
   synchronized private MasterState getMasterState() {
     return state;
   }
@@ -233,15 +232,14 @@ public class Master implements LiveTServ
   static final boolean X = true;
   static final boolean _ = false;
   static final boolean transitionOK[][] = {
-      //                            INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT
STOP
-      /* INITIAL                 */{X,      X,        _,        _,     _,          _,   
      X},
-      /* HAVE_LOCK               */{_,      X,        X,        X,     _,          _,   
      X},
-      /* SAFE_MODE               */{_,      _,        X,        X,     X,          _,   
      X},
-      /* NORMAL                  */{_,      _,        X,        X,     X,          _,   
      X},
-      /* UNLOAD_METADATA_TABLETS */{_,      _,        X,        X,     X,          X,   
      X},
-      /* UNLOAD_ROOT_TABLET      */{_,      _,        _,        _,     _,          X,   
      X},
-      /* STOP                    */{_,      _,        _,        _,     _,          _,   
      X},
-      };
+      // INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT STOP
+      /* INITIAL */{X, X, _, _, _, _, X},
+      /* HAVE_LOCK */{_, X, X, X, _, _, X},
+      /* SAFE_MODE */{_, _, X, X, X, _, X},
+      /* NORMAL */{_, _, X, X, X, _, X},
+      /* UNLOAD_METADATA_TABLETS */{_, _, X, X, X, X, X},
+      /* UNLOAD_ROOT_TABLET */{_, _, _, _, _, X, X},
+      /* STOP */{_, _, _, _, _, _, X},};
   
   synchronized private void setMasterState(MasterState newState) {
     if (state.equals(newState))
@@ -275,12 +273,11 @@ public class Master implements LiveTServ
     }
   }
   
-
   private void upgradeZookeeper() {
     if (Accumulo.getAccumuloPersistentVersion(fs) == Constants.PREV_DATA_VERSION) {
       try {
         log.info("Upgrading zookeeper");
-
+        
         IZooReaderWriter zoo = ZooReaderWriter.getInstance();
         
         TablePropUtil.setTableProperty(Constants.METADATA_TABLE_ID, Property.TABLE_ITERATOR_PREFIX.getKey()
+ "majc.bulkLoadFilter", "20,"
@@ -289,9 +286,9 @@ public class Master implements LiveTServ
         zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLE_LOCKS, new byte[0],
NodeExistsPolicy.SKIP);
         zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZHDFS_RESERVATIONS, new
byte[0], NodeExistsPolicy.SKIP);
         zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZNEXT_FILE, new byte[]
{'0'}, NodeExistsPolicy.SKIP);
-
+        
         String[] tablePropsToDelete = new String[] {"table.scan.cache.size", "table.scan.cache.enable"};
-
+        
         for (String id : Tables.getIdToNameMap(instance).keySet()) {
           zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id
+ Constants.ZTABLE_FLUSH_ID, "0".getBytes(), NodeExistsPolicy.SKIP);
           zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id
+ Constants.ZTABLE_COMPACT_ID, "0".getBytes(), NodeExistsPolicy.SKIP);
@@ -304,7 +301,7 @@ public class Master implements LiveTServ
         }
         
         setACLs(zoo, ZooUtil.getRoot(instance), ZooUtil.getRoot(instance) + Constants.ZUSERS);
-
+        
       } catch (Exception ex) {
         log.fatal("Error performing upgrade", ex);
         System.exit(1);
@@ -326,11 +323,11 @@ public class Master implements LiveTServ
       }
     }
   }
-
-  private AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
-
-  private ServerConfiguration serverConfig;
-
+  
+  private final AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);
+  
+  private final ServerConfiguration serverConfig;
+  
   private void upgradeMetadata() {
     if (Accumulo.getAccumuloPersistentVersion(fs) == Constants.PREV_DATA_VERSION) {
       if (upgradeMetadataRunning.compareAndSet(false, true)) {
@@ -343,7 +340,7 @@ public class Master implements LiveTServ
               log.info("Adding bulk dir delete entries to !METADATA table for upgrade");
               
               BatchWriter bw = getConnector().createBatchWriter(Constants.METADATA_TABLE_NAME,
new BatchWriterConfig());
-
+              
               FileStatus[] tables = fs.globStatus(new Path(Constants.getTablesDir(getSystemConfiguration())
+ "/*"));
               for (FileStatus tableDir : tables) {
                 FileStatus[] bulkDirs = fs.globStatus(new Path(tableDir.getPath() + "/bulk_*"));
@@ -357,7 +354,7 @@ public class Master implements LiveTServ
               Accumulo.updateAccumuloVersion(fs);
               
               log.info("Upgrade complete");
-
+              
             } catch (Exception ex) {
               log.fatal("Error performing upgrade", ex);
               System.exit(1);
@@ -372,7 +369,7 @@ public class Master implements LiveTServ
       }
     }
   }
-
+  
   private int assignedOrHosted(Text tableId) {
     int result = 0;
     for (TabletGroupWatcher watcher : watchers) {
@@ -535,7 +532,7 @@ public class Master implements LiveTServ
     this.hostname = hostname;
     
     AccumuloConfiguration aconf = serverConfig.getConfiguration();
-
+    
     log.info("Version " + Constants.VERSION);
     log.info("Instance " + instance.getInstanceID());
     ThriftTransportPool.getInstance().setIdleTime(aconf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
@@ -556,7 +553,7 @@ public class Master implements LiveTServ
   private class MasterClientServiceHandler implements MasterClientService.Iface {
     
     protected String checkTableId(String tableName, TableOperation operation) throws ThriftTableOperationException
{
-      final String tableId = Tables.getNameToIdMap(HdfsZooInstance.getInstance()).get(tableName);
+      final String tableId = Tables.getNameToIdMap(getConfiguration().getInstance()).get(tableName);
       if (tableId == null)
         throw new ThriftTableOperationException(null, tableName, operation, TableOperationExceptionType.NOTFOUND,
null);
       return tableId;
@@ -566,7 +563,8 @@ public class Master implements LiveTServ
     public long initiateFlush(TInfo tinfo, AuthInfo c, String tableId) throws ThriftSecurityException,
ThriftTableOperationException, TException {
       verify(c, tableId, TableOperation.FLUSH, check(c, tableId, TablePermission.WRITE) ||
check(c, tableId, TablePermission.ALTER_TABLE));
       
-      String zTablePath = Constants.ZROOT + "/" + HdfsZooInstance.getInstance().getInstanceID()
+ Constants.ZTABLES + "/" + tableId + Constants.ZTABLE_FLUSH_ID;
+      String zTablePath = Constants.ZROOT + "/" + getConfiguration().getInstance().getInstanceID()
+ Constants.ZTABLES + "/" + tableId
+          + Constants.ZTABLE_FLUSH_ID;
       
       IZooReaderWriter zoo = ZooReaderWriter.getInstance();
       byte fid[];
@@ -846,7 +844,7 @@ public class Master implements LiveTServ
         log.info("tablet balancer changed to " + tabletBalancer.getClass().getName());
       }
     }
-
+    
     @Override
     public void removeSystemProperty(TInfo info, AuthInfo c, String property) throws ThriftSecurityException,
TException {
       
@@ -1058,7 +1056,7 @@ public class Master implements LiveTServ
           fate.seedTransaction(opid, new TraceRepo<Master>(new ExportTable(tableName,
tableId, exportDir)), autoCleanup);
           break;
         }
-
+        
         default:
           throw new UnsupportedOperationException();
       }
@@ -1310,6 +1308,7 @@ public class Master implements LiveTServ
       return stats.getLast(tableId);
     }
     
+    @Override
     public void run() {
       
       Thread.currentThread().setName("Watching " + store.name());
@@ -1486,7 +1485,7 @@ public class Master implements LiveTServ
       }
       return result;
     }
-
+    
     private void sendSplitRequest(MergeInfo info, TabletState state, TabletLocationState
tls) {
       // Already split?
       if (!info.getState().equals(MergeState.SPLITTING))
@@ -1566,7 +1565,7 @@ public class Master implements LiveTServ
           if (update != stats.getMergeInfo().getState()) {
             setMergeState(stats.getMergeInfo(), update);
           }
-
+          
           if (update == MergeState.MERGING) {
             try {
               if (stats.getMergeInfo().isDelete()) {
@@ -1584,7 +1583,7 @@ public class Master implements LiveTServ
         }
       }
     }
-
+    
     private void deleteTablets(MergeInfo info) throws AccumuloException {
       KeyExtent range = info.getRange();
       log.debug("Deleting tablets for " + range);
@@ -1638,7 +1637,7 @@ public class Master implements LiveTServ
         } finally {
           bw.close();
         }
-
+        
         if (followingTablet != null) {
           log.debug("Updating prevRow of " + followingTablet + " to " + range.getPrevEndRow());
           bw = conn.createBatchWriter(Constants.METADATA_TABLE_NAME, new BatchWriterConfig());
@@ -1676,7 +1675,7 @@ public class Master implements LiveTServ
       Range scanRange = new Range(KeyExtent.getMetadataEntry(range.getTableId(), start),
false, stopRow, false);
       if (range.isMeta())
         scanRange = scanRange.clip(Constants.METADATA_ROOT_TABLET_KEYSPACE);
-
+      
       BatchWriter bw = null;
       try {
         long fileCount = 0;
@@ -1730,7 +1729,7 @@ public class Master implements LiveTServ
         }
         
         bw.flush();
-
+        
         log.debug("Moved " + fileCount + " files to " + stop);
         
         if (firstPrevRowValue == null) {
@@ -1743,7 +1742,7 @@ public class Master implements LiveTServ
         log.debug("Setting the prevRow for last tablet: " + stop);
         bw.addMutation(updatePrevRow);
         bw.flush();
-
+        
         deleteTablets(scanRange, bw, conn);
         
         // Clean-up the last chopped marker
@@ -1763,7 +1762,7 @@ public class Master implements LiveTServ
           }
       }
     }
-
+    
     private void deleteTablets(Range scanRange, BatchWriter bw, Connector conn) throws TableNotFoundException,
MutationsRejectedException {
       Scanner scanner;
       Mutation m;
@@ -1781,7 +1780,7 @@ public class Master implements LiveTServ
         while (row.hasNext()) {
           Entry<Key,Value> entry = row.next();
           Key key = entry.getKey();
-
+          
           if (m == null)
             m = new Mutation(key.getRow());
           
@@ -1790,7 +1789,7 @@ public class Master implements LiveTServ
         }
         bw.addMutation(m);
       }
-
+      
       bw.flush();
     }
     
@@ -1859,9 +1858,9 @@ public class Master implements LiveTServ
     
   }
   
-
   private class MigrationCleanupThread extends Daemon {
     
+    @Override
     public void run() {
       setName("Migration Cleanup Thread");
       while (stillMaster()) {
@@ -1897,6 +1896,7 @@ public class Master implements LiveTServ
   
   private class StatusThread extends Daemon {
     
+    @Override
     public void run() {
       setName("Status Thread");
       EventCoordinator.Listener eventListener = nextEvent.getListener();
@@ -2109,7 +2109,7 @@ public class Master implements LiveTServ
     }
     return recoveryNeeded;
   }
-
+  
   public void run() throws IOException, InterruptedException, KeeperException {
     final String zroot = ZooUtil.getRoot(instance);
     
@@ -2124,7 +2124,7 @@ public class Master implements LiveTServ
     migrationCleanupThread.start();
     
     tserverSet.startListeningForTabletServerChanges();
-
+    
     // TODO: add shutdown for fate object
     try {
       final AgeOffStore<Master> store = new AgeOffStore<Master>(new org.apache.accumulo.fate.ZooStore<Master>(ZooUtil.getRoot(instance)
+ Constants.ZFATE,
@@ -2161,11 +2161,8 @@ public class Master implements LiveTServ
     });
     
     AuthInfo systemAuths = SecurityConstants.getSystemCredentials();
-    final TabletStateStore stores[] = {
-        new ZooTabletStateStore(new ZooStore(zroot)), 
-        new RootTabletStateStore(instance, systemAuths, this),
-        new MetaDataStateStore(instance, systemAuths, this)
-        };
+    final TabletStateStore stores[] = {new ZooTabletStateStore(new ZooStore(zroot)), new
RootTabletStateStore(instance, systemAuths, this),
+        new MetaDataStateStore(instance, systemAuths, this)};
     watchers.add(new TabletGroupWatcher(stores[2], null));
     watchers.add(new TabletGroupWatcher(stores[1], watchers.get(0)));
     watchers.add(new TabletGroupWatcher(stores[0], watchers.get(1)));
@@ -2206,6 +2203,7 @@ public class Master implements LiveTServ
   private void getMasterLock(final String zMasterLoc) throws KeeperException, InterruptedException
{
     log.info("trying to get master lock");
     LockWatcher masterLockWatcher = new ZooLock.LockWatcher() {
+      @Override
       public void lostLock(LockLossReason reason) {
         Halt.halt("Master lock in zookeeper lost (reason = " + reason + "), exiting!", -1);
       }
@@ -2370,7 +2368,7 @@ public class Master implements LiveTServ
   public FileSystem getFileSystem() {
     return this.fs;
   }
-
+  
   public void updateRecoveryInProgress(String file) {
     recoveriesInProgress.add(file);
   }

Modified: accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java?rev=1438335&r1=1438334&r2=1438335&view=diff
==============================================================================
--- accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
(original)
+++ accumulo/trunk/server/src/main/java/org/apache/accumulo/server/master/balancer/TableLoadBalancer.java
Fri Jan 25 06:17:25 2013
@@ -31,7 +31,6 @@ import org.apache.accumulo.core.client.a
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.server.client.HdfsZooInstance;
 import org.apache.accumulo.server.master.state.TServerInstance;
 import org.apache.accumulo.server.master.state.TabletMigration;
 import org.apache.accumulo.server.security.SecurityConstants;
@@ -73,16 +72,16 @@ public class TableLoadBalancer extends T
             balancer.init(configuration);
           }
         } catch (Exception e) {
-          log.warn("Failed to load table balancer class ", e);
+          log.warn("Failed to load table balancer class " + clazzName + " for table " + table,
e);
         }
       }
     }
     if (balancer == null) {
       try {
         balancer = constructNewBalancerForTable(clazzName, table);
-        log.info("Loaded class : " + clazzName);
+        log.info("Loaded class " + clazzName + " for table " + table);
       } catch (Exception e) {
-        log.warn("Failed to load table balancer class ", e);
+        log.warn("Failed to load table balancer class " + clazzName + " for table " + table,
e);
       }
       
       if (balancer == null) {
@@ -120,7 +119,7 @@ public class TableLoadBalancer extends T
   protected TableOperations getTableOperations() {
     if (tops == null)
       try {
-        tops = HdfsZooInstance.getInstance().getConnector(SecurityConstants.getSystemCredentials()).tableOperations();
+        tops = configuration.getInstance().getConnector(SecurityConstants.getSystemCredentials()).tableOperations();
       } catch (AccumuloException e) {
         log.error("Unable to access table operations from within table balancer", e);
       } catch (AccumuloSecurityException e) {



Mime
View raw message