accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1328076 - in /accumulo/branches/1.3: src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java test/system/auto/stress/bigRootTablet.py
Date Thu, 19 Apr 2012 18:41:18 GMT
Author: ecn
Date: Thu Apr 19 18:41:17 2012
New Revision: 1328076

URL: http://svn.apache.org/viewvc?rev=1328076&view=rev
Log:
ACCUMULO-542: just use a scanner to read the root tablet

Added:
    accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py   (with props)
Modified:
    accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
    accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java

Modified: accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java?rev=1328076&r1=1328075&r2=1328076&view=diff
==============================================================================
--- accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
(original)
+++ accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
Thu Apr 19 18:41:17 2012
@@ -1032,10 +1032,6 @@ public class Tablet {
     
     if (extent.equals(Constants.ROOT_TABLET_EXTENT)) {
       return null;
-    } else if (extent.getTableId().toString().equals(Constants.METADATA_TABLE_ID)) {
-      SortedSet<Column> columns = new TreeSet<Column>();
-      columns.add(Constants.METADATA_TIME_COLUMN.toColumn());
-      entries = MetadataTable.getRootMetadataDataEntries(extent, columns, SecurityConstants.systemCredentials);
     } else {
       entries = new TreeMap<Key,Value>();
       Text rowName = extent.getMetadataEntry();
@@ -1077,44 +1073,39 @@ public class Tablet {
       
       SortedMap<Key,Value> datafilesMetadata;
       
-      if (extent.getTableId().toString().equals(Constants.METADATA_TABLE_ID)) {
-        datafilesMetadata = MetadataTable.getRootMetadataDataFileEntries(extent, SecurityConstants.systemCredentials);
+      Text rowName = extent.getMetadataEntry();
+      
+      if (tabletsKeyValues != null && tabletsKeyValues.size() > 0) {
+        datafilesMetadata = new TreeMap<Key,Value>();
+        for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
+          if (entry.getKey().compareRow(rowName) == 0 && entry.getKey().compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY)
== 0) {
+            datafilesMetadata.put(new Key(entry.getKey()), new Value(entry.getValue()));
+          }
+        }
       } else {
         
-        Text rowName = extent.getMetadataEntry();
+        ScannerImpl mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), SecurityConstants.systemCredentials,
Constants.METADATA_TABLE_ID,
+            Constants.NO_AUTHS);
         
-        if (tabletsKeyValues != null && tabletsKeyValues.size() > 0) {
-          datafilesMetadata = new TreeMap<Key,Value>();
-          for (Entry<Key,Value> entry : tabletsKeyValues.entrySet()) {
-            if (entry.getKey().compareRow(rowName) == 0 && entry.getKey().compareColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY)
== 0) {
-              datafilesMetadata.put(new Key(entry.getKey()), new Value(entry.getValue()));
-            }
-          }
-        } else {
-          
-          ScannerImpl mdScanner = new ScannerImpl(HdfsZooInstance.getInstance(), SecurityConstants.systemCredentials,
Constants.METADATA_TABLE_ID,
-              Constants.NO_AUTHS);
-          
-          // Commented out because when no data file is present, each tablet will scan through
metadata table and return nothing
-          // reduced batch size to improve performance
-          // changed here after endKeys were implemented from 10 to 1000
-          mdScanner.setBatchSize(1000);
-          
-          // leave these in, again, now using endKey for safety
-          mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
-          
-          mdScanner.setRange(new Range(rowName));
-          
-          datafilesMetadata = new TreeMap<Key,Value>();
+        // Commented out because when no data file is present, each tablet will scan through
metadata table and return nothing
+        // reduced batch size to improve performance
+        // changed here after endKeys were implemented from 10 to 1000
+        mdScanner.setBatchSize(1000);
+        
+        // leave these in, again, now using endKey for safety
+        mdScanner.fetchColumnFamily(Constants.METADATA_DATAFILE_COLUMN_FAMILY);
+        
+        mdScanner.setRange(new Range(rowName));
+        
+        datafilesMetadata = new TreeMap<Key,Value>();
+        
+        for (Entry<Key,Value> entry : mdScanner) {
           
-          for (Entry<Key,Value> entry : mdScanner) {
-            
-            if (entry.getKey().compareRow(rowName) != 0) {
-              break;
-            }
-            
-            datafilesMetadata.put(new Key(entry.getKey()), new Value(entry.getValue()));
+          if (entry.getKey().compareRow(rowName) != 0) {
+            break;
           }
+          
+          datafilesMetadata.put(new Key(entry.getKey()), new Value(entry.getValue()));
         }
       }
       

Modified: accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1328076&r1=1328075&r2=1328076&view=diff
==============================================================================
--- accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
(original)
+++ accumulo/branches/1.3/src/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
Thu Apr 19 18:41:17 2012
@@ -304,13 +304,13 @@ public class MetadataTable extends org.a
    * convenience method for reading a metadata tablet's data file entries from the root tablet
    * 
    */
-  public static SortedMap<Key,Value> getRootMetadataDataFileEntries(KeyExtent extent,
AuthInfo credentials) {
+  private static SortedMap<Key,Value> getRootMetadataDataFileEntries(KeyExtent extent,
AuthInfo credentials) {
     SortedSet<Column> columns = new TreeSet<Column>();
     columns.add(new Column(TextUtil.getBytes(Constants.METADATA_DATAFILE_COLUMN_FAMILY),
null, null));
     return getRootMetadataDataEntries(extent, columns, credentials);
   }
   
-  public static SortedMap<Key,Value> getRootMetadataDataEntries(KeyExtent extent, SortedSet<Column>
columns, AuthInfo credentials) {
+  private static SortedMap<Key,Value> getRootMetadataDataEntries(KeyExtent extent,
SortedSet<Column> columns, AuthInfo credentials) {
     
     try {
       SortedMap<Key,Value> entries = new TreeMap<Key,Value>();

Added: accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py
URL: http://svn.apache.org/viewvc/accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py?rev=1328076&view=auto
==============================================================================
--- accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py (added)
+++ accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py Thu Apr 19 18:41:17 2012
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import logging
+import unittest
+
+from TestUtils import TestUtilsMixin
+
+log = logging.getLogger('test.auto')
+
+class BigRootTablet(unittest.TestCase, TestUtilsMixin):
+    "ACCUMULO-542: A large root tablet will fail to load if it does't fit in the tserver
scan buffers"
+
+    order = 80
+
+    settings = TestUtilsMixin.settings.copy()
+    settings['table.scan.max.memory'] = '1024'
+    settings['tserver.compaction.major.delay'] = '60m'
+
+    def setUp(self):
+        TestUtilsMixin.setUp(self);
+    
+    def runTest(self):
+	cmd = 'table !METADATA\naddsplits 0 1 2 3 4 5 6 7 8 9 a\n'
+        for i in range(10):
+	    cmd += 'createtable %s\nflush -t !METADATA\n' % i
+        self.shell(self.masterHost(), cmd)
+	self.stop_accumulo()
+	self.start_accumulo()
+
+def suite():
+    result = unittest.TestSuite()
+    result.addTest(BigRootTablet())
+    return result

Propchange: accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: accumulo/branches/1.3/test/system/auto/stress/bigRootTablet.py
------------------------------------------------------------------------------
    svn:executable = *



Mime
View raw message