accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject [5/5] git commit: ACCUMULO-1452 merged branches to master, fixed test
Date Fri, 13 Sep 2013 16:26:21 GMT
ACCUMULO-1452 merged branches to master, fixed test


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/eb5e5a48
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/eb5e5a48
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/eb5e5a48

Branch: refs/heads/master
Commit: eb5e5a4890c1346bf59be3327b442be76d734a1b
Parents: 886e56a
Author: Eric Newton <eric.newton@gmail.com>
Authored: Fri Sep 13 12:26:24 2013 -0400
Committer: Eric Newton <eric.newton@gmail.com>
Committed: Fri Sep 13 12:26:24 2013 -0400

----------------------------------------------------------------------
 .../minicluster/MiniAccumuloCluster.java        | 19 ++++++--
 .../minicluster/MiniAccumuloConfig.java         |  2 +-
 .../accumulo/server/tabletserver/Tablet.java    |  8 +---
 .../accumulo/test/functional/CleanTmpIT.java    | 50 +++++++++++++++-----
 4 files changed, 57 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/eb5e5a48/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
index 473fe44..2e061dd 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloCluster.java
@@ -24,6 +24,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.InetSocketAddress;
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -58,6 +59,7 @@ import org.apache.accumulo.start.Main;
 import org.apache.accumulo.start.classloader.vfs.MiniDFSUtil;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.zookeeper.server.ZooKeeperServerMain;
@@ -124,6 +126,7 @@ public class MiniAccumuloCluster {
   private Set<Pair<ServerType,Integer>> debugPorts = new HashSet<Pair<ServerType,Integer>>();
 
   private File zooCfgFile;
+  private String dfsUri;
 
   public List<LogWriter> getLogWriters() {
     return logWriters;
@@ -245,16 +248,18 @@ public class MiniAccumuloCluster {
         System.setProperty("test.build.data", oldTestBuildData);
       miniDFS.waitClusterUp();
       InetSocketAddress dfsAddress = miniDFS.getNameNode().getNameNodeAddress();
-      String uri = "hdfs://" + dfsAddress.getHostName() + ":" + dfsAddress.getPort();
+      dfsUri = "hdfs://" + dfsAddress.getHostName() + ":" + dfsAddress.getPort();
       File coreFile = new File(config.getConfDir(), "core-site.xml");
-      writeConfig(coreFile, Collections.singletonMap("fs.default.name", uri).entrySet());
+      writeConfig(coreFile, Collections.singletonMap("fs.default.name", dfsUri).entrySet());
       File hdfsFile = new File(config.getConfDir(), "hdfs-site.xml");
       writeConfig(hdfsFile, conf);
 
       Map<String,String> siteConfig = config.getSiteConfig();
-      siteConfig.put(Property.INSTANCE_DFS_URI.getKey(), uri);
+      siteConfig.put(Property.INSTANCE_DFS_URI.getKey(), dfsUri);
       siteConfig.put(Property.INSTANCE_DFS_DIR.getKey(), "/accumulo");
       config.setSiteConfig(siteConfig);
+    } else {
+      dfsUri = "file://";
     }
 
     File siteFile = new File(config.getConfDir(), "accumulo-site.xml");
@@ -501,4 +506,12 @@ public class MiniAccumuloCluster {
     Instance instance = new ZooKeeperInstance(this.getInstanceName(), this.getZooKeepers());
     return instance.getConnector(user, new PasswordToken(passwd));
   }
+  
+  public FileSystem getFileSystem() {
+    try {
+      return FileSystem.get(new URI(dfsUri), new Configuration());
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/eb5e5a48/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
----------------------------------------------------------------------
diff --git a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
index f8ad3af..0b6c42c 100644
--- a/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
+++ b/minicluster/src/main/java/org/apache/accumulo/minicluster/MiniAccumuloConfig.java
@@ -265,7 +265,7 @@ public class MiniAccumuloConfig {
     return zooKeeperDir;
   }
 
-  File getAccumuloDir() {
+  public File getAccumuloDir() {
     return accumuloDir;
   }
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/eb5e5a48/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
index d33b26c..4e06c1d 100644
--- a/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
+++ b/server/src/main/java/org/apache/accumulo/server/tabletserver/Tablet.java
@@ -1347,7 +1347,6 @@ public class Tablet {
     configObserver.propertiesChanged();
     
     tabletResources.setTablet(this, acuTableConf);
-    
     if (!logEntries.isEmpty()) {
       log.info("Starting Write-Ahead Log recovery for " + this.extent);
       final long[] count = new long[2];
@@ -1428,12 +1427,6 @@ public class Tablet {
       // look for any temp files hanging around
       removeOldTemporaryFiles();
     }
-
-    // look for hints of a failure on the previous tablet server
-    if (!logEntries.isEmpty() || needsMajorCompaction(MajorCompactionReason.NORMAL)) {
-      // look for any temp files hanging around
-      removeOldTemporaryFiles();
-    }
     
     log.log(TLevel.TABLET_HIST, extent + " opened ");
   }
@@ -1443,6 +1436,7 @@ public class Tablet {
     try {
       for (FileStatus tmp : fs.globStatus(new Path(location, "*_tmp"))){
         try {
+          log.debug("Removing old temp file " + tmp.getPath());
           fs.delete(tmp.getPath());
         } catch (IOException ex) {
           log.error("Unable to remove old temp file " + tmp.getPath() + ": " + ex);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/eb5e5a48/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java b/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
index 93b21f1..c4fb6df 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
@@ -16,20 +16,43 @@
  */
 package org.apache.accumulo.test.functional;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
 
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.BatchWriterConfig;
 import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.minicluster.MiniAccumuloConfig;
+import org.apache.accumulo.minicluster.ProcessReference;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.junit.Test;
 
-public class CleanTmpIT extends SimpleMacIT {
+public class CleanTmpIT extends ConfigurableMacIT {
   
-  @Test(timeout = 2 * 60 * 1000)
+  @Override
+  public void configure(MiniAccumuloConfig cfg) {
+    Map<String, String> props = new HashMap<String, String>();
+    props.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "3s");
+    cfg.setSiteConfig(props);
+    cfg.setNumTservers(1);
+    cfg.useMiniDFS(true);
+  }
+
+  @Test(timeout = 4 * 60 * 1000)
   public void test() throws Exception {
     Connector c = getConnector();
     // make a table
@@ -41,15 +64,20 @@ public class CleanTmpIT extends SimpleMacIT {
     m.put("cf", "cq", "value");
     bw.addMutation(m);
     bw.close();
-    // take it offline
-    c.tableOperations().offline(tableName);
     
     // create a fake _tmp file in its directory
     String id = c.tableOperations().tableIdMap().get(tableName);
-    File tmp = File.createTempFile("junk", "_tmp", new File(getCluster().getConfig().getDir(),
"accumulo/tables/" + id + "/default_tablet"));
-    assertTrue(tmp.exists());
-    c.tableOperations().online(tableName);
-    UtilWaitThread.sleep(1000);
-    assertFalse(tmp.exists());
+    FileSystem fs = getCluster().getFileSystem();
+    Path tmp = new Path(getCluster().getConfig().getAccumuloDir().getPath() + "/tables/"
+ id + "/default_tablet/junk.rf_tmp");
+    fs.create(tmp).close();
+    for (ProcessReference tserver: getCluster().getProcesses().get(ServerType.TABLET_SERVER))
{
+      getCluster().killProcess(ServerType.TABLET_SERVER, tserver);
+    }
+    getCluster().start();
+    
+    Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
+    for (@SuppressWarnings("unused") Entry<Key,Value> entry : scanner)
+      ;
+    assertFalse(!fs.exists(tmp));
   }
 }


Mime
View raw message