accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject accumulo git commit: ACCUMULO-2462 Remove confusing indirect use of instance.dfs.uri
Date Thu, 05 Feb 2015 23:00:01 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 8c2294df8 -> d7bb9ac17


ACCUMULO-2462 Remove confusing indirect use of instance.dfs.uri

This removes a confusing indirect usage of the deprecated
instance.dfs.uri property, to resolve relative paths on the command-line
of PrintInfo. This removes the reliance on accumulo-site.xml for
determining which filesystem to use when not specifying a
fully-qualified file name.

There remains some reliance on accumulo-site.xml, however, due to
ACCUMULO-3566. It also doesn't remove the somewhat confusing behavior of
automatically looking at the local filesystem if the default hadoop
filesystem doesn't contain the file specified. To handle that special
case, I print out a message showing the actual file which it resolves
to.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d7bb9ac1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d7bb9ac1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d7bb9ac1

Branch: refs/heads/master
Commit: d7bb9ac1748a8d794282a99045c7104bd7e9513a
Parents: 8c2294d
Author: Christopher Tubbs <ctubbsii@apache.org>
Authored: Thu Feb 5 17:31:07 2015 -0500
Committer: Christopher Tubbs <ctubbsii@apache.org>
Committed: Thu Feb 5 17:31:07 2015 -0500

----------------------------------------------------------------------
 .../accumulo/core/file/rfile/PrintInfo.java      | 19 ++++++-------------
 .../core/file/rfile/bcfile/PrintInfo.java        | 12 ++++--------
 2 files changed, 10 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/d7bb9ac1/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 591d477..37b3962 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -22,7 +22,6 @@ import java.util.Map;
 import java.util.Map.Entry;
 
 import org.apache.accumulo.core.cli.Help;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.DefaultConfiguration;
 import org.apache.accumulo.core.conf.SiteConfiguration;
 import org.apache.accumulo.core.data.ByteSequence;
@@ -31,7 +30,6 @@ import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.file.blockfile.impl.CachableBlockFile;
 import org.apache.accumulo.core.file.rfile.RFile.Reader;
-import org.apache.accumulo.core.volume.VolumeConfiguration;
 import org.apache.accumulo.start.spi.KeywordExecutable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -72,13 +70,7 @@ public class PrintInfo implements KeywordExecutable {
   public void execute(final String[] args) throws Exception {
     Configuration conf = new Configuration();
 
-    AccumuloConfiguration aconf = SiteConfiguration.getInstance(DefaultConfiguration.getInstance());
-    // TODO ACCUMULO-2462 This will only work for RFiles (path only, not URI) in HDFS when
the correct filesystem for the given file
-    // is on Property.INSTANCE_DFS_DIR or, when INSTANCE_DFS_DIR is not defined, is on the
default filesystem
-    // defined in the Hadoop's core-site.xml
-    //
-    // A workaround is to always provide a URI to this class
-    FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem();
+    FileSystem hadoopFs = FileSystem.get(conf);
     FileSystem localFs = FileSystem.getLocal(conf);
     Opts opts = new Opts();
     opts.parseArgs(PrintInfo.class.getName(), args);
@@ -97,14 +89,15 @@ public class PrintInfo implements KeywordExecutable {
       if (arg.contains(":"))
         fs = path.getFileSystem(conf);
       else {
-        // Recommend a URI is given for the above todo reason
         log.warn("Attempting to find file across filesystems. Consider providing URI instead
of path");
         fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
       }
+      System.out.println("Working on: " + path.makeQualified(fs.getUri(), fs.getWorkingDirectory()).toString());
 
-      CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null,
null, aconf);
+      CachableBlockFile.Reader _rdr = new CachableBlockFile.Reader(fs, path, conf, null,
null,
+          SiteConfiguration.getInstance(DefaultConfiguration.getInstance()));
       Reader iter = new RFile.Reader(_rdr);
-      MetricsGatherer<Map<String, ArrayList<VisibilityMetric>>> vmg = new
VisMetricsGatherer();
+      MetricsGatherer<Map<String,ArrayList<VisibilityMetric>>> vmg = new
VisMetricsGatherer();
 
       if (opts.vis || opts.hash)
         iter.registerMetrics(vmg);
@@ -113,7 +106,7 @@ public class PrintInfo implements KeywordExecutable {
       System.out.println();
       org.apache.accumulo.core.file.rfile.bcfile.PrintInfo.main(new String[] {arg});
 
-      Map<String, ArrayList<ByteSequence>> localityGroupCF = null;
+      Map<String,ArrayList<ByteSequence>> localityGroupCF = null;
 
       if (opts.histogram || opts.dump || opts.vis || opts.hash) {
         localityGroupCF = iter.getLocalityGroupCF();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/d7bb9ac1/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
index 9ac1e96..7b2abe8 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/bcfile/PrintInfo.java
@@ -21,22 +21,20 @@ import java.io.PrintStream;
 import java.util.Map.Entry;
 import java.util.Set;
 
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.DefaultConfiguration;
 import org.apache.accumulo.core.conf.SiteConfiguration;
 import org.apache.accumulo.core.file.rfile.bcfile.BCFile.MetaIndexEntry;
-import org.apache.accumulo.core.volume.VolumeConfiguration;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
 public class PrintInfo {
-  public static void printMetaBlockInfo(Configuration conf, FileSystem fs, Path path, AccumuloConfiguration
accumuloConfiguration) throws IOException {
+  public static void printMetaBlockInfo(Configuration conf, FileSystem fs, Path path) throws
IOException {
     FSDataInputStream fsin = fs.open(path);
     BCFile.Reader bcfr = null;
     try {
-      bcfr = new BCFile.Reader(fsin, fs.getFileStatus(path).getLen(), conf, accumuloConfiguration);
+      bcfr = new BCFile.Reader(fsin, fs.getFileStatus(path).getLen(), conf, SiteConfiguration.getInstance(DefaultConfiguration.getInstance()));
 
       Set<Entry<String,MetaIndexEntry>> es = bcfr.metaIndex.index.entrySet();
 
@@ -57,9 +55,7 @@ public class PrintInfo {
 
   public static void main(String[] args) throws Exception {
     Configuration conf = new Configuration();
-    AccumuloConfiguration siteConf = SiteConfiguration.getInstance(DefaultConfiguration.getInstance());
-    // TODO ACCUMULO-2462 not going to operate as expected with volumes when a path, not
URI, is given
-    FileSystem hadoopFs = VolumeConfiguration.getDefaultVolume(conf, siteConf).getFileSystem();
+    FileSystem hadoopFs = FileSystem.get(conf);
     FileSystem localFs = FileSystem.getLocal(conf);
     Path path = new Path(args[0]);
     FileSystem fs;
@@ -67,6 +63,6 @@ public class PrintInfo {
       fs = path.getFileSystem(conf);
     else
       fs = hadoopFs.exists(path) ? hadoopFs : localFs; // fall back to local
-    printMetaBlockInfo(conf, fs, path, siteConf);
+    printMetaBlockInfo(conf, fs, path);
   }
 }


Mime
View raw message