hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1552855 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql: Context.java plan/ConditionalResolverCommonJoin.java
Date Sat, 21 Dec 2013 01:17:38 GMT
Author: hashutosh
Date: Sat Dec 21 01:17:37 2013
New Revision: 1552855

URL: http://svn.apache.org/r1552855
Log:
HIVE-3616 : Use Paths consistently (Ashutosh Chauhan via Harish Butani)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverCommonJoin.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java?rev=1552855&r1=1552854&r2=1552855&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java Sat Dec 21 01:17:37 2013
@@ -78,7 +78,7 @@ public class Context {
   private final String scratchDirPermission;
 
   // Keeps track of scratch directories created for different scheme/authority
-  private final Map<String, String> fsScratchDirs = new HashMap<String, String>();
+  private final Map<String, Path> fsScratchDirs = new HashMap<String, Path>();
 
   private final Configuration conf;
   protected int pathid = 10000;
@@ -189,11 +189,11 @@ public class Context {
    * @param mkdir create the directory if true
    * @param scratchDir path of tmp directory
    */
-  private String getScratchDir(String scheme, String authority,
+  private Path getScratchDir(String scheme, String authority,
                                boolean mkdir, String scratchDir) {
 
     String fileSystem =  scheme + ":" + authority;
-    String dir = fsScratchDirs.get(fileSystem + "-" + TaskRunner.getTaskRunnerID());
+    Path dir = fsScratchDirs.get(fileSystem + "-" + TaskRunner.getTaskRunnerID());
 
     if (dir == null) {
       Path dirPath = new Path(scheme, authority,
@@ -216,7 +216,7 @@ public class Context {
           throw new RuntimeException (e);
         }
       }
-      dir = dirPath.toString();
+      dir = dirPath;
       fsScratchDirs.put(fileSystem + "-" + TaskRunner.getTaskRunnerID(), dir);
 
     }
@@ -227,7 +227,7 @@ public class Context {
   /**
    * Create a local scratch directory on demand and return it.
    */
-  public String getLocalScratchDir(boolean mkdir) {
+  public Path getLocalScratchDir(boolean mkdir) {
     try {
       FileSystem fs = FileSystem.getLocal(conf);
       URI uri = fs.getUri();
@@ -243,7 +243,7 @@ public class Context {
    * Create a map-reduce scratch directory on demand and return it.
    *
    */
-  public String getMRScratchDir() {
+  public Path getMRScratchDir() {
 
     // if we are executing entirely on the client side - then
     // just (re)use the local scratch directory
@@ -254,7 +254,7 @@ public class Context {
     try {
       Path dir = FileUtils.makeQualified(nonLocalScratchPath, conf);
       URI uri = dir.toUri();
-      String newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(),
+      Path newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(),
                            !explain, uri.getPath());
       LOG.info("New scratch dir is " + newScratchDir);
       return newScratchDir;
@@ -266,7 +266,7 @@ public class Context {
     }
   }
 
-  private String getExternalScratchDir(URI extURI) {
+  private Path getExternalScratchDir(URI extURI) {
     return getScratchDir(extURI.getScheme(), extURI.getAuthority(),
                          !explain, nonLocalScratchPath.toUri().getPath());
   }
@@ -275,9 +275,9 @@ public class Context {
    * Remove any created scratch directories.
    */
   private void removeScratchDir() {
-    for (Map.Entry<String, String> entry : fsScratchDirs.entrySet()) {
+    for (Map.Entry<String, Path> entry : fsScratchDirs.entrySet()) {
       try {
-        Path p = new Path(entry.getValue());
+        Path p = entry.getValue();
         p.getFileSystem(conf).delete(p, true);
       } catch (Exception e) {
         LOG.warn("Error Removing Scratch: "
@@ -328,7 +328,7 @@ public class Context {
    */
   public String localizeMRTmpFileURI(String originalURI) {
     Path o = new Path(originalURI);
-    Path mrbase = new Path(getMRScratchDir());
+    Path mrbase = getMRScratchDir();
 
     URI relURI = mrbase.toUri().relativize(o.toUri());
     if (relURI.equals(o.toUri())) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverCommonJoin.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverCommonJoin.java?rev=1552855&r1=1552854&r2=1552855&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverCommonJoin.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverCommonJoin.java
Sat Dec 21 01:17:37 2013
@@ -48,8 +48,8 @@ public class ConditionalResolverCommonJo
     HashMap<String, Long> aliasToKnownSize;
     private Task<? extends Serializable> commonJoinTask;
 
-    private String localTmpDir;
-    private String hdfsTmpDir;
+    private Path localTmpDir;
+    private Path hdfsTmpDir;
 
     public ConditionalResolverCommonJoinCtx() {
     }
@@ -86,19 +86,19 @@ public class ConditionalResolverCommonJo
       this.pathToAliases = pathToAliases;
     }
 
-    public String getLocalTmpDir() {
+    public Path getLocalTmpDir() {
       return localTmpDir;
     }
 
-    public void setLocalTmpDir(String localTmpDir) {
+    public void setLocalTmpDir(Path localTmpDir) {
       this.localTmpDir = localTmpDir;
     }
 
-    public String getHdfsTmpDir() {
+    public Path getHdfsTmpDir() {
       return hdfsTmpDir;
     }
 
-    public void setHdfsTmpDir(String hdfsTmpDir) {
+    public void setHdfsTmpDir(Path hdfsTmpDir) {
       this.hdfsTmpDir = hdfsTmpDir;
     }
   }
@@ -155,8 +155,8 @@ public class ConditionalResolverCommonJo
   private String resolveMapJoinTask(
       HashMap<String, ArrayList<String>> pathToAliases,
       HashMap<String, Task<? extends Serializable>> aliasToTask,
-      HashMap<String, Long> aliasToKnownSize, String hdfsTmpDir,
-      String localTmpDir, HiveConf conf) {
+      HashMap<String, Long> aliasToKnownSize, Path hdfsTmpDir,
+      Path localTmpDir, HiveConf conf) {
 
     String bigTableFileAlias = null;
     long smallTablesFileSizeSum = 0;
@@ -175,7 +175,7 @@ public class ConditionalResolverCommonJo
           .entrySet()) {
         String p = oneEntry.getKey();
         // this path is intermediate data
-        if (p.startsWith(hdfsTmpDir) || p.startsWith(localTmpDir)) {
+        if (p.startsWith(hdfsTmpDir.toString()) || p.startsWith(localTmpDir.toString()))
{
           ArrayList<String> aliasArray = oneEntry.getValue();
           if (aliasArray.size() <= 0) {
             continue;



Mime
View raw message