carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [07/50] [abbrv] carbondata git commit: [CARBONDATA-1236] Support absolute path without scheme in loading - change in logic
Date Wed, 12 Jul 2017 15:04:43 GMT
[CARBONDATA-1236] Support absolute path without scheme in loading - change in logic


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c671c5b6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c671c5b6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c671c5b6

Branch: refs/heads/datamap
Commit: c671c5b60ad4989643e9b6f92540b60c2c540145
Parents: 126a041
Author: dhatchayani <dhatcha.official@gmail.com>
Authored: Thu Jun 29 17:59:33 2017 +0530
Committer: dhatchayani <dhatcha.official@gmail.com>
Committed: Thu Jun 29 18:02:50 2017 +0530

----------------------------------------------------------------------
 .../apache/carbondata/core/util/CarbonUtil.java | 50 +++++------
 .../carbondata/core/util/CarbonUtilTest.java    | 89 +++++++++++++++++++-
 2 files changed, 112 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/c671c5b6/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 6e0348b..8298600 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -697,32 +697,32 @@ public final class CarbonUtil {
    */
   public static String checkAndAppendHDFSUrl(String filePath) {
     String currentPath = filePath;
-    if (null != filePath && filePath.length() != 0
-        && FileFactory.getFileType(filePath) != FileFactory.FileType.HDFS
-        && FileFactory.getFileType(filePath) != FileFactory.FileType.VIEWFS) {
-      if (!filePath.startsWith("/")) {
-        filePath = "/" + filePath;
-      }
-      String baseDFSUrl = CarbonProperties.getInstance()
-          .getProperty(CarbonCommonConstants.CARBON_DDL_BASE_HDFS_URL);
-      String dfsUrl = conf.get(FS_DEFAULT_FS);
-      if (null != baseDFSUrl) {
-        if (!baseDFSUrl.startsWith("/")) {
-          baseDFSUrl = "/" + baseDFSUrl;
-        }
-        if (null != dfsUrl && (dfsUrl.startsWith(HDFS_PREFIX) || dfsUrl
-            .startsWith(VIEWFS_PREFIX))) {
-          baseDFSUrl = dfsUrl + baseDFSUrl;
-        }
-        if (baseDFSUrl.endsWith("/")) {
-          baseDFSUrl = baseDFSUrl.substring(0, baseDFSUrl.length() - 1);
-        }
-        currentPath = baseDFSUrl + filePath;
-      } else {
-        currentPath = dfsUrl + filePath;
-      }
+    String defaultFsUrl = conf.get(FS_DEFAULT_FS);
+    String baseDFSUrl = CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.CARBON_DDL_BASE_HDFS_URL, "");
+    if (checkIfPrefixExists(filePath)) {
+      return currentPath;
     }
-    return currentPath;
+    if (baseDFSUrl.endsWith("/")) {
+      baseDFSUrl = baseDFSUrl.substring(0, baseDFSUrl.length() - 1);
+    }
+    if (!filePath.startsWith("/")) {
+      filePath = "/" + filePath;
+    }
+    currentPath = baseDFSUrl + filePath;
+    if (checkIfPrefixExists(currentPath)) {
+      return currentPath;
+    }
+    if (defaultFsUrl == null) {
+      return currentPath;
+    }
+    return defaultFsUrl + currentPath;
+  }
+
+  private static boolean checkIfPrefixExists(String path) {
+    final String lowerPath = path.toLowerCase();
+    return lowerPath.startsWith(HDFS_PREFIX) || lowerPath.startsWith(VIEWFS_PREFIX) || lowerPath
+        .startsWith("file://") || lowerPath.startsWith(ALLUXIO_PREFIX);
   }
 
   public static String getCarbonStorePath() {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c671c5b6/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index b42c57e..776059f 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -342,7 +342,7 @@ public class CarbonUtilTest {
       }
     };
     String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("../core/src/test/resources/testDatabase");
-    assertEquals(hdfsURL, "/BASE_URL/../core/src/test/resources/testDatabase");
+    assertEquals(hdfsURL, "file:///BASE_URL/../core/src/test/resources/testDatabase");
   }
 
   @Test public void testToCheckAndAppendHDFSUrlWithBlackSlash() {
@@ -357,7 +357,7 @@ public class CarbonUtilTest {
       }
     };
     String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("../core/src/test/resources/testDatabase");
-    assertEquals(hdfsURL, "/BASE_URL/../core/src/test/resources/testDatabase");
+    assertEquals(hdfsURL, "file:///BASE_URL/../core/src/test/resources/testDatabase");
   }
 
   @Test public void testToCheckAndAppendHDFSUrlWithNull() {
@@ -375,6 +375,91 @@ public class CarbonUtilTest {
     assertEquals(hdfsURL, "file:////../core/src/test/resources/testDatabase");
   }
 
+  @Test public void testToCheckAndAppendHDFSUrlWithHdfs() {
+    new MockUp<FileFactory>() {
+      @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path)
{
+        return FileFactory.FileType.HDFS;
+      }
+    };
+    new MockUp<org.apache.hadoop.conf.Configuration>() {
+      @SuppressWarnings("unused") @Mock public String get(String name) {
+        return "hdfs://";
+      }
+    };
+    String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("hdfs://ha/core/src/test/resources/testDatabase");
+    assertEquals(hdfsURL, "hdfs://ha/core/src/test/resources/testDatabase");
+  }
+
+  @Test public void testToCheckAndAppendHDFSUrlWithDoubleSlashLocal() {
+    new MockUp<FileFactory>() {
+      @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path)
{
+        return FileFactory.FileType.LOCAL;
+      }
+    };
+    new MockUp<CarbonProperties>() {
+      @SuppressWarnings("unused") @Mock public String getProperty(String key) {
+        return "/opt/";
+      }
+    };
+    String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase");
+    assertEquals(hdfsURL, "file:////opt/core/src/test/resources/testDatabase");
+  }
+
+  @Test public void testToCheckAndAppendHDFSUrlWithDoubleSlashHDFS() {
+    new MockUp<FileFactory>() {
+      @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path)
{
+        return FileFactory.FileType.HDFS;
+      }
+    };
+    new MockUp<org.apache.hadoop.conf.Configuration>() {
+      @SuppressWarnings("unused") @Mock public String get(String name) {
+        return "hdfs://";
+      }
+    };
+    new MockUp<CarbonProperties>() {
+      @SuppressWarnings("unused") @Mock public String getProperty(String key) {
+        return "/opt/";
+      }
+    };
+    String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase");
+    assertEquals(hdfsURL, "hdfs:///opt/core/src/test/resources/testDatabase");
+  }
+
+  @Test public void testToCheckAndAppendHDFSUrlWithBaseURLPrefix() {
+    new MockUp<FileFactory>() {
+      @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path)
{
+        return FileFactory.FileType.HDFS;
+      }
+    };
+    new MockUp<CarbonProperties>() {
+      @SuppressWarnings("unused") @Mock public String getProperty(String key) {
+        return "hdfs://ha/opt/";
+      }
+    };
+    String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase");
+    assertEquals(hdfsURL, "hdfs://ha/opt/core/src/test/resources/testDatabase");
+  }
+
+  @Test public void testToCheckAndAppendHDFSUrlWithBaseURLFile() {
+    new MockUp<FileFactory>() {
+      @SuppressWarnings("unused") @Mock public FileFactory.FileType getFileType(String path)
{
+        return FileFactory.FileType.HDFS;
+      }
+    };
+    new MockUp<CarbonProperties>() {
+      @SuppressWarnings("unused") @Mock public String getProperty(String key) {
+        return "file:///";
+      }
+    };
+    String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("/core/src/test/resources/testDatabase");
+    assertEquals(hdfsURL, "file:///core/src/test/resources/testDatabase");
+  }
+
+  @Test public void testToCheckAndAppendHDFSUrlWithFilepathPrefix() {
+    String hdfsURL = CarbonUtil.checkAndAppendHDFSUrl("file:///core/src/test/resources/testDatabase");
+    assertEquals(hdfsURL, "file:///core/src/test/resources/testDatabase");
+  }
+
   @Test public void testForisFileExists() {
     assertTrue(CarbonUtil.isFileExists("../core/src/test/resources/testFile.txt"));
   }


Mime
View raw message