kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject [kudu] branch master updated: KUDU-2796: Fix HDFS path handling
Date Tue, 23 Apr 2019 21:17:45 GMT
This is an automated email from the ASF dual-hosted git repository.

granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git


The following commit(s) were added to refs/heads/master by this push:
     new 963a358  KUDU-2796: Fix HDFS path handling
963a358 is described below

commit 963a358c7745eb5173e5168c6072242a400b2184
Author: Grant Henke <granthenke@apache.org>
AuthorDate: Tue Apr 23 14:45:49 2019 -0500

    KUDU-2796: Fix HDFS path handling
    
    This patch changes the SessionIO class to use only
    Hadoop Paths instances.
    
    Before it was converting between
    Hadoop Path objects and Java Path objects which
    seems to cause issues with the NameService URI.
    
    Change-Id: I565d753d4ac70597dc9d1cb151c51e84b8db87f5
    Reviewed-on: http://gerrit.cloudera.org:8080/13089
    Reviewed-by: Grant Henke <granthenke@apache.org>
    Tested-by: Grant Henke <granthenke@apache.org>
---
 .../scala/org/apache/kudu/backup/BackupGraph.scala |  3 +--
 .../scala/org/apache/kudu/backup/SessionIO.scala   | 24 +++++++++-------------
 2 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/BackupGraph.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
index 1a6e92a..88f1f35 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/BackupGraph.scala
@@ -16,8 +16,7 @@
 // under the License.
 package org.apache.kudu.backup
 
-import java.nio.file.Path
-
+import org.apache.hadoop.fs.Path
 import org.apache.kudu.backup.Backup.TableMetadataPB
 import org.apache.yetus.audience.InterfaceAudience
 import org.apache.yetus.audience.InterfaceStability
diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/SessionIO.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/SessionIO.scala
index 0030362..f64531f 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/SessionIO.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/SessionIO.scala
@@ -19,14 +19,12 @@ package org.apache.kudu.backup
 import java.io.InputStreamReader
 import java.net.URLEncoder
 import java.nio.charset.StandardCharsets
-import java.nio.file.Path
-import java.nio.file.Paths
 
 import com.google.common.io.CharStreams
 import com.google.protobuf.util.JsonFormat
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.FileSystem
-import org.apache.hadoop.fs.{Path => HPath}
+import org.apache.hadoop.fs.Path
 import org.apache.kudu.Schema
 import org.apache.kudu.backup.Backup.TableMetadataPB
 import org.apache.kudu.backup.SessionIO._
@@ -40,7 +38,6 @@ import org.apache.yetus.audience.InterfaceStability
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
-import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 /**
@@ -63,7 +60,7 @@ class SessionIO(val session: SparkSession, options: CommonOptions) {
   val log: Logger = LoggerFactory.getLogger(getClass)
 
   val conf: Configuration = session.sparkContext.hadoopConfiguration
-  val rootHPath: HPath = new HPath(options.rootPath)
+  val rootHPath: Path = new Path(options.rootPath)
   val fs: FileSystem = rootHPath.getFileSystem(conf)
 
   /**
@@ -97,21 +94,21 @@ class SessionIO(val session: SparkSession, options: CommonOptions) {
    * @return the path to the table directory.
    */
   def tablePath(tableName: String): Path = {
-    Paths.get(options.rootPath).resolve(URLEncoder.encode(tableName, "UTF-8"))
+    new Path(options.rootPath, URLEncoder.encode(tableName, "UTF-8"))
   }
 
   /**
    * @return the backup path for a table and time.
    */
   def backupPath(tableName: String, timestampMs: Long): Path = {
-    tablePath(tableName).resolve(timestampMs.toString)
+    new Path(tablePath(tableName), timestampMs.toString)
   }
 
   /**
    * @return the path to the metadata file within a backup path.
    */
   def backupMetadataPath(backupPath: Path): Path = {
-    backupPath.resolve(MetadataFileName)
+    new Path(backupPath, MetadataFileName)
   }
 
   /**
@@ -121,8 +118,7 @@ class SessionIO(val session: SparkSession, options: CommonOptions) {
    */
   def writeTableMetadata(tableMetadata: TableMetadataPB, metadataPath: Path): Unit = {
     log.error(s"Writing metadata to $metadataPath")
-    val hPath = new HPath(metadataPath.toString)
-    val out = fs.create(hPath, /* overwrite= */ false)
+    val out = fs.create(metadataPath, /* overwrite= */ false)
     val json = JsonFormat.printer().print(tableMetadata)
     out.write(json.getBytes(StandardCharsets.UTF_8))
     out.flush()
@@ -152,17 +148,17 @@ class SessionIO(val session: SparkSession, options: CommonOptions) {
    */
   // TODO: Also use table-id to find backups.
   private def readTableBackups(tableName: String): Seq[(Path, TableMetadataPB)] = {
-    val hPath = new HPath(tablePath(tableName).toString)
+    val hPath = new Path(tablePath(tableName).toString)
     val results = new mutable.ListBuffer[(Path, TableMetadataPB)]()
     if (fs.exists(hPath)) {
       val iter = fs.listLocatedStatus(hPath)
       while (iter.hasNext) {
         val file = iter.next()
         if (file.isDirectory) {
-          val metadataHPath = new HPath(file.getPath, MetadataFileName)
+          val metadataHPath = new Path(file.getPath, MetadataFileName)
           if (fs.exists(metadataHPath)) {
             val metadata = readTableMetadata(metadataHPath)
-            results += ((Paths.get(file.getPath.toString), metadata))
+            results += ((file.getPath, metadata))
           }
         }
       }
@@ -176,7 +172,7 @@ class SessionIO(val session: SparkSession, options: CommonOptions) {
    * @param metadataPath the path to the metadata file.
    * @return the deserialized table metadata.
    */
-  private def readTableMetadata(metadataPath: HPath): TableMetadataPB = {
+  private def readTableMetadata(metadataPath: Path): TableMetadataPB = {
     val in = new InputStreamReader(fs.open(metadataPath), StandardCharsets.UTF_8)
     val json = CharStreams.toString(in)
     in.close()


Mime
View raw message