hudi-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From le...@apache.org
Subject [incubator-hudi] branch redo-log updated: [HUDI-454] Redo hudi-cli log statements using SLF4J (#1178)
Date Tue, 07 Jan 2020 01:21:50 GMT
This is an automated email from the ASF dual-hosted git repository.

leesf pushed a commit to branch redo-log
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git


The following commit(s) were added to refs/heads/redo-log by this push:
     new f88d6ca  [HUDI-454] Redo hudi-cli log statements using SLF4J (#1178)
f88d6ca is described below

commit f88d6cab7e586f7273bb3a39311b643408638f86
Author: ForwardXu <forwardxu315@gmail.com>
AuthorDate: Tue Jan 7 09:21:40 2020 +0800

    [HUDI-454] Redo hudi-cli log statements using SLF4J (#1178)
---
 hudi-cli/pom.xml                                                  | 5 +++--
 .../main/java/org/apache/hudi/cli/commands/CompactionCommand.java | 6 +++---
 .../org/apache/hudi/cli/commands/HDFSParquetImportCommand.java    | 4 ----
 .../src/main/java/org/apache/hudi/cli/commands/SparkMain.java     | 7 ++++---
 .../main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java  | 8 ++++----
 hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java   | 2 --
 hudi-cli/src/main/scala/org/apache/hudi/cli/DedupeSparkJob.scala  | 6 +++---
 7 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/hudi-cli/pom.xml b/hudi-cli/pom.xml
index 62b55de..74b31bc 100644
--- a/hudi-cli/pom.xml
+++ b/hudi-cli/pom.xml
@@ -146,8 +146,9 @@
 
     <!-- Logging -->
     <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>log4j</artifactId>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
     </dependency>
 
     <dependency>
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java
index 716b810..a75249e 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java
@@ -42,10 +42,10 @@ import org.apache.hudi.exception.HoodieIOException;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
 import org.apache.spark.launcher.SparkLauncher;
 import org.apache.spark.util.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.shell.core.CommandMarker;
 import org.springframework.shell.core.annotation.CliCommand;
 import org.springframework.shell.core.annotation.CliOption;
@@ -68,7 +68,7 @@ import java.util.stream.Collectors;
 @Component
 public class CompactionCommand implements CommandMarker {
 
-  private static final Logger LOG = LogManager.getLogger(CompactionCommand.class);
+  private static final Logger LOG = LoggerFactory.getLogger(CompactionCommand.class);
 
   private static final String TMP_DIR = "/tmp/";
 
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HDFSParquetImportCommand.java
b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HDFSParquetImportCommand.java
index e892ca9..06a59ff 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HDFSParquetImportCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HDFSParquetImportCommand.java
@@ -24,8 +24,6 @@ import org.apache.hudi.cli.utils.InputStreamConsumer;
 import org.apache.hudi.cli.utils.SparkUtil;
 import org.apache.hudi.utilities.HDFSParquetImporter.FormatValidator;
 
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
 import org.apache.spark.launcher.SparkLauncher;
 import org.apache.spark.util.Utils;
 import org.springframework.shell.core.CommandMarker;
@@ -41,8 +39,6 @@ import scala.collection.JavaConverters;
 @Component
 public class HDFSParquetImportCommand implements CommandMarker {
 
-  private static final Logger LOG = LogManager.getLogger(HDFSParquetImportCommand.class);
-
   @CliCommand(value = "hdfsparquetimport", help = "Imports Parquet dataset to a hoodie dataset")
   public String convert(
       @CliOption(key = "upsert", mandatory = false, unspecifiedDefaultValue = "false",
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
index 5d08921..15fe31b 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java
@@ -32,16 +32,17 @@ import org.apache.hudi.utilities.HoodieCompactionAdminTool;
 import org.apache.hudi.utilities.HoodieCompactionAdminTool.Operation;
 import org.apache.hudi.utilities.HoodieCompactor;
 
-import org.apache.log4j.Logger;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.sql.SQLContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class deals with initializing spark context based on command entered to hudi-cli.
  */
 public class SparkMain {
 
-  private static final Logger LOG = Logger.getLogger(SparkMain.class);
+  private static final Logger LOG = LoggerFactory.getLogger(SparkMain.class);
 
   /**
    * Commands.
@@ -52,7 +53,7 @@ public class SparkMain {
 
   public static void main(String[] args) throws Exception {
     String command = args[0];
-    LOG.info("Invoking SparkMain:" + command);
+    LOG.info("Invoking SparkMain:{}", command);
 
     SparkCommand cmd = SparkCommand.valueOf(command);
 
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java
index 73aa45c..33a9a15 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java
@@ -22,14 +22,15 @@ import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.util.logging.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * This class is responsible to read a Process output.
  */
 public class InputStreamConsumer extends Thread {
 
-  private static final Logger LOG = Logger.getLogger(InputStreamConsumer.class.getName());
+  private static final Logger LOG = LoggerFactory.getLogger(InputStreamConsumer.class.getName());
   private InputStream is;
 
   public InputStreamConsumer(InputStream is) {
@@ -46,8 +47,7 @@ public class InputStreamConsumer extends Thread {
         LOG.info(line);
       }
     } catch (IOException ioe) {
-      LOG.severe(ioe.toString());
-      ioe.printStackTrace();
+      LOG.error("Run the InputStreamConsumer is failed:", ioe);
     }
   }
 
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
index 5b5a3f5..4ea6f50 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java
@@ -24,7 +24,6 @@ import org.apache.hudi.cli.commands.SparkMain;
 import org.apache.hudi.common.util.FSUtils;
 import org.apache.hudi.common.util.StringUtils;
 
-import org.apache.log4j.Logger;
 import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.launcher.SparkLauncher;
@@ -38,7 +37,6 @@ import java.util.Map;
  */
 public class SparkUtil {
 
-  private static final Logger LOG = Logger.getLogger(SparkUtil.class);
   public static final String DEFUALT_SPARK_MASTER = "yarn-client";
 
   /**
diff --git a/hudi-cli/src/main/scala/org/apache/hudi/cli/DedupeSparkJob.scala b/hudi-cli/src/main/scala/org/apache/hudi/cli/DedupeSparkJob.scala
index f40501a..e192795 100644
--- a/hudi-cli/src/main/scala/org/apache/hudi/cli/DedupeSparkJob.scala
+++ b/hudi-cli/src/main/scala/org/apache/hudi/cli/DedupeSparkJob.scala
@@ -25,8 +25,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient
 import org.apache.hudi.common.table.view.HoodieTableFileSystemView
 import org.apache.hudi.common.util.FSUtils
 import org.apache.hudi.exception.HoodieException
-import org.apache.log4j.Logger
 import org.apache.spark.sql.{DataFrame, SQLContext}
+import org.slf4j.{Logger, LoggerFactory}
 
 import scala.collection.JavaConversions._
 import scala.collection.mutable._
@@ -43,7 +43,7 @@ class DedupeSparkJob(basePath: String,
 
 
   val sparkHelper = new SparkHelper(sqlContext, fs)
-  val LOG = Logger.getLogger(this.getClass)
+  val LOG: Logger = LoggerFactory.getLogger(this.getClass)
 
 
   /**
@@ -150,7 +150,7 @@ class DedupeSparkJob(basePath: String,
       val commitTime = FSUtils.getCommitTime(fileNameToPathMap(fileName).getName)
       val badFilePath = new Path(s"$repairOutputPath/${fileNameToPathMap(fileName).getName}.bad")
       val newFilePath = new Path(s"$repairOutputPath/${fileNameToPathMap(fileName).getName}")
-      LOG.info(" Skipping and writing new file for : " + fileName)
+      LOG.info(" Skipping and writing new file for : {}", fileName)
       SparkHelpers.skipKeysAndWriteNewFile(commitTime, fs, badFilePath, newFilePath, dupeFixPlan(fileName))
       fs.delete(badFilePath, false)
     }


Mime
View raw message