hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject [09/50] [abbrv] hive git commit: HIVE-11304: Migrate to Log4j2 from Log4j 1.x (Prasanth Jayachandran reviewed by Thejas Nair, Sergey Shelukhin)
Date Tue, 25 Aug 2015 21:28:39 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index f156cc4..acce713 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -94,7 +94,7 @@
                       }
                       mkdir -p $DOWNLOAD_DIR
                       download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz" "spark"
-                      cp -f $HIVE_ROOT/data/conf/spark/log4j.properties $BASE_DIR/spark/conf/
+                      cp -f $HIVE_ROOT/data/conf/spark/log4j2.xml $BASE_DIR/spark/conf/
                       sed '/package /d' ${basedir}/${hive.path.to.root}/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java > /tmp/UDFExampleAdd.java
                       javac -cp  ${settings.localRepository}/org/apache/hive/hive-exec/${project.version}/hive-exec-${project.version}.jar /tmp/UDFExampleAdd.java -d /tmp
                       jar -cf /tmp/udfexampleadd-1.0.jar -C /tmp UDFExampleAdd.class

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index b33cb58..65117c4 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -5931,7 +5931,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
 
     // If the log4j.configuration property hasn't already been explicitly set,
     // use Hive's default log4j configuration
-    if (System.getProperty("log4j.configuration") == null) {
+    if (System.getProperty("log4j.configurationFile") == null) {
       // NOTE: It is critical to do this here so that log4j is reinitialized
       // before any of the other core hive classes are loaded
       try {

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
index ad99427..df42f1a 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
@@ -17,15 +17,11 @@
  */
 package org.apache.hadoop.hive.metastore.txn;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.*;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -34,7 +30,29 @@ import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
 
-import static junit.framework.Assert.*;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
+import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockLevel;
+import org.apache.hadoop.hive.metastore.api.LockRequest;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
+import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
+import org.apache.hadoop.hive.metastore.api.UnlockRequest;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
 
 /**
  * Tests for TxnHandler.
@@ -43,11 +61,9 @@ public class TestCompactionTxnHandler {
 
   private HiveConf conf = new HiveConf();
   private CompactionTxnHandler txnHandler;
-  static final private Log LOG = LogFactory.getLog(TestCompactionTxnHandler.class);
 
   public TestCompactionTxnHandler() throws Exception {
     TxnDbUtil.setConfValues(conf);
-    LogManager.getLogger(TxnHandler.class.getName()).setLevel(Level.DEBUG);
     tearDown();
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
index f478184..6dc0bd3 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
@@ -17,16 +17,11 @@
  */
 package org.apache.hadoop.hive.metastore.txn;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.*;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertFalse;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
 
 import java.sql.Connection;
 import java.sql.SQLException;
@@ -36,21 +31,66 @@ import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static junit.framework.Assert.*;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CheckLockRequest;
+import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.HeartbeatRequest;
+import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeRequest;
+import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockLevel;
+import org.apache.hadoop.hive.metastore.api.LockRequest;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchLockException;
+import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
+import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
+import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
+import org.apache.hadoop.hive.metastore.api.ShowLocksRequest;
+import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
+import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement;
+import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
+import org.apache.hadoop.hive.metastore.api.TxnInfo;
+import org.apache.hadoop.hive.metastore.api.TxnOpenException;
+import org.apache.hadoop.hive.metastore.api.TxnState;
+import org.apache.hadoop.hive.metastore.api.UnlockRequest;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Tests for TxnHandler.
  */
 public class TestTxnHandler {
   static final private String CLASS_NAME = TxnHandler.class.getName();
-  static final private Log LOG = LogFactory.getLog(CLASS_NAME);
+  private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
 
   private HiveConf conf = new HiveConf();
   private TxnHandler txnHandler;
 
   public TestTxnHandler() throws Exception {
     TxnDbUtil.setConfValues(conf);
-    LogManager.getLogger(TxnHandler.class.getName()).setLevel(Level.DEBUG);
+    LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+    Configuration conf = ctx.getConfiguration();
+    conf.getLoggerConfig(CLASS_NAME).setLevel(Level.DEBUG);
+    ctx.updateLoggers(conf);
     tearDown();
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/packaging/src/main/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml
index 63253c5..0fa6af8 100644
--- a/packaging/src/main/assembly/bin.xml
+++ b/packaging/src/main/assembly/bin.xml
@@ -275,7 +275,7 @@
       <fileMode>644</fileMode>
       <includes>
         <include>webhcat-default.xml</include>
-        <include>webhcat-log4j.properties</include>
+        <include>webhcat-log4j2.xml</include>
       </includes>
       <filtered>true</filtered>
       <outputDirectory>hcatalog/etc/webhcat</outputDirectory>
@@ -323,19 +323,19 @@
 
   <files>
     <file>
-      <source>${project.parent.basedir}/common/src/main/resources/hive-log4j.properties</source>
+      <source>${project.parent.basedir}/common/src/main/resources/hive-log4j2.xml</source>
       <outputDirectory>conf</outputDirectory>
-      <destName>hive-log4j.properties.template</destName>
+      <destName>hive-log4j2.xml.template</destName>
     </file>
     <file>
-      <source>${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j.properties</source>
+      <source>${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j2.xml</source>
       <outputDirectory>conf</outputDirectory>
-      <destName>hive-exec-log4j.properties.template</destName>
+      <destName>hive-exec-log4j2.xml.template</destName>
     </file>
     <file>
-      <source>${project.parent.basedir}/beeline/src/main/resources/beeline-log4j.properties</source>
+      <source>${project.parent.basedir}/beeline/src/main/resources/beeline-log4j2.xml</source>
       <outputDirectory>conf</outputDirectory>
-      <destName>beeline-log4j.properties.template</destName>
+      <destName>beeline-log4j2.xml.template</destName>
     </file>
     <file>
       <source>${project.parent.basedir}/hcatalog/README.txt</source>

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0383e01..15c2805 100644
--- a/pom.xml
+++ b/pom.xml
@@ -148,8 +148,7 @@
     <kryo.version>2.22</kryo.version>
     <libfb303.version>0.9.2</libfb303.version>
     <libthrift.version>0.9.2</libthrift.version>
-    <log4j.version>1.2.16</log4j.version>
-    <log4j-extras.version>1.2.17</log4j-extras.version>
+    <log4j2.version>2.3</log4j2.version>
     <opencsv.version>2.3</opencsv.version>
     <mockito-all.version>1.9.5</mockito-all.version>
     <mina.version>2.0.0-M5</mina.version>
@@ -366,14 +365,24 @@
         <version>${junit.version}</version>
       </dependency>
       <dependency>
-        <groupId>log4j</groupId>
-        <artifactId>log4j</artifactId>
-        <version>${log4j.version}</version>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-1.2-api</artifactId>
+        <version>${log4j2.version}</version>
       </dependency>
       <dependency>
-        <groupId>log4j</groupId>
-        <artifactId>apache-log4j-extras</artifactId>
-        <version>${log4j-extras.version}</version>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-web</artifactId>
+        <version>${log4j2.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-slf4j-impl</artifactId>
+        <version>${log4j2.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.logging.log4j</groupId>
+        <artifactId>log4j-jcl</artifactId>
+        <version>${log4j2.version}</version>
       </dependency>
       <dependency>
         <groupId>org.antlr</groupId>
@@ -584,11 +593,6 @@
         <version>${slf4j.version}</version>
       </dependency>
       <dependency>
-        <groupId>org.slf4j</groupId>
-        <artifactId>slf4j-log4j12</artifactId>
-        <version>${slf4j.version}</version>
-      </dependency>
-      <dependency>
         <groupId>xerces</groupId>
         <artifactId>xercesImpl</artifactId>
         <version>${xerces.version}</version>
@@ -604,11 +608,6 @@
       <artifactId>slf4j-api</artifactId>
       <version>${slf4j.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
-      <version>${slf4j.version}</version>
-    </dependency>
   </dependencies>
 
   <build>
@@ -872,7 +871,7 @@
             <!-- required for hive-exec jar path and tests which reference a jar -->
             <maven.local.repository>${maven.repo.local}</maven.local.repository>
             <mapred.job.tracker>local</mapred.job.tracker>
-            <log4j.configuration>${test.log4j.scheme}${test.tmp.dir}/conf/hive-log4j.properties</log4j.configuration>
+            <log4j.configurationFile>${test.log4j.scheme}${test.tmp.dir}/conf/hive-log4j2.xml</log4j.configurationFile>
             <log4j.debug>true</log4j.debug>
             <!-- don't diry up /tmp -->
             <java.io.tmpdir>${test.tmp.dir}</java.io.tmpdir>

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/pom.xml
----------------------------------------------------------------------
diff --git a/ql/pom.xml b/ql/pom.xml
index e7a8e7b..36b3433 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -112,14 +112,19 @@
       <version>${javolution.version}</version>
     </dependency>
     <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>log4j</artifactId>
-      <version>${log4j.version}</version>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-1.2-api</artifactId>
+      <version>${log4j2.version}</version>
     </dependency>
     <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>apache-log4j-extras</artifactId>
-      <version>${log4j-extras.version}</version>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <version>${log4j2.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-jcl</artifactId>
+      <version>${log4j2.version}</version>
     </dependency>
     <dependency>
       <groupId>org.antlr</groupId>

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index a2cf712..82345ee 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -27,7 +27,6 @@ import java.lang.management.MemoryMXBean;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Enumeration;
 import java.util.List;
 import java.util.Properties;
 
@@ -57,13 +56,14 @@ import org.apache.hadoop.hive.ql.exec.PartitionKeySampler;
 import org.apache.hadoop.hive.ql.exec.TableScanOperator;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
 import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager;
+import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
 import org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
 import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
 import org.apache.hadoop.hive.ql.io.HiveKey;
 import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl;
 import org.apache.hadoop.hive.ql.io.IOPrepareCache;
+import org.apache.hadoop.hive.ql.log.NullAppender;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.plan.FetchWork;
 import org.apache.hadoop.hive.ql.plan.MapWork;
@@ -88,11 +88,12 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.Partitioner;
 import org.apache.hadoop.mapred.RunningJob;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.log4j.Appender;
-import org.apache.log4j.BasicConfigurator;
-import org.apache.log4j.FileAppender;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.varia.NullAppender;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.appender.FileAppender;
+import org.apache.logging.log4j.core.appender.RollingFileAppender;
 
 /**
  * ExecDriver is the central class in co-ordinating execution of any map-reduce task.
@@ -687,8 +688,10 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
     if (noLog) {
       // If started from main(), and noLog is on, we should not output
       // any logs. To turn the log on, please set -Dtest.silent=false
-      BasicConfigurator.resetConfiguration();
-      BasicConfigurator.configure(new NullAppender());
+      Logger logger = org.apache.logging.log4j.LogManager.getRootLogger();
+      NullAppender appender = NullAppender.createNullAppender();
+      appender.addToLogger(logger.getName(), Level.ERROR);
+      appender.start();
     } else {
       setupChildLog4j(conf);
     }
@@ -703,10 +706,12 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
 
     // print out the location of the log file for the user so
     // that it's easy to find reason for local mode execution failures
-    for (Appender appender : Collections.list((Enumeration<Appender>) LogManager.getRootLogger()
-        .getAllAppenders())) {
+    for (Appender appender : ((org.apache.logging.log4j.core.Logger) LogManager.getRootLogger())
+            .getAppenders().values()) {
       if (appender instanceof FileAppender) {
-        console.printInfo("Execution log at: " + ((FileAppender) appender).getFile());
+        console.printInfo("Execution log at: " + ((FileAppender) appender).getFileName());
+      } else if (appender instanceof RollingFileAppender) {
+        console.printInfo("Execution log at: " + ((RollingFileAppender) appender).getFileName());
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
index 6a6593c..44dfe3e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
@@ -24,7 +24,6 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Collections;
-import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -33,8 +32,6 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.MapRedStats;
@@ -59,9 +56,11 @@ import org.apache.hadoop.mapred.JobStatus;
 import org.apache.hadoop.mapred.RunningJob;
 import org.apache.hadoop.mapred.TaskCompletionEvent;
 import org.apache.hadoop.mapred.TaskReport;
-import org.apache.log4j.Appender;
-import org.apache.log4j.FileAppender;
-import org.apache.log4j.LogManager;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Logger;
+import org.apache.logging.log4j.core.appender.FileAppender;
+import org.apache.logging.log4j.core.appender.RollingFileAppender;
 
 public class HadoopJobExecHelper {
 
@@ -492,10 +491,11 @@ public class HadoopJobExecHelper {
     sb.append("Logs:\n");
     console.printError(sb.toString());
 
-    for (Appender a : Collections.list((Enumeration<Appender>)
-          LogManager.getRootLogger().getAllAppenders())) {
-      if (a instanceof FileAppender) {
-        console.printError((new Path(((FileAppender)a).getFile())).toUri().getPath());
+    for (Appender appender : ((Logger) LogManager.getRootLogger()).getAppenders().values()) {
+      if (appender instanceof FileAppender) {
+        console.printError(((FileAppender) appender).getFileName());
+      } else if (appender instanceof RollingFileAppender) {
+        console.printError(((RollingFileAppender) appender).getFileName());
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
index 3cb9e9c..cee0878 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.hive.ql.io.rcfile.stats;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Enumeration;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
@@ -59,9 +57,11 @@ import org.apache.hadoop.mapred.InputFormat;
 import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.RunningJob;
-import org.apache.log4j.Appender;
-import org.apache.log4j.FileAppender;
-import org.apache.log4j.LogManager;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Logger;
+import org.apache.logging.log4j.core.appender.FileAppender;
+import org.apache.logging.log4j.core.appender.RollingFileAppender;
 
 /**
  * PartialScanTask.
@@ -335,15 +335,15 @@ public class PartialScanTask extends Task<PartialScanWork> implements
 
     // print out the location of the log file for the user so
     // that it's easy to find reason for local mode execution failures
-    for (Appender appender : Collections
-        .list((Enumeration<Appender>) LogManager.getRootLogger()
-            .getAllAppenders())) {
+    for (Appender appender : ((Logger) LogManager.getRootLogger()).getAppenders().values()) {
       if (appender instanceof FileAppender) {
-        console.printInfo("Execution log at: "
-            + ((FileAppender) appender).getFile());
+        console.printInfo("Execution log at: " + ((FileAppender) appender).getFileName());
+      } else if (appender instanceof RollingFileAppender) {
+        console.printInfo("Execution log at: " + ((RollingFileAppender) appender).getFileName());
       }
     }
 
+
     PartialScanWork mergeWork = new PartialScanWork(inputPaths);
     DriverContext driverCxt = new DriverContext();
     PartialScanTask taskExec = new PartialScanTask();

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java b/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java
new file mode 100644
index 0000000..46662c4
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.log;
+
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicLongArray;
+
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
+import org.apache.logging.log4j.core.config.plugins.PluginElement;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * A log4J2 Appender that simply counts logging events in four levels:
+ * fatal, error, warn and info. The class name is used in log4j2.xml
+ */
+@Plugin(name = "HiveEventCounter", category = "Core", elementType = "appender", printObject = true)
+public class HiveEventCounter extends AbstractAppender {
+  private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
+  private static Configuration configuration = context.getConfiguration();
+  private static final String APPENDER_NAME = "HiveEventCounter";
+  private static final int FATAL = 0;
+  private static final int ERROR = 1;
+  private static final int WARN = 2;
+  private static final int INFO = 3;
+
+  private static class EventCounts {
+    private final AtomicLongArray counts = new AtomicLongArray(4);
+
+    private void incr(int i) {
+      counts.incrementAndGet(i);
+    }
+
+    private long get(int i) {
+      return counts.get(i);
+    }
+  }
+
+  private static EventCounts counts = new EventCounts();
+
+  protected HiveEventCounter(String name, Filter filter,
+      Layout<? extends Serializable> layout, boolean ignoreExceptions) {
+    super(name, filter, layout, ignoreExceptions);
+  }
+
+  @PluginFactory
+  public static HiveEventCounter createInstance(@PluginAttribute("name") String name,
+      @PluginAttribute("ignoreExceptions") boolean ignoreExceptions,
+      @PluginElement("Layout") Layout layout,
+      @PluginElement("Filters") Filter filter) {
+    if (name == null) {
+      name = APPENDER_NAME;
+    }
+
+    if (layout == null) {
+      layout = PatternLayout.createDefaultLayout();
+    }
+    return new HiveEventCounter(name, filter, layout, ignoreExceptions);
+  }
+
+  @InterfaceAudience.Private
+  public static long getFatal() {
+    return counts.get(FATAL);
+  }
+
+  @InterfaceAudience.Private
+  public static long getError() {
+    return counts.get(ERROR);
+  }
+
+  @InterfaceAudience.Private
+  public static long getWarn() {
+    return counts.get(WARN);
+  }
+
+  @InterfaceAudience.Private
+  public static long getInfo() {
+    return counts.get(INFO);
+  }
+
+  @VisibleForTesting
+  public void addToLogger(String loggerName, Level level) {
+    LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+    loggerConfig.addAppender(this, level, null);
+    context.updateLoggers();
+  }
+
+  @VisibleForTesting
+  public void removeFromLogger(String loggerName) {
+    LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+    loggerConfig.removeAppender(APPENDER_NAME);
+    context.updateLoggers();
+  }
+
+  public void append(LogEvent event) {
+    Level level = event.getLevel();
+    if (level.equals(Level.INFO)) {
+      counts.incr(INFO);
+    } else if (level.equals(Level.WARN)) {
+      counts.incr(WARN);
+    } else if (level.equals(Level.ERROR)) {
+      counts.incr(ERROR);
+    } else if (level.equals(Level.FATAL)) {
+      counts.incr(FATAL);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java b/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java
new file mode 100644
index 0000000..c4cb7dd
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.log;
+
+import java.io.Serializable;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+
+/**
+ * A NullAppender merely exists, it never outputs a message to any device.
+ */
+@Plugin(name = "NullAppender", category = "Core", elementType = "appender", printObject = false)
+public class NullAppender extends AbstractAppender {
+
+  private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
+  private static Configuration configuration = context.getConfiguration();
+
+  protected NullAppender(String name, Filter filter,
+      Layout<? extends Serializable> layout, boolean ignoreExceptions) {
+    super(name, filter, layout, ignoreExceptions);
+  }
+
+  @PluginFactory
+  public static NullAppender createNullAppender() {
+    return new NullAppender("NullAppender", null, PatternLayout.createDefaultLayout(), true);
+  }
+
+  public void addToLogger(String loggerName, Level level) {
+    LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+    loggerConfig.addAppender(this, level, null);
+    context.updateLoggers();
+  }
+
+  public void append(LogEvent event) {
+    // no-op
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java b/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java
deleted file mode 100644
index 6a59d4a..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.log;
-
-import java.lang.management.ManagementFactory;
-import java.lang.management.RuntimeMXBean;
-
-import org.apache.log4j.DailyRollingFileAppender;
-
-public class PidDailyRollingFileAppender extends DailyRollingFileAppender {
-
-  @Override
-  public void setFile(String file) {
-    RuntimeMXBean rt = ManagementFactory.getRuntimeMXBean();
-    super.setFile(file + '.' + rt.getName());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java
new file mode 100644
index 0000000..4db10bb
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.log;
+
+import java.lang.management.ManagementFactory;
+
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.pattern.AbstractPatternConverter;
+import org.apache.logging.log4j.core.pattern.ArrayPatternConverter;
+import org.apache.logging.log4j.core.pattern.ConverterKeys;
+
+/**
+ * FilePattern converter that converts %pid pattern to <process-id>@<hostname> information
+ * obtained at runtime.
+ *
+ * Example usage:
+ * <RollingFile name="Rolling-default" fileName="test.log" filePattern="test.log.%pid.gz">
+ *
+ * Will generate output file with name containing <process-id>@<hostname> like below
+ * test.log.95232@localhost.gz
+ */
+@Plugin(name = "PidFilePatternConverter", category = "FileConverter")
+@ConverterKeys({ "pid" })
+public class PidFilePatternConverter extends AbstractPatternConverter implements
+    ArrayPatternConverter {
+
+  /**
+   * Private constructor.
+   */
+  private PidFilePatternConverter() {
+    super("pid", "pid");
+  }
+
+  @PluginFactory
+  public static PidFilePatternConverter newInstance() {
+    return new PidFilePatternConverter();
+  }
+
+  public void format(StringBuilder toAppendTo, Object... objects) {
+    toAppendTo.append(ManagementFactory.getRuntimeMXBean().getName());
+  }
+
+  public void format(Object obj, StringBuilder toAppendTo) {
+    toAppendTo.append(ManagementFactory.getRuntimeMXBean().getName());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/main/resources/hive-exec-log4j.properties
----------------------------------------------------------------------
diff --git a/ql/src/main/resources/hive-exec-log4j.properties b/ql/src/main/resources/hive-exec-log4j.properties
deleted file mode 100644
index 9eaa6b6..0000000
--- a/ql/src/main/resources/hive-exec-log4j.properties
+++ /dev/null
@@ -1,77 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hive.log.threshold=ALL
-hive.root.logger=INFO,FA
-hive.log.dir=${java.io.tmpdir}/${user.name}
-hive.query.id=hadoop
-hive.log.file=${hive.query.id}.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hive.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshhold=${hive.log.threshold}
-
-#
-# File Appender
-#
-
-log4j.appender.FA=org.apache.log4j.FileAppender
-log4j.appender.FA.File=${hive.log.dir}/${hive.log.file}
-log4j.appender.FA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t] %p %c{2}: %m%n
-
-#custom logging levels
-#log4j.logger.xxx=DEBUG
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
-
-
-log4j.category.DataNucleus=ERROR,FA
-log4j.category.Datastore=ERROR,FA
-log4j.category.Datastore.Schema=ERROR,FA
-log4j.category.JPOX.Datastore=ERROR,FA
-log4j.category.JPOX.Plugin=ERROR,FA
-log4j.category.JPOX.MetaData=ERROR,FA
-log4j.category.JPOX.Query=ERROR,FA
-log4j.category.JPOX.General=ERROR,FA
-log4j.category.JPOX.Enhancer=ERROR,FA
-
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/main/resources/hive-exec-log4j2.xml
----------------------------------------------------------------------
diff --git a/ql/src/main/resources/hive-exec-log4j2.xml b/ql/src/main/resources/hive-exec-log4j2.xml
new file mode 100644
index 0000000..c93437c
--- /dev/null
+++ b/ql/src/main/resources/hive-exec-log4j2.xml
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="HiveExecLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+  <Properties>
+    <Property name="hive.log.threshold">ALL</Property>
+    <Property name="hive.log.level">INFO</Property>
+    <Property name="hive.root.logger">FA</Property>
+    <Property name="hive.log.dir">${sys:java.io.tmpdir}/${sys:user.name}</Property>
+    <Property name="hive.query.id">hadoop</Property>
+    <Property name="hive.log.file">${sys:hive.query.id}.log</Property>
+  </Properties>
+
+  <Appenders>
+    <Console name="console" target="SYSTEM_ERR">
+      <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
+    </Console>
+
+    <!-- Regular File Appender -->
+    <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+    <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
+      <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+    </File>
+
+    <!-- Daily Rolling File Appender -->
+    <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+    <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
+    <!-- <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+     filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
+      <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+      <Policies>
+        <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+      </Policies>
+      <DefaultRolloverStrategy max="30"/>
+    </RollingFile> -->
+
+    <!-- Size based Rolling File Appender -->
+    <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+    <!-- <RollingFile name="RFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+     filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%i">
+      <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+      <Policies>
+        <SizeBasedTriggeringPolicy size="256 MB" />
+      </Policies>
+      <DefaultRolloverStrategy max="10"/>
+    </RollingFile> -->
+
+    <!-- HiveEventCounter appender is loaded from Configuration packages attribute.Sends counts of logging messages at different severity levels to Hadoop Metrics. -->
+    <HiveEventCounter name="EventCounter"/>
+  </Appenders>
+
+  <Loggers>
+    <Root level="${sys:hive.log.threshold}">
+      <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
+      <AppenderRef ref="EventCounter" />
+    </Root>
+
+    <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="DataNucleus" level="ERROR">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="Datastore" level="ERROR">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="Datastore.Schema" level="ERROR">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="JPOX.Datastore" level="ERROR">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="JPOX.Plugin" level="ERROR">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="JPOX.Metadata" level="ERROR">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="JPOX.Query" level="ERROR">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="JPOX.General" level="ERROR">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+    <Logger name="JPOX.Enhancer" level="ERROR">
+      <AppenderRef ref="${sys:hive.root.logger}"/>
+    </Logger>
+  </Loggers>
+
+</Configuration>

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java b/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java
new file mode 100644
index 0000000..bdd837e
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.log;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.hive.ql.metadata.StringAppender;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class TestLog4j2Appenders {
+
+  @Before
+  public void setup() {
+    // programmatically set root logger level to INFO. By default if log4j2-test.xml is not
+    // available root logger will use ERROR log level
+    LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+    Configuration config = ctx.getConfiguration();
+    LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME);
+    loggerConfig.setLevel(Level.INFO);
+    ctx.updateLoggers();
+  }
+
+  @Test
+  public void testStringAppender() throws Exception {
+    // Get the RootLogger which, if you don't have log4j2-test.xml defined, will only log ERRORs
+    Logger logger = LogManager.getRootLogger();
+    // Create a String Appender to capture log output
+    StringAppender appender = StringAppender.createStringAppender("%m");
+    appender.addToLogger(logger.getName(), Level.INFO);
+    appender.start();
+
+    // Log to the string appender
+    logger.info("Hello!");
+    logger.info(" World");
+
+    assertEquals("Hello! World", appender.getOutput());
+    appender.removeFromLogger(LogManager.getRootLogger().getName());
+  }
+
+  @Test
+  public void testHiveEventCounterAppender() throws Exception {
+    Logger logger = LogManager.getRootLogger();
+    HiveEventCounter appender = HiveEventCounter.createInstance("EventCounter", true, null, null);
+    appender.addToLogger(logger.getName(), Level.INFO);
+    appender.start();
+
+    logger.info("Test");
+    logger.info("Test");
+    logger.info("Test");
+    logger.info("Test");
+
+    logger.error("Test");
+    logger.error("Test");
+    logger.error("Test");
+
+    logger.warn("Test");
+    logger.warn("Test");
+
+    logger.fatal("Test");
+
+    // HiveEventCounter will be loaded from hive-log4j2-test.xml before tests are run. The 2 log
+    // info msgs from previous test case will also be counted along with 4 log info msgs in this
+    // test and hence we assert for 6 here
+    assertEquals(6, appender.getInfo());
+    assertEquals(3, appender.getError());
+    assertEquals(2, appender.getWarn());
+    assertEquals(1, appender.getFatal());
+    appender.removeFromLogger(LogManager.getRootLogger().getName());
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java
new file mode 100644
index 0000000..17b64d6
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.metadata;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender;
+import org.apache.logging.log4j.core.appender.OutputStreamManager;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Log4j2 appender that writers to in-memory string object.
+ */
+@Plugin(name = "StringAppender", category = "Core", elementType = "appender", printObject = true)
+public class StringAppender
+    extends AbstractOutputStreamAppender<StringAppender.StringOutputStreamManager> {
+
+  private static final String APPENDER_NAME = "StringAppender";
+  private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
+  private static Configuration configuration = context.getConfiguration();
+  private StringOutputStreamManager manager;
+
+  /**
+   * Instantiate a WriterAppender and set the output destination to a
+   * new {@link OutputStreamWriter} initialized with <code>os</code>
+   * as its {@link OutputStream}.
+   *
+   * @param name             The name of the Appender.
+   * @param layout           The layout to format the message.
+   * @param filter
+   * @param ignoreExceptions
+   * @param immediateFlush
+   * @param manager          The OutputStreamManager.
+   */
+  protected StringAppender(String name,
+      Layout<? extends Serializable> layout, Filter filter,
+      boolean ignoreExceptions, boolean immediateFlush,
+      StringOutputStreamManager manager) {
+    super(name, layout, filter, ignoreExceptions, immediateFlush, manager);
+    this.manager = manager;
+  }
+
+  @PluginFactory
+  public static StringAppender createStringAppender(
+      @PluginAttribute("name") String nullablePatternString) {
+    PatternLayout layout;
+    if (nullablePatternString == null) {
+      layout = PatternLayout.createDefaultLayout();
+    } else {
+      layout = PatternLayout.createLayout(nullablePatternString, configuration,
+          null, null, true, false, null, null);
+    }
+
+    return new StringAppender(APPENDER_NAME, layout, null, false, true,
+        new StringOutputStreamManager(new ByteArrayOutputStream(), "StringStream", layout));
+  }
+
+  @VisibleForTesting
+  public void addToLogger(String loggerName, Level level) {
+    LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+    loggerConfig.addAppender(this, level, null);
+    context.updateLoggers();
+  }
+
+  @VisibleForTesting
+  public void removeFromLogger(String loggerName) {
+    LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+    loggerConfig.removeAppender(APPENDER_NAME);
+    context.updateLoggers();
+  }
+
+  public String getOutput() {
+    manager.flush();
+    return new String(manager.getStream().toByteArray());
+  }
+
+  public void reset() {
+    manager.reset();
+  }
+
+  protected static class StringOutputStreamManager extends OutputStreamManager {
+    ByteArrayOutputStream stream;
+
+    protected StringOutputStreamManager(ByteArrayOutputStream os, String streamName,
+        Layout<?> layout) {
+      super(os, streamName, layout);
+      stream = os;
+    }
+
+    public ByteArrayOutputStream getStream() {
+      return stream;
+    }
+
+    public void reset() {
+      stream.reset();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
index 99fbd5d..1e2feaa 100755
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.metadata;
 
 import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
 
-import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -29,8 +28,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
 
-import junit.framework.TestCase;
-
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -56,15 +53,19 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.hadoop.mapred.TextInputFormat;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-import org.apache.log4j.WriterAppender;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.junit.Assert;
 
 import com.google.common.collect.ImmutableMap;
 
+import junit.framework.TestCase;
+
 /**
  * TestHive.
  *
@@ -248,36 +249,39 @@ public class TestHive extends TestCase {
    * @throws Throwable
    */
   public void testMetaStoreApiTiming() throws Throwable {
-    // set log level to DEBUG, as this is logged at debug level
-    Logger logger = Logger.getLogger("hive.ql.metadata.Hive");
-    Level origLevel = logger.getLevel();
-    logger.setLevel(Level.DEBUG);
-
-    // create an appender to capture the logs in a string
-    StringWriter writer = new StringWriter();
-    WriterAppender appender = new WriterAppender(new PatternLayout(), writer);
+    // Get the RootLogger which, if you don't have log4j2-test.xml defined, will only log ERRORs
+    Logger logger = LogManager.getLogger("hive.ql.metadata.Hive");
+    Level oldLevel = logger.getLevel();
+    LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+    Configuration config = ctx.getConfiguration();
+    LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName());
+    loggerConfig.setLevel(Level.DEBUG);
+    ctx.updateLoggers();
+
+    // Create a String Appender to capture log output
+    StringAppender appender = StringAppender.createStringAppender("%m");
+    appender.addToLogger(logger.getName(), Level.DEBUG);
+    appender.start();
 
     try {
-      logger.addAppender(appender);
-
       hm.clearMetaCallTiming();
       hm.getAllDatabases();
       hm.dumpAndClearMetaCallTiming("test");
-      String logStr = writer.toString();
+      String logStr = appender.getOutput();
       String expectedString = "getAllDatabases_()=";
       Assert.assertTrue(logStr + " should contain <" + expectedString,
           logStr.contains(expectedString));
 
       // reset the log buffer, verify new dump without any api call does not contain func
-      writer.getBuffer().setLength(0);
+      appender.reset();
       hm.dumpAndClearMetaCallTiming("test");
-      logStr = writer.toString();
+      logStr = appender.getOutput();
       Assert.assertFalse(logStr + " should not contain <" + expectedString,
           logStr.contains(expectedString));
-
     } finally {
-      logger.setLevel(origLevel);
-      logger.removeAppender(appender);
+      loggerConfig.setLevel(oldLevel);
+      ctx.updateLoggers();
+      appender.removeFromLogger(logger.getName());
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java b/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
index 9d64b10..876ade8 100644
--- a/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
+++ b/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
@@ -18,9 +18,6 @@
 
 package org.apache.hive.service.cli;
 
-import org.apache.log4j.Layout;
-import org.apache.log4j.PatternLayout;
-
 /**
  * CLIServiceUtils.
  *
@@ -29,10 +26,6 @@ public class CLIServiceUtils {
 
 
   private static final char SEARCH_STRING_ESCAPE = '\\';
-  public static final Layout verboseLayout = new PatternLayout(
-    "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n");
-  public static final Layout nonVerboseLayout = new PatternLayout(
-    "%-5p : %m%n");
 
   /**
    * Convert a SQL search pattern into an equivalent Java Regex.

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java b/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
index 70340bd..fb3921f 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
@@ -6,44 +6,94 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
 package org.apache.hive.service.cli.operation;
-import java.io.CharArrayWriter;
-import java.util.Enumeration;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
 import java.util.regex.Pattern;
 
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.log.PerfLogger;
 import org.apache.hadoop.hive.ql.session.OperationLog;
-import org.apache.hadoop.hive.ql.session.OperationLog.LoggingLevel;
-import org.apache.hive.service.cli.CLIServiceUtils;
-import org.apache.log4j.Appender;
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.Layout;
-import org.apache.log4j.Logger;
-import org.apache.log4j.WriterAppender;
-import org.apache.log4j.spi.Filter;
-import org.apache.log4j.spi.LoggingEvent;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender;
+import org.apache.logging.log4j.core.appender.ConsoleAppender;
+import org.apache.logging.log4j.core.appender.OutputStreamManager;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.filter.AbstractFilter;
+import org.apache.logging.log4j.core.layout.PatternLayout;
 
 import com.google.common.base.Joiner;
 
 /**
- * An Appender to divert logs from individual threads to the LogObject they belong to.
+ * Divert appender to redirect operation logs to separate files.
  */
-public class LogDivertAppender extends WriterAppender {
-  private static final Logger LOG = Logger.getLogger(LogDivertAppender.class.getName());
+public class LogDivertAppender
+    extends AbstractOutputStreamAppender<LogDivertAppender.StringOutputStreamManager> {
+  private static final Logger LOG = LogManager.getLogger(LogDivertAppender.class.getName());
+  private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
+  private static Configuration configuration = context.getConfiguration();
+  public static final Layout<? extends Serializable> verboseLayout = PatternLayout.createLayout(
+      "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n", configuration, null, null, true, false, null, null);
+  public static final Layout<? extends Serializable> nonVerboseLayout = PatternLayout.createLayout(
+      "%-5p : %m%n", configuration, null, null, true, false, null, null);
+
   private final OperationManager operationManager;
+  private StringOutputStreamManager manager;
   private boolean isVerbose;
-  private Layout verboseLayout;
+  private final Layout<? extends Serializable> layout;
+
+  /**
+   * Instantiate a WriterAppender and set the output destination to a
+   * new {@link OutputStreamWriter} initialized with <code>os</code>
+   * as its {@link OutputStream}.
+   *
+   * @param name             The name of the Appender.
+   * @param filter           Filter
+   * @param manager          The OutputStreamManager.
+   * @param operationManager Operation manager
+   */
+  protected LogDivertAppender(String name, Filter filter,
+      StringOutputStreamManager manager, OperationManager operationManager,
+      OperationLog.LoggingLevel loggingMode) {
+    super(name, null, filter, false, true, manager);
+    this.operationManager = operationManager;
+    this.manager = manager;
+    this.isVerbose = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
+    this.layout = getDefaultLayout();
+  }
+
+  public Layout<? extends Serializable> getDefaultLayout() {
+    // There should be a ConsoleAppender. Copy its Layout.
+    Logger root = LogManager.getRootLogger();
+    Layout layout = null;
+
+    for (Appender ap : ((org.apache.logging.log4j.core.Logger) root).getAppenders().values()) {
+      if (ap.getClass().equals(ConsoleAppender.class)) {
+        layout = ap.getLayout();
+        break;
+      }
+    }
+
+    return layout;
+  }
 
   /**
    * A log filter that filters messages coming from the logger with the given names.
@@ -52,31 +102,31 @@ public class LogDivertAppender extends WriterAppender {
    * they don't generate more logs for themselves when they process logs.
    * White list filter is used for less verbose log collection
    */
-  private static class NameFilter extends Filter {
+  private static class NameFilter extends AbstractFilter {
     private Pattern namePattern;
-    private LoggingLevel loggingMode;
+    private OperationLog.LoggingLevel loggingMode;
     private OperationManager operationManager;
 
     /* Patterns that are excluded in verbose logging level.
      * Filter out messages coming from log processing classes, or we'll run an infinite loop.
      */
     private static final Pattern verboseExcludeNamePattern = Pattern.compile(Joiner.on("|").
-      join(new String[] {LOG.getName(), OperationLog.class.getName(),
-      OperationManager.class.getName()}));
+        join(new String[]{LOG.getName(), OperationLog.class.getName(),
+            OperationManager.class.getName()}));
 
     /* Patterns that are included in execution logging level.
      * In execution mode, show only select logger messages.
      */
     private static final Pattern executionIncludeNamePattern = Pattern.compile(Joiner.on("|").
-      join(new String[] {"org.apache.hadoop.mapreduce.JobSubmitter",
-      "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(),
-      "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor"}));
+        join(new String[]{"org.apache.hadoop.mapreduce.JobSubmitter",
+            "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(),
+            "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor"}));
 
     /* Patterns that are included in performance logging level.
      * In performance mode, show execution and performance logger messages.
      */
     private static final Pattern performanceIncludeNamePattern = Pattern.compile(
-      executionIncludeNamePattern.pattern() + "|" + PerfLogger.class.getName());
+        executionIncludeNamePattern.pattern() + "|" + PerfLogger.class.getName());
 
     private void setCurrentNamePattern(OperationLog.LoggingLevel mode) {
       if (mode == OperationLog.LoggingLevel.VERBOSE) {
@@ -88,26 +138,25 @@ public class LogDivertAppender extends WriterAppender {
       }
     }
 
-    public NameFilter(
-      OperationLog.LoggingLevel loggingMode, OperationManager op) {
+    public NameFilter(OperationLog.LoggingLevel loggingMode, OperationManager op) {
       this.operationManager = op;
       this.loggingMode = loggingMode;
       setCurrentNamePattern(loggingMode);
     }
 
     @Override
-    public int decide(LoggingEvent ev) {
+    public Result filter(LogEvent event) {
       OperationLog log = operationManager.getOperationLogByThread();
       boolean excludeMatches = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
 
       if (log == null) {
-        return Filter.DENY;
+        return Result.DENY;
       }
 
       OperationLog.LoggingLevel currentLoggingMode = log.getOpLoggingLevel();
       // If logging is disabled, deny everything.
       if (currentLoggingMode == OperationLog.LoggingLevel.NONE) {
-        return Filter.DENY;
+        return Result.DENY;
       }
       // Look at the current session's setting
       // and set the pattern and excludeMatches accordingly.
@@ -116,88 +165,58 @@ public class LogDivertAppender extends WriterAppender {
         setCurrentNamePattern(loggingMode);
       }
 
-      boolean isMatch = namePattern.matcher(ev.getLoggerName()).matches();
+      boolean isMatch = namePattern.matcher(event.getLoggerName()).matches();
 
       if (excludeMatches == isMatch) {
         // Deny if this is black-list filter (excludeMatches = true) and it
-        // matched
-        // or if this is whitelist filter and it didn't match
-        return Filter.DENY;
+        // matched or if this is whitelist filter and it didn't match
+        return Result.DENY;
       }
-      return Filter.NEUTRAL;
+      return Result.NEUTRAL;
     }
   }
 
-  /** This is where the log message will go to */
-  private final CharArrayWriter writer = new CharArrayWriter();
-
-  private void setLayout (boolean isVerbose, Layout lo) {
-    if (isVerbose) {
-      if (lo == null) {
-        lo = CLIServiceUtils.verboseLayout;
-        LOG.info("Cannot find a Layout from a ConsoleAppender. Using default Layout pattern.");
-      }
-    } else {
-      lo = CLIServiceUtils.nonVerboseLayout;
-    }
-    setLayout(lo);
+  public static LogDivertAppender createInstance(OperationManager operationManager,
+      OperationLog.LoggingLevel loggingMode) {
+    return new LogDivertAppender("LogDivertAppender", new NameFilter(loggingMode, operationManager),
+        new StringOutputStreamManager(new ByteArrayOutputStream(), "StringStream", null),
+        operationManager, loggingMode);
   }
 
-  private void initLayout(boolean isVerbose) {
-    // There should be a ConsoleAppender. Copy its Layout.
-    Logger root = Logger.getRootLogger();
-    Layout layout = null;
-
-    Enumeration<?> appenders = root.getAllAppenders();
-    while (appenders.hasMoreElements()) {
-      Appender ap = (Appender) appenders.nextElement();
-      if (ap.getClass().equals(ConsoleAppender.class)) {
-        layout = ap.getLayout();
-        break;
-      }
-    }
-    setLayout(isVerbose, layout);
+  public String getOutput() {
+    return new String(manager.getStream().toByteArray());
   }
 
-  public LogDivertAppender(OperationManager operationManager,
-    OperationLog.LoggingLevel loggingMode) {
-    isVerbose = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
-    initLayout(isVerbose);
-    setWriter(writer);
-    setName("LogDivertAppender");
-    this.operationManager = operationManager;
-    this.verboseLayout = isVerbose ? layout : CLIServiceUtils.verboseLayout;
-    addFilter(new NameFilter(loggingMode, operationManager));
+  @Override
+  public void start() {
+    super.start();
   }
 
   @Override
-  public void doAppend(LoggingEvent event) {
-    OperationLog log = operationManager.getOperationLogByThread();
+  public Layout<? extends Serializable> getLayout() {
 
-    // Set current layout depending on the verbose/non-verbose mode.
+    // If there is a logging level change from verbose->non-verbose or vice-versa since
+    // the last subAppend call, change the layout to preserve consistency.
+    OperationLog log = operationManager.getOperationLogByThread();
     if (log != null) {
-      boolean isCurrModeVerbose = (log.getOpLoggingLevel() == OperationLog.LoggingLevel.VERBOSE);
+      isVerbose = (log.getOpLoggingLevel() == OperationLog.LoggingLevel.VERBOSE);
+    }
 
-      // If there is a logging level change from verbose->non-verbose or vice-versa since
-      // the last subAppend call, change the layout to preserve consistency.
-      if (isCurrModeVerbose != isVerbose) {
-        isVerbose = isCurrModeVerbose;
-        setLayout(isVerbose, verboseLayout);
-      }
+    // layout is immutable in log4j2, so we cheat here and return a different layout when
+    // verbosity changes
+    if (isVerbose) {
+      return verboseLayout;
+    } else {
+      return layout == null ? nonVerboseLayout : layout;
     }
-    super.doAppend(event);
   }
 
-  /**
-   * Overrides WriterAppender.subAppend(), which does the real logging. No need
-   * to worry about concurrency since log4j calls this synchronously.
-   */
   @Override
-  protected void subAppend(LoggingEvent event) {
-    super.subAppend(event);
-    // That should've gone into our writer. Notify the LogContext.
-    String logOutput = writer.toString();
-    writer.reset();
+  public void append(LogEvent event) {
+    super.append(event);
+
+    String logOutput = getOutput();
+    manager.reset();
 
     OperationLog log = operationManager.getOperationLogByThread();
     if (log == null) {
@@ -206,4 +225,22 @@ public class LogDivertAppender extends WriterAppender {
     }
     log.writeOperationLog(logOutput);
   }
+
+  protected static class StringOutputStreamManager extends OutputStreamManager {
+    ByteArrayOutputStream stream;
+
+    protected StringOutputStreamManager(ByteArrayOutputStream os, String streamName,
+        Layout<?> layout) {
+      super(os, streamName, layout);
+      stream = os;
+    }
+
+    public ByteArrayOutputStream getStream() {
+      return stream;
+    }
+
+    public void reset() {
+      stream.reset();
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
index 9b0a519..304a525 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
@@ -41,8 +41,11 @@ import org.apache.hive.service.cli.RowSet;
 import org.apache.hive.service.cli.RowSetFactory;
 import org.apache.hive.service.cli.TableSchema;
 import org.apache.hive.service.cli.session.HiveSession;
-import org.apache.log4j.Appender;
-import org.apache.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
 
 /**
  * OperationManager.
@@ -50,7 +53,6 @@ import org.apache.log4j.Logger;
  */
 public class OperationManager extends AbstractService {
   private final Log LOG = LogFactory.getLog(OperationManager.class.getName());
-
   private final Map<OperationHandle, Operation> handleToOperation =
       new HashMap<OperationHandle, Operation>();
 
@@ -83,8 +85,13 @@ public class OperationManager extends AbstractService {
 
   private void initOperationLogCapture(String loggingMode) {
     // Register another Appender (with the same layout) that talks to us.
-    Appender ap = new LogDivertAppender(this, OperationLog.getLoggingLevel(loggingMode));
-    Logger.getRootLogger().addAppender(ap);
+    Appender ap = LogDivertAppender.createInstance(this, OperationLog.getLoggingLevel(loggingMode));
+    LoggerContext context = (LoggerContext) LogManager.getContext(false);
+    Configuration configuration = context.getConfiguration();
+    LoggerConfig loggerConfig = configuration.getLoggerConfig(LogManager.getLogger().getName());
+    loggerConfig.addAppender(ap, null, null);
+    context.updateLoggers();
+    ap.start();
   }
 
   public ExecuteStatementOperation newExecuteStatementOperation(HiveSession parentSession,

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/shims/common/pom.xml
----------------------------------------------------------------------
diff --git a/shims/common/pom.xml b/shims/common/pom.xml
index 9e9a3b7..dfdec2b 100644
--- a/shims/common/pom.xml
+++ b/shims/common/pom.xml
@@ -41,14 +41,19 @@
       <version>${commons-logging.version}</version>
     </dependency>
     <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>log4j</artifactId>
-      <version>${log4j.version}</version>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-1.2-api</artifactId>
+      <version>${log4j2.version}</version>
     </dependency>
     <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>apache-log4j-extras</artifactId>
-      <version>${log4j-extras.version}</version>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <version>${log4j2.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-jcl</artifactId>
+      <version>${log4j2.version}</version>
     </dependency>
     <dependency>
       <groupId>com.google.guava</groupId>

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java
deleted file mode 100644
index 224b135..0000000
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.shims;
-
-import org.apache.log4j.Appender;
-import org.apache.log4j.AppenderSkeleton;
-import org.apache.log4j.Layout;
-import org.apache.log4j.spi.ErrorHandler;
-import org.apache.log4j.spi.Filter;
-import org.apache.log4j.spi.LoggingEvent;
-import org.apache.log4j.spi.OptionHandler;
-
-public class HiveEventCounter implements Appender, OptionHandler {
-
-  AppenderSkeleton hadoopEventCounter;
-
-  public HiveEventCounter() {
-    hadoopEventCounter = ShimLoader.getEventCounter();
-  }
-
-  @Override
-  public void close() {
-    hadoopEventCounter.close();
-  }
-
-  @Override
-  public boolean requiresLayout() {
-    return hadoopEventCounter.requiresLayout();
-  }
-
-  @Override
-  public void addFilter(Filter filter) {
-    hadoopEventCounter.addFilter(filter);
-  }
-
-  @Override
-  public void clearFilters() {
-    hadoopEventCounter.clearFilters();
-  }
-
-  @Override
-  public void doAppend(LoggingEvent event) {
-    hadoopEventCounter.doAppend(event);
-  }
-
-  @Override
-  public ErrorHandler getErrorHandler() {
-    return hadoopEventCounter.getErrorHandler();
-  }
-
-  @Override
-  public Filter getFilter() {
-    return hadoopEventCounter.getFilter();
-  }
-
-  @Override
-  public Layout getLayout() {
-    return hadoopEventCounter.getLayout();
-  }
-
-  @Override
-  public String getName() {
-    return hadoopEventCounter.getName();
-  }
-
-  @Override
-  public void setErrorHandler(ErrorHandler handler) {
-    hadoopEventCounter.setErrorHandler(handler);
-  }
-
-  @Override
-  public void setLayout(Layout layout) {
-    hadoopEventCounter.setLayout(layout);
-  }
-
-  @Override
-  public void setName(String name) {
-    hadoopEventCounter.setName(name);
-  }
-
-  @Override
-  public void activateOptions() {
-    hadoopEventCounter.activateOptions();
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/spark-client/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/spark-client/src/test/resources/log4j.properties b/spark-client/src/test/resources/log4j.properties
deleted file mode 100644
index 93a60cc..0000000
--- a/spark-client/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Set everything to be logged to the file target/unit-tests.log
-log4j.rootCategory=DEBUG, console
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/spark-client/src/test/resources/log4j2.xml
----------------------------------------------------------------------
diff --git a/spark-client/src/test/resources/log4j2.xml b/spark-client/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..a435069
--- /dev/null
+++ b/spark-client/src/test/resources/log4j2.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="SparkClientLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+  <Properties>
+    <Property name="spark.log.level">DEBUG</Property>
+    <Property name="spark.root.logger">console</Property>
+  </Properties>
+
+  <Appenders>
+    <Console name="console" target="SYSTEM_ERR">
+      <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n"/>
+    </Console>
+  </Appenders>
+
+  <Loggers>
+    <Root level="DEBUG">
+      <AppenderRef ref="${sys:spark.root.logger}" level="${sys:spark.log.level}"/>
+    </Root>
+  </Loggers>
+
+</Configuration>

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/storage-api/pom.xml
----------------------------------------------------------------------
diff --git a/storage-api/pom.xml b/storage-api/pom.xml
index 71b51b8..71b79f1 100644
--- a/storage-api/pom.xml
+++ b/storage-api/pom.xml
@@ -32,13 +32,6 @@
   </properties>
 
   <dependencies>
-    <!-- dependencies are always listed in sorted order by groupId, artifectId -->
-    <!-- inter-project -->
-    <dependency>
-      <groupId>log4j</groupId>
-      <artifactId>log4j</artifactId>
-      <version>${log4j.version}</version>
-    </dependency>
     <!-- test inter-project -->
     <dependency>
       <groupId>junit</groupId>

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/testutils/ptest2/pom.xml
----------------------------------------------------------------------
diff --git a/testutils/ptest2/pom.xml b/testutils/ptest2/pom.xml
index 211678e..2cf7f45 100644
--- a/testutils/ptest2/pom.xml
+++ b/testutils/ptest2/pom.xml
@@ -64,6 +64,26 @@ limitations under the License.
       <version>15.0</version>
     </dependency>
     <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-1.2-api</artifactId>
+      <version>${log4j2.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-web</artifactId>
+      <version>${log4j2.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <version>${log4j2.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-jcl</artifactId>
+      <version>${log4j2.version}</version>
+    </dependency>
+    <dependency>
       <groupId>log4j</groupId>
       <artifactId>log4j</artifactId>
       <version>1.2.17</version>

http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/testutils/ptest2/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/log4j.properties b/testutils/ptest2/src/main/resources/log4j.properties
deleted file mode 100644
index edb9696..0000000
--- a/testutils/ptest2/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,37 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-hive.ptest.logdir=target
-
-log4j.rootLogger=DEBUG,FILE
-log4j.threshhold=ALL
-
-log4j.appender.FILE=org.apache.log4j.RollingFileAppender
-log4j.appender.FILE.File=${hive.ptest.logdir}/ptest.log
-log4j.appender.FILE.MaxFileSize=50MB
-log4j.appender.FILE.MaxBackupIndex=1
-log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.FILE.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-
-log4j.logger.org.apache.http=INFO
-log4j.logger.org.springframework=INFO
-log4j.logger.org.jclouds=INFO
-log4j.logger.jclouds=INFO
-log4j.logger.org.apache.hive=DEBUG
-log4j.logger.org.apache.http=TRACE
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN


Mime
View raw message