pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cheol...@apache.org
Subject svn commit: r1578679 - in /pig/trunk: src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/fetch/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/parser/ src/org/apache/pig/tools/pigstats/ src/org/a...
Date Tue, 18 Mar 2014 00:22:44 GMT
Author: cheolsoo
Date: Tue Mar 18 00:22:43 2014
New Revision: 1578679

URL: http://svn.apache.org/r1578679
Log:
PIG-3812: Several clean-ups in PigStats and JobStats (cheolsoo)

Added:
    pig/trunk/src/org/apache/pig/tools/pigstats/EmptyPigStats.java
Removed:
    pig/trunk/src/org/apache/pig/tools/pigstats/SimpleFetchPigStats.java
Modified:
    pig/trunk/src/org/apache/pig/PigServer.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
    pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
    pig/trunk/src/org/apache/pig/parser/SourceLocation.java
    pig/trunk/src/org/apache/pig/tools/pigstats/InputStats.java
    pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java
    pig/trunk/src/org/apache/pig/tools/pigstats/OutputStats.java
    pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
    pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java
    pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
    pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRJobStats.java
    pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRPigStatsUtil.java
    pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRScriptState.java
    pig/trunk/test/org/apache/pig/test/TestPigRunner.java

Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Tue Mar 18 00:22:43 2014
@@ -104,7 +104,7 @@ import org.apache.pig.tools.pigstats.Out
 import org.apache.pig.tools.pigstats.PigStats;
 import org.apache.pig.tools.pigstats.PigStats.JobGraph;
 import org.apache.pig.tools.pigstats.ScriptState;
-import org.apache.pig.tools.pigstats.SimpleFetchPigStats;
+import org.apache.pig.tools.pigstats.EmptyPigStats;
 import org.apache.pig.validator.BlackAndWhitelistFilter;
 import org.apache.pig.validator.BlackAndWhitelistValidator;
 import org.apache.pig.validator.PigCommandFilter;
@@ -233,17 +233,17 @@ public class PigServer {
         if (connect) {
             pigContext.connect();
         }
-        
+
         this.filter = new BlackAndWhitelistFilter(this);
 
         addJarsFromProperties();
         markPredeployedJarsFromProperties();
 
-        if (PigStats.get() == null) {
-            PigStats.start(pigContext.getExecutionEngine().instantiatePigStats());
-        }
+        PigStats.start(pigContext.getExecutionEngine().instantiatePigStats());
 
         if (ScriptState.get() == null) {
+            // If Pig was started via command line, ScriptState should have been
+            // already initialized in Main. If so, we should not overwrite it.
             ScriptState.start(pigContext.getExecutionEngine().instantiateScriptState());
         }
     }
@@ -277,7 +277,7 @@ public class PigServer {
     private void markPredeployedJarsFromProperties() throws ExecException {
         // mark jars as predeployed from properties
         String jar_str = pigContext.getProperties().getProperty("pig.predeployed.jars");
-		
+
         if(jar_str != null){
             // Use File.pathSeparator (":" on Linux, ";" on Windows)
             // to correctly handle path aggregates as they are represented
@@ -426,7 +426,7 @@ public class PigServer {
      */
     protected List<ExecJob> getJobs(PigStats stats) {
         LinkedList<ExecJob> jobs = new LinkedList<ExecJob>();
-        if (stats instanceof SimpleFetchPigStats) {
+        if (stats instanceof EmptyPigStats) {
             HJob job = new HJob(HJob.JOB_STATUS.COMPLETED, pigContext, stats.result(null)
                     .getPOStore(), null);
             jobs.add(job);
@@ -1061,7 +1061,6 @@ public class PigServer {
      * @param suffix Suffix of file names 
      * @throws IOException if the requested alias cannot be found.
      */
-    @SuppressWarnings("unchecked")
     public void explain(String alias,
                         String format,
                         boolean verbose,

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java Tue
Mar 18 00:22:43 2014
@@ -41,7 +41,7 @@ import org.apache.pig.impl.plan.PlanExce
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.util.UDFContext;
 import org.apache.pig.tools.pigstats.PigStats;
-import org.apache.pig.tools.pigstats.SimpleFetchPigStats;
+import org.apache.pig.tools.pigstats.EmptyPigStats;
 import org.joda.time.DateTimeZone;
 
 /**
@@ -78,7 +78,7 @@ public class FetchLauncher {
                 new DependencyOrderWalker<PhysicalOperator, PhysicalPlan>(pp));
         udfFinisher.visit();
 
-        return PigStats.start(new SimpleFetchPigStats(pigContext, poStore));
+        return PigStats.start(new EmptyPigStats(pigContext, poStore));
     }
 
     /**

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
(original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Tue Mar 18 00:22:43 2014
@@ -168,6 +168,7 @@ public class MapReduceLauncher extends L
         MRScriptState.get().addWorkflowAdjacenciesToConf(mrp, conf);
 
         // start collecting statistics
+        PigStats.start(pc.getExecutionEngine().instantiatePigStats());
         MRPigStatsUtil.startCollection(pc, statsJobClient, jcc, mrp);
 
         // Find all the intermediate data stores. The plan will be destroyed during compile/execution

Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Tue Mar 18 00:22:43 2014
@@ -491,7 +491,7 @@ public class QueryParserDriver {
 
         // sometimes the script has no filename, like when a string is passed to PigServer
for
         // example. See PIG-2866.
-        if (fname != null) {
+        if (!fname.isEmpty()) {
             FetchFileRet localFileRet = getMacroFile(fname);
             fname = localFileRet.file.getAbsolutePath();
         }
@@ -594,7 +594,7 @@ public class QueryParserDriver {
             ScriptState ss = ScriptState.get();
             if (ss != null) file = ss.getFileName();
         }
-        if (file != null && !file.equals(importFile)) {
+        if (!file.isEmpty() && !file.equals(importFile)) {
             sb.append("at ").append(file).append(", ");
         }
         sb.append("line ").append(t.getLine()).append("> ").append(header);

Modified: pig/trunk/src/org/apache/pig/parser/SourceLocation.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/SourceLocation.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/SourceLocation.java (original)
+++ pig/trunk/src/org/apache/pig/parser/SourceLocation.java Tue Mar 18 00:22:43 2014
@@ -83,7 +83,7 @@ public class SourceLocation {
             sb.append("\n");
         }
         sb.append( "<" );
-        if( file != null )
+        if( file != null && !file.isEmpty() )
             sb.append( "file " + file + ", " );
         sb.append( "line " + line +", column " + offset + "> " );
         

Added: pig/trunk/src/org/apache/pig/tools/pigstats/EmptyPigStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/EmptyPigStats.java?rev=1578679&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/EmptyPigStats.java (added)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/EmptyPigStats.java Tue Mar 18 00:22:43 2014
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.tools.pigstats;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.impl.PigContext;
+
+/**
+ * EmptyPigStats encapsulates dummy statistics of a fetch task, since during a
+ * fetch no MR jobs are executed
+ *
+ */
+public class EmptyPigStats extends PigStats {
+
+    private final List<OutputStats> outputStatsList;
+    private final List<InputStats> inputStatsList;
+    private final JobGraph emptyJobPlan = new JobGraph();
+
+    public EmptyPigStats() {
+        // initialize empty stats
+        OutputStats os = new OutputStats(null, -1, -1, true);
+        this.outputStatsList = Collections.unmodifiableList(Arrays.asList(os));
+
+        InputStats is = new InputStats(null, -1, -1, true);
+        this.inputStatsList = Collections.unmodifiableList(Arrays.asList(is));
+    }
+
+    public EmptyPigStats(PigContext pigContext, POStore poStore) {
+        super.pigContext = pigContext;
+        super.startTime = super.endTime = System.currentTimeMillis();
+        super.userId = System.getProperty("user.name");
+
+        Configuration conf = ConfigurationUtil.toConfiguration(pigContext.getProperties());
+
+        // initialize empty stats
+        OutputStats os = new OutputStats(null, -1, -1, true);
+        os.setConf(conf);
+        os.setPOStore(poStore);
+        this.outputStatsList = Collections.unmodifiableList(Arrays.asList(os));
+
+        InputStats is = new InputStats(null, -1, -1, true);
+        is.setConf(conf);
+        this.inputStatsList = Collections.unmodifiableList(Arrays.asList(is));
+    }
+
+    @Override
+    public JobClient getJobClient() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public boolean isEmbedded() {
+        return false;
+    }
+
+    @Override
+    public Map<String, List<PigStats>> getAllStats() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public List<String> getAllErrorMessages() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public JobGraph getJobGraph() {
+       return emptyJobPlan;
+    }
+
+    @Override
+    public List<String> getOutputLocations() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public List<String> getOutputNames() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public long getNumberBytes(String location) {
+        return -1L;
+    }
+
+    @Override
+    public long getNumberRecords(String location) {
+        return -1L;
+    }
+
+    @Override
+    public String getOutputAlias(String location) {
+        return null;
+    }
+
+    @Override
+    public long getSMMSpillCount() {
+        return 0L;
+    }
+
+    @Override
+    public long getProactiveSpillCountRecords() {
+        return 0L;
+    }
+
+    @Override
+    public long getProactiveSpillCountObjects() {
+        return 0L;
+    }
+
+    @Override
+    public long getBytesWritten() {
+        return 0L;
+    }
+
+    @Override
+    public long getRecordWritten() {
+        return 0L;
+    }
+
+    @Override
+    public int getNumberJobs() {
+        return 0;
+    }
+
+    @Override
+    public List<OutputStats> getOutputStats() {
+        return outputStatsList;
+    }
+
+    @Override
+    public OutputStats result(String alias) {
+        return outputStatsList.get(0);
+    }
+
+    @Override
+    public List<InputStats> getInputStats() {
+        return inputStatsList;
+    }
+
+    @Override
+    public void setBackendException(String jobId, Exception e) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void start() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void stop() {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int getNumberSuccessfulJobs() {
+        return -1;
+    }
+
+    @Override
+    public int getNumberFailedJobs() {
+        return -1;
+    }
+
+}

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/InputStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/InputStats.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/InputStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/InputStats.java Tue Mar 18 00:22:43 2014
@@ -36,22 +36,22 @@ public final class InputStats {
     private long records;
 
     private boolean success;
-    
+
     public static enum INPUT_TYPE { regular, sampler, indexer, side };
- 
+
     private INPUT_TYPE type = INPUT_TYPE.regular;
-    
+
     private Configuration conf;
 
     public InputStats(String location, long bytes, long records, boolean success) {
         this.location = location;
         this.bytes = bytes;
-        this.records = records;        
+        this.records = records;
         this.success = success;
         try {
             this.name = new Path(location).getName();
         } catch (Exception e) {
-            // location is a mal formatted URL 
+            // location is a mal formatted URL
             this.name = location;
         }
     }
@@ -79,14 +79,14 @@ public final class InputStats {
     public Configuration getConf() {
         return conf;
     }
-    
+
     public INPUT_TYPE getInputType() {
         return type;
     }
-    
+
     public String getDisplayString(boolean local) {
         StringBuilder sb = new StringBuilder();
-        if (success) {            
+        if (success) {
             sb.append("Successfully ");
             if (type == INPUT_TYPE.sampler) {
                 sb.append("sampled ");
@@ -95,7 +95,7 @@ public final class InputStats {
             } else {
                 sb.append("read ");
             }
-            
+
             if (!local && records >= 0) {
                 sb.append(records).append(" records ");
             } else {
@@ -115,19 +115,19 @@ public final class InputStats {
         }
         return sb.toString();
     }
-    
+
     public void setConf(Configuration conf) {
         this.conf = conf;
     }
-    
+
     public void markSampleInput() {
         type = INPUT_TYPE.sampler;
     }
-    
+
     public void markIndexerInput() {
         type = INPUT_TYPE.indexer;
     }
-    
+
     public void markSideFileInput() {
         type = INPUT_TYPE.side;
     }

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java Tue Mar 18 00:22:43 2014
@@ -24,8 +24,6 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.Counters;
 import org.apache.pig.classification.InterfaceAudience;
@@ -47,7 +45,6 @@ public abstract class JobStats extends O
     public static final String ALIAS_LOCATION = "JobStatistics:alias_location";
     public static final String FEATURE = "JobStatistics:feature";
 
-    private static final Log LOG = LogFactory.getLog(JobStats.class);
     public static final String SUCCESS_HEADER = null;
     public static final String FAILURE_HEADER = null;
 
@@ -61,6 +58,9 @@ public abstract class JobStats extends O
 
     protected Configuration conf;
 
+    protected long hdfsBytesRead = 0;
+    protected long hdfsBytesWritten = 0;
+
     private String errorMsg;
 
     private Exception exception = null;
@@ -112,6 +112,14 @@ public abstract class JobStats extends O
         return (String)getAnnotation(FEATURE);
     }
 
+    public long getHdfsBytesRead() {
+        return hdfsBytesRead;
+    }
+
+    public long getHdfsBytesWritten() {
+        return hdfsBytesWritten;
+    }
+
     /**
      * Returns the total bytes written to user specified HDFS
      * locations of this job.
@@ -241,7 +249,7 @@ public abstract class JobStats extends O
      * then use {@link org.apache.pig.tools.pigstats.mapreduce.MRJobStats#getAvgREduceTime}
instead.
      */
     @Deprecated
-    abstract public long getAvgREduceTime();
+    abstract public long getAvgReduceTime();
 
     /**
      * @deprecated If you are using mapreduce, please cast JobStats to org.apache.pig.tools.pigstats.mapreduce.MRJobStats,
@@ -259,17 +267,17 @@ public abstract class JobStats extends O
 
     /**
      * @deprecated If you are using mapreduce, please cast JobStats to org.apache.pig.tools.pigstats.mapreduce.MRJobStats,
-     * then use {@link org.apache.pig.tools.pigstats.mapreduce.MRJobStats#getReduceOutputRecords}
instead.
+     * then use {@link org.apache.pig.tools.pigstats.mapreduce.MRJobStats#getReduceInputRecords}
instead.
      */
     @Deprecated
-    abstract public long getReduceOutputRecords();
+    abstract public long getReduceInputRecords();
 
     /**
      * @deprecated If you are using mapreduce, please cast JobStats to org.apache.pig.tools.pigstats.mapreduce.MRJobStats,
-     * then use {@link org.apache.pig.tools.pigstats.mapreduce.MRJobStats#getReduceInputRecords}
instead.
+     * then use {@link org.apache.pig.tools.pigstats.mapreduce.MRJobStats#getReduceOutputRecords}
instead.
      */
     @Deprecated
-    abstract public long getReduceInputRecords();
+    abstract public long getReduceOutputRecords();
 
     /**
      * @deprecated If you are using mapreduce, please cast JobStats to org.apache.pig.tools.pigstats.mapreduce.MRJobStats,
@@ -294,23 +302,23 @@ public abstract class JobStats extends O
 
     /**
      * @deprecated If you are using mapreduce, please cast JobStats to org.apache.pig.tools.pigstats.mapreduce.MRJobStats,
-     * then use {@link org.apache.pig.tools.pigstats.mapreduce.MRJobStats#getHdfsBytesWritten}
instead.
+     * then use {@link org.apache.pig.tools.pigstats.mapreduce.MRJobStats#getHadoopCounters}
instead.
      */
     @Deprecated
-    abstract public long getHdfsBytesWritten();
+    abstract public Counters getHadoopCounters();
 
     /**
      * @deprecated If you are using mapreduce, please cast JobStats to org.apache.pig.tools.pigstats.mapreduce.MRJobStats,
-     * then use {@link org.apache.pig.tools.pigstats.mapreduce.MRJobStats#getHadoopCounters}
instead.
+     * then use {@link org.apache.pig.tools.pigstats.mapreduce.MRJobStats#getMultiStoreCounters}
instead.
      */
     @Deprecated
-    abstract public Counters getHadoopCounters();
+    abstract public Map<String, Long> getMultiStoreCounters();
 
     /**
      * @deprecated If you are using mapreduce, please cast JobStats to org.apache.pig.tools.pigstats.mapreduce.MRJobStats,
-     * then use {@link org.apache.pig.tools.pigstats.mapreduce.MRJobStats#getMultiStoreCounters}
instead.
+     * then use {@link org.apache.pig.tools.pigstats.mapreduce.MRJobStats#getMultiInputCounters}
instead.
      */
     @Deprecated
-    abstract public Map<String, Long> getMultiStoreCounters();
+    abstract public Map<String, Long> getMultiInputCounters();
 
 }

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/OutputStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/OutputStats.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/OutputStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/OutputStats.java Tue Mar 18 00:22:43 2014
@@ -35,7 +35,6 @@ import org.apache.pig.classification.Int
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.io.ReadToEndLoader;
-import org.apache.pig.newplan.Operator;
 
 /**
  * This class encapsulates the runtime statistics of an user specified output.

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java Tue Mar 18 00:22:43 2014
@@ -268,6 +268,10 @@ public abstract class PigStats {
         return ScriptState.get().getId();
     }
 
+    public String getFileName() {
+        return ScriptState.get().getFileName();
+    }
+
     public String getFeatures() {
         return ScriptState.get().getScriptFeatures();
     }

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/PigStatsUtil.java Tue Mar 18 00:22:43 2014
@@ -32,89 +32,60 @@ import org.apache.pig.tools.pigstats.map
  */
 public class PigStatsUtil {
 
-    /**
-     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#MULTI_STORE_RECORD_COUNTER}
instead.
-     */
-    @Deprecated
-    public static final String MULTI_STORE_RECORD_COUNTER
-            = "Output records in ";
-
-    /**
-     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#MULTI_STORE_COUNTER_GROUP}
instead.
-     */
-    @Deprecated
-    public static final String MULTI_STORE_COUNTER_GROUP
-            = "MultiStoreCounters";
-
-    /**
-     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#TASK_COUNTER_GROUP}
instead.
-     */
-    @Deprecated
-    public static final String TASK_COUNTER_GROUP
-            = "org.apache.hadoop.mapred.Task$Counter";
-
-    /**
-     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#FS_COUNTER_GROUP}
instead.
-     */
-    @Deprecated
-    public static final String FS_COUNTER_GROUP
-            = HadoopShims.getFsCounterGroupName();
-
-    /**
-     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#MAP_INPUT_RECORDS}
instead.
-     */
-    @Deprecated
     public static final String MAP_INPUT_RECORDS
             = "MAP_INPUT_RECORDS";
-
-    /**
-     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#MAP_OUTPUT_RECORDS}
instead.
-     */
-    @Deprecated
     public static final String MAP_OUTPUT_RECORDS
             = "MAP_OUTPUT_RECORDS";
+    public static final String REDUCE_INPUT_RECORDS
+            = "REDUCE_INPUT_RECORDS";
+    public static final String REDUCE_OUTPUT_RECORDS
+            = "REDUCE_OUTPUT_RECORDS";
+    public static final String HDFS_BYTES_WRITTEN
+            = "HDFS_BYTES_WRITTEN";
+    public static final String HDFS_BYTES_READ
+            = "HDFS_BYTES_READ";
 
     /**
-     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#REDUCE_INPUT_RECORDS}
instead.
+     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#MULTI_INPUT_RECORD_COUNTER}
instead.
      */
     @Deprecated
-    public static final String REDUCE_INPUT_RECORDS
-            = "REDUCE_INPUT_RECORDS";
+    public static final String MULTI_INPUTS_RECORD_COUNTER
+            = "Input records from ";
 
     /**
-     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#REDUCE_OUTPUT_RECORDS}
instead.
+     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#MULTI_INPUT_COUNTER_GROUP}
instead.
      */
     @Deprecated
-    public static final String REDUCE_OUTPUT_RECORDS
-            = "REDUCE_OUTPUT_RECORDS";
+    public static final String MULTI_INPUTS_COUNTER_GROUP
+            = "MultiInputCounters";
 
     /**
-     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#HDFS_BYTES_WRITTEN}
instead.
+     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#MULTI_STORE_RECORD_COUNTER}
instead.
      */
     @Deprecated
-    public static final String HDFS_BYTES_WRITTEN
-            = "HDFS_BYTES_WRITTEN";
+    public static final String MULTI_STORE_RECORD_COUNTER
+            = "Output records in ";
 
     /**
-     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#HDFS_BYTES_READ}
instead.
+     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#MULTI_STORE_COUNTER_GROUP}
instead.
      */
     @Deprecated
-    public static final String HDFS_BYTES_READ
-            = "HDFS_BYTES_READ";
+    public static final String MULTI_STORE_COUNTER_GROUP
+            = "MultiStoreCounters";
 
     /**
-     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#MULTI_INPUTS_RECORD_COUNTER}
instead.
+     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#TASK_COUNTER_GROUP}
instead.
      */
     @Deprecated
-    public static final String MULTI_INPUTS_RECORD_COUNTER
-            = "Input records from ";
+    public static final String TASK_COUNTER_GROUP
+            = "org.apache.hadoop.mapred.Task$Counter";
 
     /**
-     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#MULTI_INPUTS_COUNTER_GROUP}
instead.
+     * @deprecated use {@link org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil#FS_COUNTER_GROUP}
instead.
      */
     @Deprecated
-    public static final String MULTI_INPUTS_COUNTER_GROUP
-            = "MultiInputCounters";
+    public static final String FS_COUNTER_GROUP
+            = HadoopShims.getFsCounterGroupName();
 
     /**
      * Returns an empty PigStats object Use of this method is not advised as it
@@ -136,30 +107,35 @@ public class PigStatsUtil {
      */
     public static PigStats getPigStats(int code) {
         PigStats ps = PigStats.get();
+        if (ps == null) {
+            ps = PigStats.start(new EmptyPigStats());
+        }
         ps.setReturnCode(code);
         return ps;
     }
 
-
     public static void setErrorMessage(String msg) {
         PigStats ps = PigStats.get();
-        if (ps != null) {
-            ps.setErrorMessage(msg);
+        if (ps == null) {
+            ps = PigStats.start(new EmptyPigStats());
         }
+        ps.setErrorMessage(msg);
     }
 
     public static void setErrorCode(int code) {
         PigStats ps = PigStats.get();
-        if (ps != null) {
-            ps.setErrorCode(code);
+        if (ps == null) {
+            ps = PigStats.start(new EmptyPigStats());
         }
+        ps.setErrorCode(code);
     }
 
     public static void setErrorThrowable(Throwable t) {
         PigStats ps = PigStats.get();
-        if (ps != null) {
-            ps.setErrorThrowable(t);
+        if (ps == null) {
+            ps = PigStats.start(new EmptyPigStats());
         }
+        ps.setErrorThrowable(t);
     }
 
     private static Pattern pattern = Pattern.compile("tmp(-)?[\\d]{1,10}$");
@@ -173,6 +149,4 @@ public class PigStatsUtil {
         PigStats.start(new EmbeddedPigStats(statsMap));
     }
 
-
 }
-

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java Tue Mar 18 00:22:43 2014
@@ -252,7 +252,7 @@ public abstract class ScriptState {
     }
 
     public String getFileName() {
-        return fileName;
+        return (fileName == null) ? "" : fileName;
     }
 
     public void setFileName(String fileName) {

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRJobStats.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRJobStats.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRJobStats.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRJobStats.java Tue Mar 18 00:22:43
2014
@@ -88,7 +88,6 @@ public final class MRJobStats extends Jo
 
     private Boolean disableCounter = false;
 
-    @SuppressWarnings("deprecation")
     private JobID jobId;
 
     private long maxMapTime = 0;
@@ -107,8 +106,6 @@ public final class MRJobStats extends Jo
     private long mapOutputRecords = 0;
     private long reduceInputRecords = 0;
     private long reduceOutputRecords = 0;
-    private long hdfsBytesWritten = 0;
-    private long hdfsBytesRead = 0;
     private long spillCount = 0;
     private long activeSpillCountObj = 0;
     private long activeSpillCountRecs = 0;
@@ -119,10 +116,8 @@ public final class MRJobStats extends Jo
     private HashMap<String, Long> multiInputCounters
             = new HashMap<String, Long>();
 
-    @SuppressWarnings("deprecation")
     private Counters counters = null;
 
-
     public String getJobId() {
         return (jobId == null) ? null : jobId.toString();
     }
@@ -141,32 +136,32 @@ public final class MRJobStats extends Jo
 
     public long getMinReduceTime() { return minReduceTime; }
 
-    public long getAvgREduceTime() { return avgReduceTime; }
+    public long getAvgReduceTime() { return avgReduceTime; }
 
     public long getMapInputRecords() { return mapInputRecords; }
 
     public long getMapOutputRecords() { return mapOutputRecords; }
 
-    public long getReduceOutputRecords() { return reduceOutputRecords; }
-
     public long getReduceInputRecords() { return reduceInputRecords; }
 
+    public long getReduceOutputRecords() { return reduceOutputRecords; }
+
     public long getSMMSpillCount() { return spillCount; }
 
     public long getProactiveSpillCountObjects() { return activeSpillCountObj; }
 
     public long getProactiveSpillCountRecs() { return activeSpillCountRecs; }
 
-    public long getHdfsBytesWritten() { return hdfsBytesWritten; }
-
-    @SuppressWarnings("deprecation")
     public Counters getHadoopCounters() { return counters; }
 
-
     public Map<String, Long> getMultiStoreCounters() {
         return Collections.unmodifiableMap(multiStoreCounters);
     }
 
+    public Map<String, Long> getMultiInputCounters() {
+        return Collections.unmodifiableMap(multiInputCounters);
+    }
+
     public String getAlias() {
         return (String)getAnnotation(ALIAS);
     }
@@ -187,7 +182,6 @@ public final class MRJobStats extends Jo
         }
     }
 
-    @SuppressWarnings("deprecation")
     void setId(JobID jobId) {
         this.jobId = jobId;
     }
@@ -265,7 +259,6 @@ public final class MRJobStats extends Jo
         return sb.toString();
     }
 
-    @SuppressWarnings("deprecation")
     void addCounters(RunningJob rjob) {
         if (rjob != null) {
             try {

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRPigStatsUtil.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRPigStatsUtil.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRPigStatsUtil.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRPigStatsUtil.java Tue Mar 18 00:22:43
2014
@@ -90,7 +90,6 @@ public class MRPigStatsUtil extends PigS
      * @param counterName the counter name
      * @return the count of the given counter name
      */
-    @SuppressWarnings("deprecation")
     public static long getMultiStoreCount(Job job, JobClient jobClient,
             String counterName) {
         long value = -1;
@@ -166,7 +165,7 @@ public class MRPigStatsUtil extends PigS
      */
     public static void startCollection(PigContext pc, JobClient client,
             JobControlCompiler jcc, MROperPlan plan) {
-        SimplePigStats ps = (SimplePigStats)PigStats.start(new SimplePigStats());
+        SimplePigStats ps = (SimplePigStats)PigStats.get();
         ps.initialize(pc, client, jcc, plan);
 
         MRScriptState.get().emitInitialPlanNotification(plan);

Modified: pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRScriptState.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRScriptState.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRScriptState.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/mapreduce/MRScriptState.java Tue Mar 18 00:22:43
2014
@@ -211,7 +211,7 @@ public class MRScriptState extends Scrip
         setJobParents(mro, conf);
 
         conf.set("mapreduce.workflow.id", "pig_" + id);
-        conf.set("mapreduce.workflow.name", (getFileName() != null) ? getFileName() : "default");
+        conf.set("mapreduce.workflow.name", getFileName().isEmpty() ? "default" : getFileName());
         conf.set("mapreduce.workflow.node.name", mro.getOperatorKey().toString());
     }
 

Modified: pig/trunk/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigRunner.java?rev=1578679&r1=1578678&r2=1578679&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigRunner.java Tue Mar 18 00:22:43 2014
@@ -51,7 +51,7 @@ import org.apache.pig.tools.pigstats.Out
 import org.apache.pig.tools.pigstats.PigProgressNotificationListener;
 import org.apache.pig.tools.pigstats.PigStats;
 import org.apache.pig.tools.pigstats.PigStatsUtil;
-import org.apache.pig.tools.pigstats.SimpleFetchPigStats;
+import org.apache.pig.tools.pigstats.EmptyPigStats;
 import org.apache.pig.tools.pigstats.mapreduce.MRJobStats;
 import org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil;
 import org.junit.AfterClass;
@@ -192,7 +192,7 @@ public class TestPigRunner {
             String[] args = { "-Dstop.on.failure=true", "-Dopt.multiquery=false", "-Daggregate.warning=false",
PIG_FILE };
             PigStats stats = PigRunner.run(args, new TestNotificationListener());
 
-            assertTrue(stats instanceof SimpleFetchPigStats);
+            assertTrue(stats instanceof EmptyPigStats);
             assertTrue(stats.isSuccessful());
             assertEquals(0, stats.getNumberJobs());
             assertEquals(stats.getJobGraph().size(), 0);



Mime
View raw message