pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cheol...@apache.org
Subject svn commit: r1578718 [1/2] - in /pig/branches/tez: ./ conf/ src/ src/docs/src/documentation/content/xdocs/ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/datastorage/ src/org/apache/pig/backend/hadoop/executionengine/ src/org/apache/pig/backend/...
Date Tue, 18 Mar 2014 02:19:25 GMT
Author: cheolsoo
Date: Tue Mar 18 02:19:24 2014
New Revision: 1578718

URL: http://svn.apache.org/r1578718
Log:
Merge latest trunk changes

Added:
    pig/branches/tez/src/org/apache/pig/newplan/logical/rules/LogicalRelationalNodeValidator.java
      - copied unchanged from r1578679, pig/trunk/src/org/apache/pig/newplan/logical/rules/LogicalRelationalNodeValidator.java
    pig/branches/tez/src/org/apache/pig/tools/pigstats/EmptyPigStats.java
      - copied unchanged from r1578679, pig/trunk/src/org/apache/pig/tools/pigstats/EmptyPigStats.java
    pig/branches/tez/src/org/apache/pig/validator/
      - copied from r1578679, pig/trunk/src/org/apache/pig/validator/
    pig/branches/tez/src/org/apache/pig/validator/BlackAndWhitelistFilter.java
      - copied unchanged from r1578679, pig/trunk/src/org/apache/pig/validator/BlackAndWhitelistFilter.java
    pig/branches/tez/src/org/apache/pig/validator/BlackAndWhitelistValidator.java
      - copied unchanged from r1578679, pig/trunk/src/org/apache/pig/validator/BlackAndWhitelistValidator.java
    pig/branches/tez/src/org/apache/pig/validator/PigCommandFilter.java
      - copied unchanged from r1578679, pig/trunk/src/org/apache/pig/validator/PigCommandFilter.java
    pig/branches/tez/test/org/apache/pig/test/TestBlackAndWhitelistValidator.java
      - copied unchanged from r1578679, pig/trunk/test/org/apache/pig/test/TestBlackAndWhitelistValidator.java
Removed:
    pig/branches/tez/src/org/apache/pig/tools/pigstats/SimpleFetchPigStats.java
Modified:
    pig/branches/tez/   (props changed)
    pig/branches/tez/CHANGES.txt
    pig/branches/tez/conf/pig.properties
    pig/branches/tez/src/docs/src/documentation/content/xdocs/basic.xml
    pig/branches/tez/src/org/apache/pig/PigConfiguration.java
    pig/branches/tez/src/org/apache/pig/PigConstants.java
    pig/branches/tez/src/org/apache/pig/PigServer.java
    pig/branches/tez/src/org/apache/pig/PigWarning.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/InputSizeReducerEstimator.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
    pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
    pig/branches/tez/src/org/apache/pig/builtin/ToString.java
    pig/branches/tez/src/org/apache/pig/impl/util/Utils.java
    pig/branches/tez/src/org/apache/pig/newplan/logical/Util.java
    pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LOGenerate.java
    pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LogicalPlanData.java
    pig/branches/tez/src/org/apache/pig/newplan/logical/rules/InputOutputFileValidator.java
    pig/branches/tez/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java
    pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java
    pig/branches/tez/src/org/apache/pig/parser/QueryParserDriver.java
    pig/branches/tez/src/org/apache/pig/parser/SourceLocation.java
    pig/branches/tez/src/org/apache/pig/scripting/BoundScript.java
    pig/branches/tez/src/org/apache/pig/tools/ToolsPigServer.java
    pig/branches/tez/src/org/apache/pig/tools/grunt/Grunt.java
    pig/branches/tez/src/org/apache/pig/tools/grunt/GruntParser.java
    pig/branches/tez/src/org/apache/pig/tools/pigstats/PigStatsUtil.java
    pig/branches/tez/src/org/apache/pig/tools/pigstats/mapreduce/MRScriptState.java
    pig/branches/tez/src/pig-default.properties   (props changed)
    pig/branches/tez/test/e2e/pig/conf/local.conf
    pig/branches/tez/test/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/TestInputSizeReducerEstimator.java
    pig/branches/tez/test/org/apache/pig/pigunit/pig/GruntParser.java
    pig/branches/tez/test/org/apache/pig/pigunit/pig/PigServer.java
    pig/branches/tez/test/org/apache/pig/test/TestLogicalPlanBuilder.java
    pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java
    pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java
    pig/branches/tez/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java
    pig/branches/tez/test/org/apache/pig/test/TestPigRunner.java
    pig/branches/tez/test/org/apache/pig/test/TestPigServer.java
    pig/branches/tez/test/org/apache/pig/test/Util.java
    pig/branches/tez/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2   (props changed)
    pig/branches/tez/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java
    pig/branches/tez/test/org/apache/pig/tools/grunt/TestGruntParser.java
    pig/branches/tez/test/perf/pigmix/bin/runpigmix.pl

Propchange: pig/branches/tez/
------------------------------------------------------------------------------
  Merged /pig/trunk:r1571422-1578679

Modified: pig/branches/tez/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/tez/CHANGES.txt?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/CHANGES.txt (original)
+++ pig/branches/tez/CHANGES.txt Tue Mar 18 02:19:24 2014
@@ -30,6 +30,16 @@ PIG-2207: Support custom counters for ag
 
 IMPROVEMENTS
 
+PIG-3802: Fix TestBlackAndWhitelistValidator failures (prkommireddi)
+
+PIG-3765: Ability to disable Pig commands and operators (prkommireddi)
+
+PIG-3731: Ability to specify local-mode specific configuration (useful for local/auto-local mode) (aniket486)
+
+PIG-3793: Provide info on number of LogicalRelationalOperator(s) used in the script through LogicalPlanData (prkommireddi)
+
+PIG-3778: Log list of running jobs along with progress (rohini)
+
 PIG-3675: Documentation for AccumuloStorage (elserj via daijy)
 
 PIG-3648: Make the sample size for RandomSampleLoader configurable (cheolsoo)
@@ -89,6 +99,22 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-3673: Divide by zero error in runpigmix.pl script (suhassatish via daijy)
+
+PIG-3805: ToString(datetime [, format string]) doesn't work without the second argument (jennythompson via daijy)
+
+PIG-3809: AddForEach optimization doesn't set the alias of the added foreach (cheolsoo)
+
+PIG-3811: PigServer.registerScript() wraps exception incorrectly on parsing errors (prkommireddi)
+
+PIG-3806: PigServer constructor throws NPE after PIG-3765 (aniket486)
+
+PIG-3801: Auto local mode does not call storeSchema (aniket486)
+
+PIG-3754: InputSizeReducerEstimator.getTotalInputFileSize reports incorrect size (aniket486)
+
+PIG-3679: e2e StreamingPythonUDFs_10 fails in trunk (cheolsoo)
+
 PIG-3776: Conflicting versions of jline is present in trunk (cheolsoo)
 
 PIG-3674: Fix TestAccumuloPigCluster on Hadoop 2 (elserj via daijy)
@@ -101,8 +127,6 @@ PIG-3747: Update skewed join documentati
 
 PIG-3755: auto local mode selection does not check lower bound for size (aniket486)
 
-PIG-3753: LOGenerate generates null schema (daijy)
-
 PIG-3447: Compiler warning message dropped for CastLineageSetter and others with no enum kind (knoguchi via cheolsoo)
 
 PIG-3627: Json storage : Doesn't work in cases , where other Store Functions (like PigStorage / AvroStorage)
@@ -227,6 +251,14 @@ PIG-3480: TFile-based tmpfile compressio
 
 BUG FIXES
 
+PIG-3753: LOGenerate generates null schema (daijy)
+
+PIG-3782: PushDownForEachFlatten + ColumnMapKeyPrune with user defined schema failing due to incorrect UID assignment (knoguchi via daijy)
+
+PIG-3779: Assert constructs ConstantExpression with null when no comment is given (thedatachef via cheolsoo)
+
+PIG-3777: Pig 12.0 Documentation (karinahauser via daijy)
+
 PIG-3774: Piggybank Over UDF get wrong result (daijy)
 
 PIG-3657: New partition filter extractor fails with NPE (cheolsoo)

Modified: pig/branches/tez/conf/pig.properties
URL: http://svn.apache.org/viewvc/pig/branches/tez/conf/pig.properties?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/conf/pig.properties (original)
+++ pig/branches/tez/conf/pig.properties Tue Mar 18 02:19:24 2014
@@ -259,3 +259,18 @@ pig.location.check.strict=false
 # Additional jar will be cached under PIG_USER_CACHE_LOCATION/${user.name}/.pigcache
 # and will be re-used across the jobs run by the user if the jar has not changed.
 # pig.user.cache.location=/tmp
+
+####################################################################################
+
+# Comma-delimited entries of commands/operators that must be disallowed. This is a
+# security feature to be used by administrators to block use of commands by users.
+# For eg, an admin might like to block all filesystem commands and setting configs
+# in pig script. In which case, the entry would be
+# pig.blacklist=fs,set
+
+# Comma-delimited entries of commands/operators that must be allowed. This is a
+# security feature to be used by administrators to block use of commands by users
+# that are not a part of the whitelist.
+# For eg, an admin might like to allow only LOAD, STORE, FILTER, GROUP
+# in pig script. In which case, the entry would be
+# pig.whitelist=load,store,filter,group

Modified: pig/branches/tez/src/docs/src/documentation/content/xdocs/basic.xml
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/docs/src/documentation/content/xdocs/basic.xml?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/docs/src/documentation/content/xdocs/basic.xml (original)
+++ pig/branches/tez/src/docs/src/documentation/content/xdocs/basic.xml Tue Mar 18 02:19:24 2014
@@ -7983,7 +7983,7 @@ X = SAMPLE A 0.01;
 a = load 'a.txt';
 b = group a all;
 c = foreach b generate COUNT(a) as num_rows;
-e = sample d 1000/num_rows;
+e = sample a 1000/c.num_rows;
 </source>
    </section></section>  
    

Modified: pig/branches/tez/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/PigConfiguration.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/branches/tez/src/org/apache/pig/PigConfiguration.java Tue Mar 18 02:19:24 2014
@@ -230,5 +230,22 @@ public class PigConfiguration {
      * and will be re-used across the jobs run by the user if the jar has not changed
      */
     public static final String PIG_USER_CACHE_LOCATION = "pig.user.cache.location";
-}
 
+    /**
+     * Comma-delimited entries of commands/operators that must be disallowed.
+     * This is a security feature to be used by administrators to block use of
+     * commands by users. For eg, an admin might like to block all filesystem
+     * commands and setting configs in pig script. In which case, the entry
+     * would be "pig.blacklist=fs,set"
+     */
+    public static final String PIG_BLACKLIST = "pig.blacklist";
+
+    /**
+     * Comma-delimited entries of commands/operators that must be allowed. This
+     * is a security feature to be used by administrators to block use of
+     * commands by users that are not a part of the whitelist. For eg, an admin
+     * might like to allow only LOAD, STORE, FILTER, GROUP in pig script. In
+     * which case, the entry would be "pig.whitelist=load,store,filter,group"
+     */
+    public static final String PIG_WHITELIST = "pig.whitelist";
+}

Modified: pig/branches/tez/src/org/apache/pig/PigConstants.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/PigConstants.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/PigConstants.java (original)
+++ pig/branches/tez/src/org/apache/pig/PigConstants.java Tue Mar 18 02:19:24 2014
@@ -47,4 +47,9 @@ public class PigConstants {
      */
     public static final String PIG_OPTIMIZER_RULES_DISABLED_KEY = "pig.optimizer.rules.disabled";
 
+    /**
+     * Prefix used by pig to configure local mode configuration
+     */
+    public static final String PIG_LOCAL_CONF_PREFIX = "pig.local.";
+
 }
\ No newline at end of file

Modified: pig/branches/tez/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/PigServer.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/PigServer.java (original)
+++ pig/branches/tez/src/org/apache/pig/PigServer.java Tue Mar 18 02:19:24 2014
@@ -104,7 +104,10 @@ import org.apache.pig.tools.pigstats.Out
 import org.apache.pig.tools.pigstats.PigStats;
 import org.apache.pig.tools.pigstats.PigStats.JobGraph;
 import org.apache.pig.tools.pigstats.ScriptState;
-import org.apache.pig.tools.pigstats.SimpleFetchPigStats;
+import org.apache.pig.tools.pigstats.EmptyPigStats;
+import org.apache.pig.validator.BlackAndWhitelistFilter;
+import org.apache.pig.validator.BlackAndWhitelistValidator;
+import org.apache.pig.validator.PigCommandFilter;
 
 import com.google.common.annotations.VisibleForTesting;
 
@@ -158,6 +161,8 @@ public class PigServer {
     private boolean validateEachStatement = false;
     private boolean skipParseInRegisterForBatch = false;
 
+    private final BlackAndWhitelistFilter filter;
+
     private String constructScope() {
         // scope servers for now as a session id
 
@@ -241,13 +246,16 @@ public class PigServer {
             pigContext.connect();
         }
 
+        this.filter = new BlackAndWhitelistFilter(this);
+
         addJarsFromProperties();
         markPredeployedJarsFromProperties();
 
         PigStats.start(pigContext.getExecutionEngine().instantiatePigStats());
-        // ScriptState may have been initialized in Main. In that case, we
-        // should not overwrite it.
+
         if (ScriptState.get() == null) {
+            // If Pig was started via command line, ScriptState should have been
+            // already initialized in Main. If so, we should not overwrite it.
             ScriptState.start(pigContext.getExecutionEngine().instantiateScriptState());
         }
     }
@@ -430,7 +438,7 @@ public class PigServer {
      */
     protected List<ExecJob> getJobs(PigStats stats) {
         LinkedList<ExecJob> jobs = new LinkedList<ExecJob>();
-        if (stats instanceof SimpleFetchPigStats) {
+        if (stats instanceof EmptyPigStats) {
             HJob job = new HJob(HJob.JOB_STATUS.COMPLETED, pigContext, stats.result(null)
                     .getPOStore(), null);
             jobs.add(job);
@@ -540,6 +548,9 @@ public class PigServer {
      * @throws IOException
      */
     public void registerJar(String name) throws IOException {
+        // Check if this operation is permitted
+        filter.validate(PigCommandFilter.Command.REGISTER);
+
         if (pigContext.hasJar(name)) {
             log.debug("Ignoring duplicate registration for jar " + name);
             return;
@@ -694,13 +705,12 @@ public class PigServer {
     public void registerScript(InputStream in, Map<String,String> params,List<String> paramsFiles) throws IOException {
         try {
             String substituted = pigContext.doParamSubstitution(in, paramMapToList(params), paramsFiles);
-            GruntParser grunt = new GruntParser(new StringReader(substituted));
+            GruntParser grunt = new GruntParser(new StringReader(substituted), this);
             grunt.setInteractive(false);
-            grunt.setParams(this);
             grunt.parseStopOnError(true);
         } catch (org.apache.pig.tools.pigscript.parser.ParseException e) {
             log.error(e.getLocalizedMessage());
-            throw new IOException(e.getCause());
+            throw new IOException(e);
         }
     }
 
@@ -1063,7 +1073,6 @@ public class PigServer {
      * @param suffix Suffix of file names
      * @throws IOException if the requested alias cannot be found.
      */
-    @SuppressWarnings("unchecked")
     public void explain(String alias,
                         String format,
                         boolean verbose,
@@ -1175,10 +1184,14 @@ public class PigServer {
      * @throws IOException
      */
     public boolean deleteFile(String filename) throws IOException {
+        // Check if this operation is permitted
+        filter.validate(PigCommandFilter.Command.RM);
+        filter.validate(PigCommandFilter.Command.RMF);
+
         ElementDescriptor elem = pigContext.getDfs().asElement(filename);
         elem.delete();
         return true;
-    }
+   }
 
     /**
      * Rename a file.
@@ -1188,6 +1201,9 @@ public class PigServer {
      * @throws IOException
      */
     public boolean renameFile(String source, String target) throws IOException {
+        // Check if this operation is permitted
+        filter.validate(PigCommandFilter.Command.MV);
+
         pigContext.rename(source, target);
         return true;
     }
@@ -1199,6 +1215,9 @@ public class PigServer {
      * @throws IOException
      */
     public boolean mkdirs(String dirs) throws IOException {
+        // Check if this operation is permitted
+        filter.validate(PigCommandFilter.Command.MKDIR);
+
         ContainerDescriptor container = pigContext.getDfs().asContainer(dirs);
         container.create();
         return true;
@@ -1211,6 +1230,9 @@ public class PigServer {
      * @throws IOException
      */
     public String[] listPaths(String dir) throws IOException {
+        // Check if this operation is permitted
+        filter.validate(PigCommandFilter.Command.LS);
+
         Collection<String> allPaths = new ArrayList<String>();
         ContainerDescriptor container = pigContext.getDfs().asContainer(dir);
         Iterator<ElementDescriptor> iter = container.iterator();
@@ -1347,9 +1369,13 @@ public class PigServer {
         return stats;
     }
 
-    private PigStats executeCompiledLogicalPlan() throws ExecException, FrontendException {
+    private PigStats executeCompiledLogicalPlan() throws ExecException,
+            FrontendException {
         // discover pig features used in this script
-        ScriptState.get().setScriptFeatures( currDAG.lp );
+        ScriptState.get().setScriptFeatures(currDAG.lp);
+
+        BlackAndWhitelistValidator validator = new BlackAndWhitelistValidator(getPigContext(), currDAG.lp);
+        validator.validate();
 
         return launchPlan(currDAG.lp, "job_pigexec_");
     }

Modified: pig/branches/tez/src/org/apache/pig/PigWarning.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/PigWarning.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/PigWarning.java (original)
+++ pig/branches/tez/src/org/apache/pig/PigWarning.java Tue Mar 18 02:19:24 2014
@@ -67,6 +67,7 @@ public enum PigWarning {
     NULL_COUNTER_COUNT,
     DELETE_FAILED,
     PROJECTION_INVALID_RANGE,
-    NO_LOAD_FUNCTION_FOR_CASTING_BYTEARRAY
+    NO_LOAD_FUNCTION_FOR_CASTING_BYTEARRAY,
+    SKIP_UDF_CALL_FOR_NULL
     ;
 }

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java Tue Mar 18 02:19:24 2014
@@ -26,6 +26,7 @@ import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.pig.ExecType;
+import org.apache.pig.PigConstants;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
 import org.apache.pig.backend.hadoop.executionengine.shims.HadoopShims;
 import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
@@ -89,4 +90,15 @@ public class ConfigurationUtil {
         Properties props = ConfigurationUtil.toProperties(localConf);
         return props;
     }
+
+    public static void replaceConfigForLocalMode(Configuration configuration) {
+        for (Entry<String, String> entry : configuration) {
+            String key = entry.getKey();
+            String value = entry.getValue();
+            if(key.startsWith(PigConstants.PIG_LOCAL_CONF_PREFIX)) {
+                String realConfKey = key.substring(PigConstants.PIG_LOCAL_CONF_PREFIX.length());
+                configuration.set(realConfKey, value);
+            }
+        }
+    }
 }

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Tue Mar 18 02:19:24 2014
@@ -64,10 +64,12 @@ import org.apache.pig.newplan.logical.re
 import org.apache.pig.newplan.logical.relational.LogicalPlan;
 import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
 import org.apache.pig.newplan.logical.rules.InputOutputFileValidator;
+import org.apache.pig.newplan.logical.rules.LogicalRelationalNodeValidator;
 import org.apache.pig.newplan.logical.visitor.SortInfoSetter;
 import org.apache.pig.newplan.logical.visitor.StoreAliasSetter;
 import org.apache.pig.pen.POOptimizeDisabler;
 import org.apache.pig.tools.pigstats.PigStats;
+import org.apache.pig.validator.BlackAndWhitelistValidator;
 
 import com.google.common.base.Splitter;
 import com.google.common.collect.Lists;
@@ -308,9 +310,15 @@ public abstract class HExecutionEngine i
             // Validate input/output file. Currently no validation framework in
             // new logical plan, put this validator here first.
             // We might decide to move it out to a validator framework in future
-            InputOutputFileValidator validator = new InputOutputFileValidator(
+            LogicalRelationalNodeValidator validator = new InputOutputFileValidator(
                     plan, pigContext);
             validator.validate();
+
+            // Check for blacklist and whitelist properties and disable
+            // commands/operators accordingly. Note if a user does not
+            // specify these, Pig will work without any filters or validations
+            validator = new BlackAndWhitelistValidator(pigContext, plan);
+            validator.validate();
         }
 
         // translate new logical plan to physical plan

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/fetch/FetchLauncher.java Tue Mar 18 02:19:24 2014
@@ -41,7 +41,7 @@ import org.apache.pig.impl.plan.PlanExce
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.util.UDFContext;
 import org.apache.pig.tools.pigstats.PigStats;
-import org.apache.pig.tools.pigstats.SimpleFetchPigStats;
+import org.apache.pig.tools.pigstats.EmptyPigStats;
 import org.joda.time.DateTimeZone;
 
 /**
@@ -78,7 +78,7 @@ public class FetchLauncher {
                 new DependencyOrderWalker<PhysicalOperator, PhysicalPlan>(pp));
         udfFinisher.visit();
 
-        return PigStats.start(new SimpleFetchPigStats(pigContext, poStore));
+        return PigStats.start(new EmptyPigStats(pigContext, poStore));
     }
 
     /**

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/InputSizeReducerEstimator.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/InputSizeReducerEstimator.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/InputSizeReducerEstimator.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/InputSizeReducerEstimator.java Tue Mar 18 02:19:24 2014
@@ -97,33 +97,35 @@ public class InputSizeReducerEstimator i
      * their size nor can pig look that up itself are excluded from this size.
      */
     static long getTotalInputFileSize(Configuration conf,
-                                      List<POLoad> lds, Job job) throws IOException {
+            List<POLoad> lds, Job job) throws IOException {
         long totalInputFileSize = 0;
-        boolean foundSize = false;
         for (POLoad ld : lds) {
             long size = getInputSizeFromLoader(ld, job);
-            if (size > -1) { foundSize = true; }
-            if (size > 0) {
+            if (size > -1) {
                 totalInputFileSize += size;
                 continue;
-            }
-            // the input file location might be a list of comma separated files,
-            // separate them out
-            for (String location : LoadFunc.getPathStrings(ld.getLFile().getFileName())) {
-                if (UriUtil.isHDFSFileOrLocalOrS3N(location)) {
-                    Path path = new Path(location);
-                    FileSystem fs = path.getFileSystem(conf);
-                    FileStatus[] status = fs.globStatus(path);
-                    if (status != null) {
-                        for (FileStatus s : status) {
-                            totalInputFileSize += MapRedUtil.getPathLength(fs, s);
-                            foundSize = true;
+            } else {
+
+                // the input file location might be a list of comma separated files,
+                // separate them out
+                for (String location : LoadFunc.getPathStrings(ld.getLFile().getFileName())) {
+                    if (UriUtil.isHDFSFileOrLocalOrS3N(location)) {
+                        Path path = new Path(location);
+                        FileSystem fs = path.getFileSystem(conf);
+                        FileStatus[] status = fs.globStatus(path);
+                        if (status != null) {
+                            for (FileStatus s : status) {
+                                totalInputFileSize += MapRedUtil.getPathLength(fs, s);
+                            }
                         }
+                    } else {
+                        // If we cannot estimate size of a location, we should report -1
+                        return -1;
                     }
                 }
             }
         }
-        return foundSize ? totalInputFileSize : -1;
+        return totalInputFileSize;
     }
 
     /**

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Tue Mar 18 02:19:24 2014
@@ -66,6 +66,7 @@ import org.apache.pig.PigException;
 import org.apache.pig.StoreFuncInterface;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.HDataType;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
 import org.apache.pig.backend.hadoop.executionengine.JobCreationException;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.SecondaryKeyPartitioner;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.SkewedPartitioner;
@@ -584,6 +585,10 @@ public class JobControlCompiler{
                             // we don't want to change fs settings back
                             continue;
                         }
+                        if (key.startsWith("io.")) {
+                            // we don't want to change io settings back
+                            continue;
+                        }
                         String value = entry.getValue();
                         if (conf.get(key) == null || !conf.get(key).equals(value)) {
                             conf.set(key, value);
@@ -616,6 +621,10 @@ public class JobControlCompiler{
                     conf.set("mapred.jar", submitJarFile.getPath());
                 }
             }
+
+            if(Utils.isLocal(pigContext, conf)) {
+                ConfigurationUtil.replaceConfigForLocalMode(conf);
+            }
             conf.set("pig.inputs", ObjectSerializer.serialize(inp));
             conf.set("pig.inpTargets", ObjectSerializer.serialize(inpTargets));
             conf.set("pig.inpSignatures", ObjectSerializer.serialize(inpSignatureLists));
@@ -1522,10 +1531,6 @@ public class JobControlCompiler{
         }
     }
 
-    private static boolean isLocal(PigContext pigContext, Configuration conf) {
-        return pigContext.getExecType().isLocal() || conf.getBoolean(PigImplConstants.CONVERTED_TO_LOCAL, false);
-    }
-
     private static String addSingleFileToDistributedCache(
             PigContext pigContext, Configuration conf, String filename,
             String prefix) throws IOException {
@@ -1540,7 +1545,7 @@ public class JobControlCompiler{
 
         // XXX Hadoop currently doesn't support distributed cache in local mode.
         // This line will be removed after the support is added by Hadoop team.
-        if (!isLocal(pigContext, conf)) {
+        if (!Utils.isLocal(pigContext, conf)) {
             symlink = prefix + "_"
                     + Integer.toString(System.identityHashCode(filename)) + "_"
                     + Long.toString(System.currentTimeMillis());
@@ -1714,7 +1719,7 @@ public class JobControlCompiler{
 
             // XXX Hadoop currently doesn't support distributed cache in local mode.
             // This line will be removed after the support is added
-            if (isLocal(pigContext, conf)) return;
+            if (Utils.isLocal(pigContext, conf)) return;
 
             // set up distributed cache for the replicated files
             FileSpec[] replFiles = join.getReplFiles();
@@ -1770,7 +1775,7 @@ public class JobControlCompiler{
 
             // XXX Hadoop currently doesn't support distributed cache in local mode.
             // This line will be removed after the support is added
-            if (isLocal(pigContext, conf)) return;
+            if (Utils.isLocal(pigContext, conf)) return;
 
             String indexFile = join.getIndexFile();
 
@@ -1794,7 +1799,7 @@ public class JobControlCompiler{
 
             // XXX Hadoop currently doesn't support distributed cache in local mode.
             // This line will be removed after the support is added
-            if (isLocal(pigContext, conf)) return;
+            if (Utils.isLocal(pigContext, conf)) return;
 
             String indexFile = mergeCoGrp.getIndexFileName();
 
@@ -1831,7 +1836,7 @@ public class JobControlCompiler{
 
             // XXX Hadoop currently doesn't support distributed cache in local mode.
             // This line will be removed after the support is added
-            if (isLocal(pigContext, conf)) return;
+            if (Utils.isLocal(pigContext, conf)) return;
 
             // set up distributed cache for files indicated by the UDF
             String[] files = func.getCacheFiles();

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java Tue Mar 18 02:19:24 2014
@@ -41,7 +41,6 @@ import org.apache.hadoop.mapred.JobID;
 import org.apache.hadoop.mapred.RunningJob;
 import org.apache.hadoop.mapred.TaskReport;
 import org.apache.hadoop.mapred.jobcontrol.Job;
-import org.apache.pig.ExecType;
 import org.apache.pig.PigConfiguration;
 import org.apache.pig.PigException;
 import org.apache.pig.PigRunner.ReturnCode;
@@ -172,6 +171,7 @@ public class MapReduceLauncher extends L
         MRScriptState.get().addWorkflowAdjacenciesToConf(mrp, conf);
 
         // start collecting statistics
+        PigStats.start(pc.getExecutionEngine().instantiatePigStats());
         MRPigStatsUtil.startCollection(pc, statsJobClient, jcc, mrp);
 
         // Find all the intermediate data stores. The plan will be destroyed during compile/execution
@@ -344,6 +344,20 @@ public class MapReduceLauncher extends L
 
                     double prog = (numMRJobsCompl+calculateProgress(jc, statsJobClient))/totalMRJobs;
                     if (notifyProgress(prog, lastProg)) {
+                        List<Job> runnJobs = jc.getRunningJobs();
+                        if (runnJobs != null) {
+                            StringBuilder msg = new StringBuilder();
+                            for (Object object : runnJobs) {
+                                Job j = (Job) object;
+                                if (j != null) {
+                                    msg.append(j.getAssignedJobID()).append(",");
+                                }
+                            }
+                            if (msg.length() > 0) {
+                                msg.setCharAt(msg.length() - 1, ']');
+                                log.info("Running jobs are [" + msg);
+                            }
+                        }
                         lastProg = prog;
                     }
 
@@ -466,7 +480,7 @@ public class MapReduceLauncher extends L
             for (Job job : succJobs) {
                 List<POStore> sts = jcc.getStores(job);
                 for (POStore st : sts) {
-                    if (pc.getExecType() == ExecType.LOCAL) {
+                    if (Utils.isLocal(pc, job.getJobConf())) {
                         HadoopShims.storeSchemaForLocal(job, st);
                     }
 

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigInputFormat.java Tue Mar 18 02:19:24 2014
@@ -49,6 +49,7 @@ import org.apache.pig.impl.io.FileSpec;
 import org.apache.pig.impl.plan.OperatorKey;
 import org.apache.pig.impl.util.ObjectSerializer;
 import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.impl.util.Utils;
 
 public class PigInputFormat extends InputFormat<Text, Tuple> {
 
@@ -227,7 +228,7 @@ public class PigInputFormat extends Inpu
 
                 // if the execution is against Mapred DFS, set
                 // working dir to /user/<userid>
-                if(!pigContext.getExecType().isLocal()) {
+                if(!Utils.isLocal(pigContext, conf)) {
                     fs.setWorkingDirectory(jobcontext.getWorkingDirectory());
                 }
 

Modified: pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java (original)
+++ pig/branches/tez/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java Tue Mar 18 02:19:24 2014
@@ -34,6 +34,7 @@ import org.apache.pig.Algebraic;
 import org.apache.pig.EvalFunc;
 import org.apache.pig.FuncSpec;
 import org.apache.pig.PigException;
+import org.apache.pig.PigWarning;
 import org.apache.pig.TerminatingAccumulator;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
@@ -61,6 +62,7 @@ public class POUserFunc extends Expressi
     private final static String TIMING_COUNTER = "approx_microsecs";
     private final static String INVOCATION_COUNTER = "approx_invocations";
     private final static int TIMING_FREQ = 100;
+    private final static TupleFactory tf = TupleFactory.getInstance();
 
     private transient String counterGroup;
     /**
@@ -186,9 +188,7 @@ public class POUserFunc extends Expressi
         }
 
         Result res = new Result();
-        Tuple inpValue = null;
         if (input == null && (inputs == null || inputs.size()==0)) {
-//			log.warn("No inputs found. Signaling End of Processing.");
             res.returnStatus = POStatus.STATUS_EOP;
             return res;
         }
@@ -269,7 +269,6 @@ public class POUserFunc extends Expressi
 
     private Result getNext() throws ExecException {
         Result result = processInput();
-        String errMsg = "";
         long startNanos = 0;
         boolean timeThis = doTiming && (numInvocations++ % TIMING_FREQ == 0);
         if (timeThis) {
@@ -279,6 +278,34 @@ public class POUserFunc extends Expressi
         }
         try {
             if(result.returnStatus == POStatus.STATUS_OK) {
+                Tuple t = (Tuple) result.result;
+
+                // For backward compatibility, we short-circuit tuples whose
+                // fields are all null. (See PIG-3679)
+                boolean allNulls = true;
+                for (int i = 0; i < t.size(); i++) {
+                    if (!t.isNull(i)) {
+                        allNulls = false;
+                        break;
+                    }
+                }
+                if (allNulls) {
+                    pigLogger.warn(this, "All the input values are null, skipping the invocation of UDF",
+                            PigWarning.SKIP_UDF_CALL_FOR_NULL);
+                    Schema outputSchema = func.outputSchema(func.getInputSchema());
+                    // If the output schema is tuple (i.e. multiple fields are
+                    // to be returned), we return a tuple where every field is
+                    // null.
+                    if (outputSchema != null && outputSchema.getField(0).type == DataType.TUPLE) {
+                        result.result = tf.newTuple(outputSchema.getField(0).schema.size());
+                    // Otherwise, we simply return null since it can be cast to
+                    // any data type.
+                    } else {
+                        result.result = null;
+                    }
+                    return result;
+                }
+
                 if (isAccumulative()) {
                     if (isAccumStarted()) {
                         if (!haveCheckedIfTerminatingAccumulator) {
@@ -296,10 +323,10 @@ public class POUserFunc extends Expressi
                             result.result = null;
                             isAccumulationDone = false;
                         } else {
-                        ((Accumulator)func).accumulate((Tuple)result.result);
-                        result.returnStatus = POStatus.STATUS_BATCH_OK;
-                        result.result = null;
-                        isAccumulationDone = false;
+                            ((Accumulator)func).accumulate((Tuple)result.result);
+                            result.returnStatus = POStatus.STATUS_BATCH_OK;
+                            result.result = null;
+                            isAccumulationDone = false;
                         }
                     }else{
                         if(isAccumulationDone){
@@ -323,7 +350,7 @@ public class POUserFunc extends Expressi
                     if (executor != null) {
                         result.result = executor.monitorExec((Tuple) result.result);
                     } else {
-                    result.result = func.exec((Tuple) result.result);
+                        result.result = func.exec((Tuple) result.result);
                     }
                 }
             }
@@ -355,7 +382,7 @@ public class POUserFunc extends Expressi
         } catch (IndexOutOfBoundsException ie) {
             int errCode = 2078;
             String msg = "Caught error from UDF: " + funcSpec.getClassName() +
-            ", Out of bounds access [" + ie.getMessage() + "]";
+                    ", Out of bounds access [" + ie.getMessage() + "]";
             throw new ExecException(msg, errCode, PigException.BUG, ie);
         }
     }

Modified: pig/branches/tez/src/org/apache/pig/builtin/ToString.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/builtin/ToString.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/builtin/ToString.java (original)
+++ pig/branches/tez/src/org/apache/pig/builtin/ToString.java Tue Mar 18 02:19:24 2014
@@ -62,6 +62,9 @@ public class ToString extends EvalFunc<S
         List<FuncSpec> funcList = new ArrayList<FuncSpec>();
         Schema s = new Schema();
         s.add(new Schema.FieldSchema(null, DataType.DATETIME));
+        funcList.add(new FuncSpec(this.getClass().getName(), s));
+        s = new Schema();
+        s.add(new Schema.FieldSchema(null, DataType.DATETIME));
         s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
         funcList.add(new FuncSpec(this.getClass().getName(), s));
         return funcList;

Modified: pig/branches/tez/src/org/apache/pig/impl/util/Utils.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/impl/util/Utils.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/impl/util/Utils.java (original)
+++ pig/branches/tez/src/org/apache/pig/impl/util/Utils.java Tue Mar 18 02:19:24 2014
@@ -61,6 +61,7 @@ import org.apache.pig.data.DataType;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.impl.PigImplConstants;
 import org.apache.pig.impl.io.InterStorage;
 import org.apache.pig.impl.io.ReadToEndLoader;
 import org.apache.pig.impl.io.SequenceFileInterStorage;
@@ -308,7 +309,7 @@ public class Utils {
                 return false;
             }
         }
-        
+
         public String supportedCodecsToString() {
             StringBuffer sb = new StringBuffer();
             boolean first = true;
@@ -316,7 +317,7 @@ public class Utils {
                 if(first) {
                     first = false;
                 } else {
-                    sb.append(",");    
+                    sb.append(",");
                 }
                 sb.append(codec.name());
             }
@@ -597,4 +598,8 @@ public class Utils {
         return true;
     }
 
+    public static boolean isLocal(PigContext pigContext, Configuration conf) {
+        return pigContext.getExecType().isLocal() || conf.getBoolean(PigImplConstants.CONVERTED_TO_LOCAL, false);
+    }
+
 }

Modified: pig/branches/tez/src/org/apache/pig/newplan/logical/Util.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/newplan/logical/Util.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/newplan/logical/Util.java (original)
+++ pig/branches/tez/src/org/apache/pig/newplan/logical/Util.java Tue Mar 18 02:19:24 2014
@@ -128,6 +128,7 @@ public class Util {
         if (next != null) {
             LogicalRelationalOperator nextOp = (LogicalRelationalOperator)next.get(branch);
             plan.insertBetween(op, foreach, nextOp);
+            foreach.setAlias(op.getAlias());
         }
         else {
             plan.connect(op, foreach);

Modified: pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LOGenerate.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LOGenerate.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LOGenerate.java (original)
+++ pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LOGenerate.java Tue Mar 18 02:19:24 2014
@@ -33,6 +33,8 @@ import org.apache.pig.newplan.logical.re
 public class LOGenerate extends LogicalRelationalOperator {
      private List<LogicalExpressionPlan> outputPlans;
      private boolean[] flattenFlags;
+     // mUserDefinedSchema is the original input from the user, we don't suppose
+     // to store uid in mUserDefinedSchema
      private List<LogicalSchema> mUserDefinedSchema = null;
      private List<LogicalSchema> outputPlanSchemas = null;
      // If LOGenerate generate new uid, cache it here.

Modified: pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LogicalPlanData.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LogicalPlanData.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LogicalPlanData.java (original)
+++ pig/branches/tez/src/org/apache/pig/newplan/logical/relational/LogicalPlanData.java Tue Mar 18 02:19:24 2014
@@ -18,44 +18,95 @@
 
 package org.apache.pig.newplan.logical.relational;
 
-import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 
+import org.apache.pig.classification.InterfaceAudience;
+import org.apache.pig.classification.InterfaceStability;
 import org.apache.pig.newplan.Operator;
 
+import com.google.common.collect.Lists;
+
 /**
  * 
- * This class provides information regarding the LogicalPlan. Make sure to
- * avoid exposing LogicalPlan itself. Only data regarding the logical plan
- * could be exposed but none of Pig internals (plans, operators etc) should
- * be.
- *
+ * This class provides information regarding the LogicalPlan. Make sure to avoid
+ * exposing LogicalPlan itself. Only data regarding the logical plan could be
+ * exposed but none of Pig internals (plans, operators etc) should be.
+ * 
  */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
 public class LogicalPlanData {
 
     // Never expose LogicalPlan
     private final LogicalPlan lp;
+    private int numLogicalRelationalOperators;
+    // Sources and Sinks here refer to Load and Store operators
+    private final List<LOLoad> sources;
+    private final List<LOStore> sinks;
 
     public LogicalPlanData(LogicalPlan lp) {
-        if(lp == null) {
+        if (lp == null) {
             throw new RuntimeException("LogicalPlan is null.");
         }
-        this.lp = lp; 
+        this.lp = lp;
+        this.numLogicalRelationalOperators = 0;
+        this.sources = Lists.newArrayList();
+        this.sinks = Lists.newArrayList();
+        init();
+    }
+
+    private void init() {
+        Iterator<Operator> ops = this.lp.getOperators();
+
+        while (ops.hasNext()) {
+            Operator op = ops.next();
+            if (op instanceof LogicalRelationalOperator) {
+                this.numLogicalRelationalOperators++;
+                if (op instanceof LOLoad) {
+                    sources.add((LOLoad) op);
+                } else if (op instanceof LOStore) {
+                    sinks.add((LOStore) op);
+                }
+            }
+        }
     }
 
     /**
+     * Returns the number of {@link LogicalRelationalOperator}s present in the
+     * pig script.
+     * 
+     * @return number of logical relational operators (Load, Join, Store etc)
      * 
-     * @return This method return the list of source paths defined 
-     *         in the script/query.
      */
-    public List<String> getSources() {
-        List<LOLoad> sources = getLOLoads();
-        if (sources == null) {
-            return null;
-        }
+    public int getNumLogicalRelationOperators() {
+        return this.numLogicalRelationalOperators;
+    }
 
-        List<String> result = new ArrayList<String>();
-        for (LOLoad load : sources) {
+    /**
+     * 
+     * @return number of Load statements in the script
+     */
+    public int getNumSources() {
+        return this.sources.size();
+    }
+
+    /**
+     * 
+     * @return number of Store statements in the script
+     */
+    public int getNumSinks() {
+        return this.sinks.size();
+    }
+
+    /**
+     * 
+     * @return This method return the list of Load paths defined in the
+     *         script/query.
+     */
+    public List<String> getSources() {
+        List<String> result = Lists.newArrayList();
+        for (LOLoad load : this.sources) {
             result.add(load.getFileSpec().getFileName());
         }
 
@@ -67,12 +118,8 @@ public class LogicalPlanData {
      * @return This method returns the list of store paths in the script/query.
      */
     public List<String> getSinks() {
-        List<LOStore> sinks = getLOStores();
-        if (sinks == null) {
-            return null;
-        }
-        List<String> result = new ArrayList<String>();
-        for (LOStore sink : sinks) {
+        List<String> result = Lists.newArrayList();
+        for (LOStore sink : this.sinks) {
             result.add(sink.getFileSpec().getFileName());
         }
 
@@ -84,12 +131,8 @@ public class LogicalPlanData {
      * @return This method returns the list of LoadFunc(s) used.
      */
     public List<String> getLoadFuncs() {
-        List<LOLoad> sources = getLOLoads();
-        if (sources == null) {
-            return null;
-        }
-        List<String> result = new ArrayList<String>();
-        for (LOLoad load : sources) {
+        List<String> result = Lists.newArrayList();
+        for (LOLoad load : this.sources) {
             result.add(load.getFileSpec().getFuncName());
         }
 
@@ -101,56 +144,11 @@ public class LogicalPlanData {
      * @return This method returns the list of StoreFunc(s) used.
      */
     public List<String> getStoreFuncs() {
-        List<LOStore> sinks = getLOStores();
-        if (sinks == null) {
-            return null;
-        }
-        List<String> storeFuncs = new ArrayList<String>();
-        for (LOStore sink : sinks) {
+        List<String> storeFuncs = Lists.newArrayList();
+        for (LOStore sink : this.sinks) {
             storeFuncs.add(sink.getFileSpec().getFuncName());
         }
 
         return storeFuncs;
     }
-
-    /**
-     * Internal to Pig. Do not expose this method
-     * @return
-     */
-    private List<LOLoad> getLOLoads() {
-        List<Operator> sources = lp.getSources();
-        if (sources == null) {
-            return null;
-        }
-        List<LOLoad> result = new ArrayList<LOLoad>();
-        for (Operator source : sources) {
-            if (source instanceof LOLoad) {
-                LOLoad load = (LOLoad) source;
-                result.add(load);
-            }
-        }
-
-        return result;
-    }
-    
-    /**
-     * Internal to Pig. Do not expose this method
-     * @return
-     */
-    private List<LOStore> getLOStores() {
-        List<Operator> sinks = lp.getSinks();
-        if (sinks == null) {
-            return null;
-        }
-        List<LOStore> result = new ArrayList<LOStore>();
-        for (Operator sink : sinks) {
-            if (sink instanceof LOStore) {
-                LOStore store = (LOStore) sink;
-                result.add(store);
-            }
-        }
-
-        return result;
-    }
-
 }

Modified: pig/branches/tez/src/org/apache/pig/newplan/logical/rules/InputOutputFileValidator.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/newplan/logical/rules/InputOutputFileValidator.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/newplan/logical/rules/InputOutputFileValidator.java (original)
+++ pig/branches/tez/src/org/apache/pig/newplan/logical/rules/InputOutputFileValidator.java Tue Mar 18 02:19:24 2014
@@ -34,7 +34,7 @@ import org.apache.pig.newplan.OperatorPl
 import org.apache.pig.newplan.logical.relational.LOStore;
 import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
 
-public class InputOutputFileValidator {
+public class InputOutputFileValidator implements LogicalRelationalNodeValidator {
     private PigContext pigCtx;
     OperatorPlan plan;
     public InputOutputFileValidator(OperatorPlan plan, PigContext pigContext) {

Modified: pig/branches/tez/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java (original)
+++ pig/branches/tez/src/org/apache/pig/newplan/logical/rules/PushDownForEachFlatten.java Tue Mar 18 02:19:24 2014
@@ -260,6 +260,7 @@ public class PushDownForEachFlatten exte
                                 fieldsToBeFlattaned.add(fieldCount);
                                 if (gen.getUserDefinedSchema()!=null && gen.getUserDefinedSchema().get(i)!=null) {
                                     cachedUserDefinedSchema.put(fieldCount, gen.getUserDefinedSchema().get(i));
+                                    cachedUserDefinedSchema.get(fieldCount).mergeUid(gen.getOutputPlanSchemas().get(i));
                                     gen.getUserDefinedSchema().set(i, null);
                                 }
                                 fieldCount++;

Modified: pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java (original)
+++ pig/branches/tez/src/org/apache/pig/parser/LogicalPlanBuilder.java Tue Mar 18 02:19:24 2014
@@ -981,7 +981,7 @@ public class LogicalPlanBuilder {
             ConstantExpression rhs = new ConstantExpression(exprPlan, new Boolean(false));
             BinCondExpression binCond = new BinCondExpression(exprPlan, expr, lhs, rhs);
             args.add(binCond);
-            ConstantExpression constExpr = new ConstantExpression(exprPlan, comment);
+            ConstantExpression constExpr = new ConstantExpression(exprPlan, (comment == null ? "" : comment));
             args.add(constExpr);
             UserFuncExpression udf = new UserFuncExpression(exprPlan, new FuncSpec( Assert.class.getName() ), args );
             exprPlan.add(udf);

Modified: pig/branches/tez/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/parser/QueryParserDriver.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/branches/tez/src/org/apache/pig/parser/QueryParserDriver.java Tue Mar 18 02:19:24 2014
@@ -491,7 +491,7 @@ public class QueryParserDriver {
 
         // sometimes the script has no filename, like when a string is passed to PigServer for
         // example. See PIG-2866.
-        if (fname != null) {
+        if (!fname.isEmpty()) {
             FetchFileRet localFileRet = getMacroFile(fname);
             fname = localFileRet.file.getAbsolutePath();
         }
@@ -594,7 +594,7 @@ public class QueryParserDriver {
             ScriptState ss = ScriptState.get();
             if (ss != null) file = ss.getFileName();
         }
-        if (file != null && !file.equals(importFile)) {
+        if (!file.isEmpty() && !file.equals(importFile)) {
             sb.append("at ").append(file).append(", ");
         }
         sb.append("line ").append(t.getLine()).append("> ").append(header);

Modified: pig/branches/tez/src/org/apache/pig/parser/SourceLocation.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/parser/SourceLocation.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/parser/SourceLocation.java (original)
+++ pig/branches/tez/src/org/apache/pig/parser/SourceLocation.java Tue Mar 18 02:19:24 2014
@@ -83,7 +83,7 @@ public class SourceLocation {
             sb.append("\n");
         }
         sb.append( "<" );
-        if( file != null )
+        if( file != null && !file.isEmpty() )
             sb.append( "file " + file + ", " );
         sb.append( "line " + line +", column " + offset + "> " );
         

Modified: pig/branches/tez/src/org/apache/pig/scripting/BoundScript.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/scripting/BoundScript.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/scripting/BoundScript.java (original)
+++ pig/branches/tez/src/org/apache/pig/scripting/BoundScript.java Tue Mar 18 02:19:24 2014
@@ -272,9 +272,8 @@ public class BoundScript {
         }
         PigServer pigServer = new PigServer(scriptContext.getPigContext(), false);
         pigServer.setBatchOn();
-        GruntParser grunt = new GruntParser(new StringReader(query));
+        GruntParser grunt = new GruntParser(new StringReader(query), pigServer);
         grunt.setInteractive(false);
-        grunt.setParams(pigServer);
         try {
             grunt.parseStopOnError(true);
         } catch (ParseException e) {
@@ -283,13 +282,12 @@ public class BoundScript {
         pigServer.executeBatch();
         return PigStats.get();
     }
-        
-    private void registerQuery(PigServer pigServer, String pl) throws IOException {                
-        GruntParser grunt = new GruntParser(new StringReader(pl));
+
+    private void registerQuery(PigServer pigServer, String pl) throws IOException {
+        GruntParser grunt = new GruntParser(new StringReader(pl), pigServer);
         grunt.setInteractive(false);
-        grunt.setParams(pigServer);
         pigServer.setBatchOn();
-        try {
+      try {
             grunt.parseStopOnError(true);
         } catch (ParseException e) {
             throw new IOException("Failed to parse query: " + pl, e);
@@ -338,9 +336,8 @@ public class BoundScript {
             ScriptState.get().registerListener(adaptor);
             PigServer pigServer = new PigServer(ctx, true);
             pigServer.setBatchOn();
-            GruntParser grunt = new GruntParser(new StringReader(query));
+            GruntParser grunt = new GruntParser(new StringReader(query), pigServer);
             grunt.setInteractive(false);
-            grunt.setParams(pigServer);
             try {
                 grunt.parseStopOnError(true);
             } catch (ParseException e) {
@@ -350,5 +347,5 @@ public class BoundScript {
             return PigStats.get();
         }
     }
-          
+
 }

Modified: pig/branches/tez/src/org/apache/pig/tools/ToolsPigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/tools/ToolsPigServer.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/tools/ToolsPigServer.java (original)
+++ pig/branches/tez/src/org/apache/pig/tools/ToolsPigServer.java Tue Mar 18 02:19:24 2014
@@ -104,7 +104,7 @@ public class ToolsPigServer extends PigS
             substituted = pigContext.doParamSubstitution(fis, paramMapToList(params), paramFiles);
         }catch (FileNotFoundException e){
             log.error(e.getLocalizedMessage());
-            throw new IOException(e.getCause());
+            throw new IOException(e);
         } finally {
             if (fis != null) {
                 fis.close();
@@ -113,15 +113,14 @@ public class ToolsPigServer extends PigS
 
         // Parse in grunt so that register commands are recognized
         try {
-            GruntParser grunt = new GruntParser(new StringReader(substituted));
+            GruntParser grunt = new GruntParser(new StringReader(substituted), this);
             grunt.setInteractive(false);
-            grunt.setParams(this);
             setBatchOn();
             //grunt.setLoadOnly(true);
             grunt.parseOnly();
         } catch (org.apache.pig.tools.pigscript.parser.ParseException e) {
             log.error(e.getLocalizedMessage());
-            throw new IOException(e.getCause());
+            throw new IOException(e);
         }
 
         Graph g = getClonedGraph();

Modified: pig/branches/tez/src/org/apache/pig/tools/grunt/Grunt.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/tools/grunt/Grunt.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/tools/grunt/Grunt.java (original)
+++ pig/branches/tez/src/org/apache/pig/tools/grunt/Grunt.java Tue Mar 18 02:19:24 2014
@@ -41,15 +41,12 @@ public class Grunt 
     PigServer pig;
     GruntParser parser;    
 
-    public Grunt(BufferedReader in, PigContext pigContext) throws ExecException
-    {
+    public Grunt(BufferedReader in, PigContext pigContext) throws ExecException {
         this.in = in;
         this.pig = new PigServer(pigContext);
-        
-        if (in != null)
-        {
-            parser = new GruntParser(in);
-            parser.setParams(pig);    
+
+        if (in != null) {
+            parser = new GruntParser(in, pig);
         }
     }
 

Modified: pig/branches/tez/src/org/apache/pig/tools/grunt/GruntParser.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/tools/grunt/GruntParser.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ pig/branches/tez/src/org/apache/pig/tools/grunt/GruntParser.java Tue Mar 18 02:19:24 2014
@@ -72,6 +72,8 @@ import org.apache.pig.tools.pigscript.pa
 import org.apache.pig.tools.pigstats.JobStats;
 import org.apache.pig.tools.pigstats.PigStats;
 import org.apache.pig.tools.pigstats.PigStats.JobGraph;
+import org.apache.pig.validator.BlackAndWhitelistFilter;
+import org.apache.pig.validator.PigCommandFilter;
 import org.fusesource.jansi.Ansi;
 import org.fusesource.jansi.AnsiConsole;
 import org.python.google.common.collect.Lists;
@@ -79,14 +81,26 @@ import org.python.google.common.collect.
 public class GruntParser extends PigScriptParser {
 
     private static final Log log = LogFactory.getLog(GruntParser.class);
+    private PigCommandFilter filter;
 
-    public GruntParser(Reader stream) {
-        super(stream);
+    public GruntParser(Reader reader) {
+        this(reader, null);
+        init();
+    }
+
+    public GruntParser(Reader reader, PigServer pigServer) {
+        super(reader);
+        mPigServer = pigServer;
         init();
     }
 
     public GruntParser(InputStream stream, String encoding) {
+        this(stream, encoding, null);
+    }
+
+    public GruntParser(InputStream stream, String encoding, PigServer pigServer) {
         super(stream, encoding);
+        mPigServer = pigServer;
         init();
     }
 
@@ -95,8 +109,19 @@ public class GruntParser extends PigScri
         init();
     }
 
+    public GruntParser(InputStream stream, PigServer pigServer) {
+        super(stream);
+        mPigServer = pigServer;
+        init();
+    }
+
     public GruntParser(PigScriptParserTokenManager tm) {
+        this(tm, null);
+    }
+
+    public GruntParser(PigScriptParserTokenManager tm, PigServer pigServer) {
         super(tm);
+        mPigServer = pigServer;
         init();
     }
 
@@ -105,12 +130,23 @@ public class GruntParser extends PigScri
         mLoadOnly = false;
         mExplain = null;
         mScriptIllustrate = false;
+
+        setProps();
+
+        filter = new BlackAndWhitelistFilter(mPigServer);
+    }
+
+    private void setProps() {
+        mDfs = mPigServer.getPigContext().getDfs();
+        mLfs = mPigServer.getPigContext().getLfs();
+        mConf = mPigServer.getPigContext().getProperties();
+        shell = new FsShell(ConfigurationUtil.toConfiguration(mConf));
     }
 
     @Override
-    public void setInteractive(boolean isInteractive){
+    public void setInteractive(boolean isInteractive) {
         super.setInteractive(isInteractive);
-        if(isInteractive){
+        if(isInteractive) {
             setValidateEachStatement(true);
         }
     }
@@ -216,16 +252,6 @@ public class GruntParser extends PigScri
         mLoadOnly = loadOnly;
     }
 
-    public void setParams(PigServer pigServer)
-    {
-        mPigServer = pigServer;
-
-        mDfs = mPigServer.getPigContext().getDfs();
-        mLfs = mPigServer.getPigContext().getLfs();
-        mConf = mPigServer.getPigContext().getProperties();
-        shell = new FsShell(ConfigurationUtil.toConfiguration(mConf));
-    }
-
     public void setScriptIllustrate() {
         mScriptIllustrate = true;
     }
@@ -442,11 +468,13 @@ public class GruntParser extends PigScri
 
     @Override
     protected void processRegister(String jar) throws IOException {
+        filter.validate(PigCommandFilter.Command.REGISTER);
         mPigServer.registerJar(jar);
     }
 
     @Override
     protected void processRegister(String path, String scriptingLang, String namespace) throws IOException, ParseException {
+        filter.validate(PigCommandFilter.Command.REGISTER);
         if(path.endsWith(".jar")) {
             if(scriptingLang != null || namespace != null) {
                 throw new ParseException("Cannot register a jar with a scripting language or namespace");
@@ -530,8 +558,7 @@ public class GruntParser extends PigScri
             throw new ParseException("Cannot access file: " + script);
         }
 
-        GruntParser parser = new GruntParser(inputReader);
-        parser.setParams(mPigServer);
+        GruntParser parser = new GruntParser(inputReader, mPigServer);
         parser.setConsoleReader(reader);
         parser.setInteractive(interactive);
         parser.setLoadOnly(loadOnly);
@@ -551,6 +578,7 @@ public class GruntParser extends PigScri
 
     @Override
     protected void processSet(String key, String value) throws IOException, ParseException {
+        filter.validate(PigCommandFilter.Command.SET);
         if (key.equals("debug"))
         {
             if (value.equals("on"))
@@ -590,21 +618,21 @@ public class GruntParser extends PigScri
            mPigServer.getPigContext().getExecutionEngine().setProperty(key, value);
         }
     }
-    
+
     @Override
     protected void processSet() throws IOException, ParseException {
+        filter.validate(PigCommandFilter.Command.SET);
         Properties jobProps = mPigServer.getPigContext().getProperties();
         Properties sysProps = System.getProperties();
-        
+
         List<String> jobPropsList = Lists.newArrayList();
         List<String> sysPropsList = Lists.newArrayList();
 
         for (Object key : jobProps.keySet()) {
-            String propStr = key + "=" + jobProps.getProperty((String)key);
+            String propStr = key + "=" + jobProps.getProperty((String) key);
             if (sysProps.containsKey(key)) {
                 sysPropsList.add("system: " + propStr);
-            }
-            else {
+            } else {
                 jobPropsList.add(propStr);
             }
         }
@@ -617,8 +645,8 @@ public class GruntParser extends PigScri
     }
 
     @Override
-    protected void processCat(String path) throws IOException
-    {
+    protected void processCat(String path) throws IOException {
+        filter.validate(PigCommandFilter.Command.CAT);
         if(mExplain == null) { // process only if not in "explain" mode
 
             executeBatch();
@@ -666,8 +694,8 @@ public class GruntParser extends PigScri
     }
 
     @Override
-    protected void processCD(String path) throws IOException
-    {
+    protected void processCD(String path) throws IOException {
+        filter.validate(PigCommandFilter.Command.CD);
         ContainerDescriptor container;
         if(mExplain == null) { // process only if not in "explain" mode
 
@@ -704,8 +732,8 @@ public class GruntParser extends PigScri
     }
 
     @Override
-    protected void processDump(String alias) throws IOException
-    {
+    protected void processDump(String alias) throws IOException {
+        filter.validate(PigCommandFilter.Command.DUMP);
         if (alias == null) {
             if (mPigServer.isBatchOn()) {
                 mPigServer.parseAndBuild();
@@ -740,6 +768,7 @@ public class GruntParser extends PigScri
     @Override
     protected void processIllustrate(String alias, String script, String target, List<String> params, List<String> files) throws IOException, ParseException
     {
+        filter.validate(PigCommandFilter.Command.ILLUSTRATE);
         if (mScriptIllustrate)
             throw new ParseException("'illustrate' statement can not appear in a script that is illustrated opon.");
 
@@ -795,13 +824,15 @@ public class GruntParser extends PigScri
     @Override
     protected void processKill(String jobid) throws IOException
     {
+        filter.validate(PigCommandFilter.Command.KILL);
         mPigServer.getPigContext().getExecutionEngine().killJob(jobid);
     }
 
     @Override
-    protected void processLS(String path) throws IOException
-    {
-        if(mExplain == null) { // process only if not in "explain" mode
+    protected void processLS(String path) throws IOException {
+        filter.validate(PigCommandFilter.Command.LS);
+
+        if (mExplain == null) { // process only if not in "explain" mode
 
             executeBatch();
 
@@ -858,6 +889,7 @@ public class GruntParser extends PigScri
     @Override
     protected void processPWD() throws IOException
     {
+        filter.validate(PigCommandFilter.Command.PWD);
         if(mExplain == null) { // process only if not in "explain" mode
 
             executeBatch();
@@ -923,6 +955,7 @@ public class GruntParser extends PigScri
     @Override
     protected void processMove(String src, String dst) throws IOException
     {
+        filter.validate(PigCommandFilter.Command.MV);
         if(mExplain == null) { // process only if not in "explain" mode
 
             executeBatch();
@@ -948,6 +981,7 @@ public class GruntParser extends PigScri
     @Override
     protected void processCopy(String src, String dst) throws IOException
     {
+        filter.validate(PigCommandFilter.Command.CP);
         if(mExplain == null) { // process only if not in "explain" mode
 
             executeBatch();
@@ -969,6 +1003,7 @@ public class GruntParser extends PigScri
     @Override
     protected void processCopyToLocal(String src, String dst) throws IOException
     {
+        filter.validate(PigCommandFilter.Command.COPYTOLOCAL);
         if(mExplain == null) { // process only if not in "explain" mode
 
             executeBatch();
@@ -990,6 +1025,7 @@ public class GruntParser extends PigScri
     @Override
     protected void processCopyFromLocal(String src, String dst) throws IOException
     {
+        filter.validate(PigCommandFilter.Command.COPYFROMLOCAL);
         if(mExplain == null) { // process only if not in "explain" mode
 
             executeBatch();
@@ -1011,6 +1047,7 @@ public class GruntParser extends PigScri
     @Override
     protected void processMkdir(String dir) throws IOException
     {
+        filter.validate(PigCommandFilter.Command.MKDIR);
         if(mExplain == null) { // process only if not in "explain" mode
 
             executeBatch();
@@ -1039,11 +1076,12 @@ public class GruntParser extends PigScri
     }
 
     @Override
-    protected void processRemove(String path, String options ) throws IOException
-    {
+    protected void processRemove(String path, String options) throws IOException {
+        filter.validate(PigCommandFilter.Command.RM);
+        filter.validate(PigCommandFilter.Command.RMF);
         int MAX_MS_TO_WAIT_FOR_FILE_DELETION = 10 * 60 * 1000;
         int MS_TO_SLEEP_WHILE_WAITING_FOR_FILE_DELETION = 250;
-        
+
         if(mExplain == null) { // process only if not in "explain" mode
             Path filePath = new Path(path);
             ElementDescriptor dfsPath = null;
@@ -1083,7 +1121,8 @@ public class GruntParser extends PigScri
     }
 
     @Override
-    protected void processFsCommand(String[] cmdTokens) throws IOException{
+    protected void processFsCommand(String[] cmdTokens) throws IOException {
+        filter.validate(PigCommandFilter.Command.FS);
         if(mExplain == null) { // process only if not in "explain" mode
 
             executeBatch();
@@ -1108,7 +1147,8 @@ public class GruntParser extends PigScri
     }
 
     @Override
-    protected void processShCommand(String[] cmdTokens) throws IOException{
+    protected void processShCommand(String[] cmdTokens) throws IOException {
+        filter.validate(PigCommandFilter.Command.SH);
         if(mExplain == null) { // process only if not in "explain" mode
             try {
                 executeBatch();

Modified: pig/branches/tez/src/org/apache/pig/tools/pigstats/PigStatsUtil.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/tools/pigstats/PigStatsUtil.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/tools/pigstats/PigStatsUtil.java (original)
+++ pig/branches/tez/src/org/apache/pig/tools/pigstats/PigStatsUtil.java Tue Mar 18 02:19:24 2014
@@ -107,30 +107,35 @@ public class PigStatsUtil {
      */
     public static PigStats getPigStats(int code) {
         PigStats ps = PigStats.get();
+        if (ps == null) {
+            ps = PigStats.start(new EmptyPigStats());
+        }
         ps.setReturnCode(code);
         return ps;
     }
 
-
     public static void setErrorMessage(String msg) {
         PigStats ps = PigStats.get();
-        if (ps != null) {
-            ps.setErrorMessage(msg);
+        if (ps == null) {
+            ps = PigStats.start(new EmptyPigStats());
         }
+        ps.setErrorMessage(msg);
     }
 
     public static void setErrorCode(int code) {
         PigStats ps = PigStats.get();
-        if (ps != null) {
-            ps.setErrorCode(code);
+        if (ps == null) {
+            ps = PigStats.start(new EmptyPigStats());
         }
+        ps.setErrorCode(code);
     }
 
     public static void setErrorThrowable(Throwable t) {
         PigStats ps = PigStats.get();
-        if (ps != null) {
-            ps.setErrorThrowable(t);
+        if (ps == null) {
+            ps = PigStats.start(new EmptyPigStats());
         }
+        ps.setErrorThrowable(t);
     }
 
     private static Pattern pattern = Pattern.compile("tmp(-)?[\\d]{1,10}$");

Modified: pig/branches/tez/src/org/apache/pig/tools/pigstats/mapreduce/MRScriptState.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/src/org/apache/pig/tools/pigstats/mapreduce/MRScriptState.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/src/org/apache/pig/tools/pigstats/mapreduce/MRScriptState.java (original)
+++ pig/branches/tez/src/org/apache/pig/tools/pigstats/mapreduce/MRScriptState.java Tue Mar 18 02:19:24 2014
@@ -211,7 +211,7 @@ public class MRScriptState extends Scrip
         setJobParents(mro, conf);
 
         conf.set("mapreduce.workflow.id", "pig_" + id);
-        conf.set("mapreduce.workflow.name", (getFileName() != null) ? getFileName() : "default");
+        conf.set("mapreduce.workflow.name", getFileName().isEmpty() ? "default" : getFileName());
         conf.set("mapreduce.workflow.node.name", mro.getOperatorKey().toString());
     }
 

Propchange: pig/branches/tez/src/pig-default.properties
------------------------------------------------------------------------------
  Merged /pig/trunk/src/pig-default.properties:r1571422-1578679

Modified: pig/branches/tez/test/e2e/pig/conf/local.conf
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/e2e/pig/conf/local.conf?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/e2e/pig/conf/local.conf (original)
+++ pig/branches/tez/test/e2e/pig/conf/local.conf Tue Mar 18 02:19:24 2014
@@ -48,6 +48,7 @@ $cfg = {
     , 'testconfigpath'   => "$ENV{PH_CLUSTER}/conf/"
     , 'funcjarPath'      => "$ENV{PH_ROOT}/lib/java"
     , 'paramPath'        => "$ENV{PH_ROOT}/paramfiles"
+    , 'piggybankjarPath' => "$ENV{PH_PIG}/contrib/piggybank/java"
     , 'pigpath'          => "$ENV{PH_PIG}"
 	, 'oldpigpath'       => "$ENV{PH_OLDPIG}"
     , 'exectype'         => 'local'

Modified: pig/branches/tez/test/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/TestInputSizeReducerEstimator.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/TestInputSizeReducerEstimator.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/TestInputSizeReducerEstimator.java (original)
+++ pig/branches/tez/test/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/TestInputSizeReducerEstimator.java Tue Mar 18 02:19:24 2014
@@ -17,16 +17,20 @@
  */
 package org.apache.pig.backend.hadoop.executionengine.mapReduceLayer;
 
-import com.google.common.collect.Lists;
+import java.util.Collections;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.pig.LoadFunc;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
 import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.impl.io.FileSpec;
 import org.apache.pig.test.PigStorageWithStatistics;
 import org.apache.pig.test.TestJobControlCompiler;
 import org.junit.Assert;
 import org.junit.Test;
 
+import com.google.common.collect.Lists;
+
 public class TestInputSizeReducerEstimator {
 
     private static final Configuration CONF = new Configuration(false);
@@ -48,6 +52,15 @@ public class TestInputSizeReducerEstimat
                 Lists.newArrayList(
                         createPOLoadWithSize(size, new PigStorage()),
                         createPOLoadWithSize(size, new PigStorageWithStatistics())),
+                        new org.apache.hadoop.mapreduce.Job(CONF)));
+
+        // Negative test - PIG-3754
+        POLoad poLoad = createPOLoadWithSize(size, new PigStorage());
+        poLoad.setLFile(new FileSpec("hbase://users", null));
+
+        Assert.assertEquals(-1, InputSizeReducerEstimator.getTotalInputFileSize(
+                CONF,
+                Collections.singletonList(poLoad),
                 new org.apache.hadoop.mapreduce.Job(CONF)));
     }
 

Modified: pig/branches/tez/test/org/apache/pig/pigunit/pig/GruntParser.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/pigunit/pig/GruntParser.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/pigunit/pig/GruntParser.java (original)
+++ pig/branches/tez/test/org/apache/pig/pigunit/pig/GruntParser.java Tue Mar 18 02:19:24 2014
@@ -36,79 +36,80 @@ import java.util.regex.Pattern;
  * e.g. pig.registerQuery replace the query of a certain alias...
  */
 public class GruntParser extends org.apache.pig.tools.grunt.GruntParser {
-  /** A mapping <alias,query> to apply to the pig script. */
-  private final Map<String, String> aliasOverride;
+    /** A mapping <alias,query> to apply to the pig script. */
+    private final Map<String, String> aliasOverride;
 
-  /**
-   * Initializes the Pig parser with its list of aliases to override.
-   *
-   * @param stream The Pig script stream.
-   * @param aliasOverride The list of aliases to override in the Pig script.
-   */
-  public GruntParser(Reader stream, Map<String, String> aliasOverride) {
-    super(stream);
-    this.aliasOverride = aliasOverride;
-  }
-
-  /**
-   * Pig relations that have been blanked are dropped.
-   */
-  @Override
-  protected void processPig(String cmd) throws IOException {
-    String command = override(cmd);
+    /**
+     * Initializes the Pig parser with its list of aliases to override.
+     * 
+     * @param stream The Pig script stream.
+     * @param aliasOverride The list of aliases to override in the Pig script.
+     */
+    public GruntParser(Reader stream, Map<String, String> aliasOverride) {
+        this(stream, null, aliasOverride);
+    }
 
-    if (! command.equals("")) {
-      super.processPig(command);
+    public GruntParser(Reader stream, PigServer pigServer, Map<String, String> aliasOverride) {
+        super(stream, pigServer);
+        this.aliasOverride = aliasOverride;
     }
-  }
 
-  /**
-   * Overrides the relations of the pig script that we want to change.
-   *
-   * @param query The current pig query processed by the parser.
-   * @return The same query, or a modified query, or blank.
-   */
-  public String override(String query) {
-    // a path to be prepended to all the file names in the script
-    String fsRoot = System.getProperty("pigunit.filesystem.prefix");
-    if(fsRoot != null) {
-      query = Pattern.compile("(LOAD\\s+'(([^:/?#]+)://)?)", Pattern.CASE_INSENSITIVE).
-        matcher(query).
-        replaceFirst("$1" + fsRoot);
-      query = Pattern.compile("(STORE\\s+([^']+)\\s+INTO\\s+'(([^:/?#]+)://)?)", Pattern.CASE_INSENSITIVE).
-        matcher(query).
-        replaceFirst("$1" + fsRoot);
+    /**
+     * Pig relations that have been blanked are dropped.
+     */
+    @Override
+    protected void processPig(String cmd) throws IOException {
+        String command = override(cmd);
+
+        if (!command.equals("")) {
+            super.processPig(command);
+        }
     }
 
-    Map<String, String> metaData = new HashMap<String, String>();
+    /**
+     * Overrides the relations of the pig script that we want to change.
+     * 
+     * @param query
+     *            The current pig query processed by the parser.
+     * @return The same query, or a modified query, or blank.
+     */
+    public String override(String query) {
+        // a path to be prepended to all the file names in the script
+        String fsRoot = System.getProperty("pigunit.filesystem.prefix");
+        if (fsRoot != null) {
+            query = Pattern.compile("(LOAD\\s+'(([^:/?#]+)://)?)", Pattern.CASE_INSENSITIVE).matcher(query).replaceFirst("$1" + fsRoot);
+            query = Pattern.compile("(STORE\\s+([^']+)\\s+INTO\\s+'(([^:/?#]+)://)?)", Pattern.CASE_INSENSITIVE).matcher(query).replaceFirst("$1" + fsRoot);
+        }
+
+        Map<String, String> metaData = new HashMap<String, String>();
+
+        for (Entry<String, String> alias : aliasOverride.entrySet()) {
+            saveLastStoreAlias(query, metaData);
+
+            if (query.toLowerCase().startsWith(alias.getKey().toLowerCase() + " ")) {
+                System.out.println(String.format("%s\n--> %s", query, alias.getValue() == "" ? "none" : alias.getValue()));
+                query = alias.getValue();
+            }
+        }
 
-    for (Entry<String, String> alias : aliasOverride.entrySet()) {
-      saveLastStoreAlias(query, metaData);
+        aliasOverride.putAll(metaData);
 
-      if (query.toLowerCase().startsWith(alias.getKey().toLowerCase() + " ")) {
-        System.out.println(
-            String.format("%s\n--> %s", query, alias.getValue() == "" ? "none" : alias.getValue()));
-        query = alias.getValue();
-      }
+        return query;
     }
 
-    aliasOverride.putAll(metaData);
-
-    return query;
-  }
-
-  /**
-   * Saves the name of the alias of the last store.
-   *
-   * <p>Maybe better to replace it by PigServer.getPigContext().getLastAlias().
-   */
-  void saveLastStoreAlias(String cmd, Map<String, String> metaData) {
-    if (cmd.toUpperCase().startsWith("STORE")) {
-      Pattern outputFile = Pattern.compile("STORE +([^']+) INTO.*", Pattern.CASE_INSENSITIVE);
-      Matcher matcher = outputFile.matcher(cmd);
-      if (matcher.matches()) {
-        metaData.put("LAST_STORE_ALIAS", matcher.group(1));
-      }
+    /**
+     * Saves the name of the alias of the last store.
+     * 
+     * <p>
+     * Maybe better to replace it by PigServer.getPigContext().getLastAlias().
+     */
+    void saveLastStoreAlias(String cmd, Map<String, String> metaData) {
+        if (cmd.toUpperCase().startsWith("STORE")) {
+            Pattern outputFile = Pattern.compile("STORE +([^']+) INTO.*", Pattern.CASE_INSENSITIVE);
+            Matcher matcher = outputFile.matcher(cmd);
+            if (matcher.matches()) {
+                metaData.put("LAST_STORE_ALIAS", matcher.group(1));
+            }
+        }
     }
-  }
 }

Modified: pig/branches/tez/test/org/apache/pig/pigunit/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/pigunit/pig/PigServer.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/pigunit/pig/PigServer.java (original)
+++ pig/branches/tez/test/org/apache/pig/pigunit/pig/PigServer.java Tue Mar 18 02:19:24 2014
@@ -50,16 +50,13 @@ public class PigServer extends org.apach
       throws IOException {
     try {
       InputStream compositeStream = Utils.getCompositeStream(new FileInputStream(fileName), pigContext.getProperties());
-      GruntParser grunt = new GruntParser(new InputStreamReader(compositeStream), aliasOverride);
+      GruntParser grunt = new GruntParser(new InputStreamReader(compositeStream), this, aliasOverride);
       grunt.setInteractive(false);
-      grunt.setParams(this);
       grunt.parseStopOnError(true);
     } catch (FileNotFoundException e) {
-      e.printStackTrace();
-      throw new IOException(e.getCause());
+      throw new IOException(e);
     } catch (org.apache.pig.tools.pigscript.parser.ParseException e) {
-      e.printStackTrace();
-      throw new IOException(e.getCause());
+      throw new IOException(e);
     }
   }
 }



Mime
View raw message