pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r669525 [2/2] - in /incubator/pig/branches/types: ./ docs/ src/org/apache/pig/ src/org/apache/pig/tools/parameters/ test/org/apache/pig/test/ test/org/apache/pig/test/data/
Date Thu, 19 Jun 2008 16:12:04 GMT
Added: incubator/pig/branches/types/test/org/apache/pig/test/TestParamSubPreproc.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestParamSubPreproc.java?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestParamSubPreproc.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestParamSubPreproc.java Thu Jun 19 09:12:03 2008
@@ -0,0 +1,1287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.FileWriter;
+
+import org.apache.pig.tools.parameters.*;
+
+import junit.framework.TestCase;
+
+public class TestParamSubPreproc extends TestCase {
+
+    private final Log log = LogFactory.getLog(getClass());
+
+    private BufferedReader pigIStream;
+    private FileWriter pigOStream;
+    private FileInputStream pigExResultStream;
+    private String basedir;
+
+
+    public TestParamSubPreproc(String name) {
+        super(name);
+        basedir = "test/org/apache/pig/test/data";
+    }
+
+    /* Test case 1   
+     * Use a parameter within a pig script and provide value on the command line.
+     */
+    @Test
+    public void testCmdlineParam() throws Exception{
+        log.info("Starting test testCmdlineParam() ...");
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date=20080228"};
+            String[] argFiles = null;
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+        log.info("Done");
+
+    }
+
+    /* Test case 2
+     * Use a parameter within a pig script and provide value through a file.
+     */
+    @Test
+    public void testFileParam() throws Exception{
+        log.info ("Starting test  testFileParam()");
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = null;
+            String[] argFiles = {basedir+"/ConfFile1.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Parameter substitution from config file failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Parameter substitution from config file failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+        log.info("Done");
+    }
+
+    /* Test case 3
+     *  Use a parameter within a pig script and provide value through running a binary or script.
+     */
+     @Test
+     public void testShellCommand() throws Exception{
+        log.info("Starting test testShellCommand()");
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/input4.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = null; //{"date=`sh generate_date.sh`"};     //`date \\T`"};
+            String[] argFiles = null; // {basedir+"/ConfFile1.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+        log.info("Done");
+    }
+
+    /* Test case 4   
+     * Use a Pig-supported parameter like "$0" and ensure that the Pig-supported parameter is not resolved
+     */
+    @Test
+    public void testPigParamNotResolved() throws Exception{
+        log.info("Starting test testPigParamNotResolved()");
+        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+        pigIStream = new BufferedReader(new FileReader(basedir + "/input3.pig"));
+        pigOStream = new FileWriter(basedir + "/output1.pig");
+
+        String[] arg = null;
+        String[] argFiles = null;
+        try {
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+        } catch (RuntimeException e) {
+            if (e.getMessage().equals("Undefined parameter : 4")) {
+                fail("Pig supported parameter $4 should not have been resolved.");
+            }
+        }
+        log.info("Done");
+    }
+
+    /* Test case 5   
+     * Use the %declare statement after the command has been used.
+     */
+    @Test
+    public void testUndefinedParam() throws Exception{
+        log.info("Starting test testUndefinedParam()");
+        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+        pigIStream = new BufferedReader(new FileReader(basedir + "/input2.pig"));
+        pigOStream = new FileWriter(basedir + "/output1.pig");
+
+        String[] arg = null;
+        String[] argFiles = null;
+        try {
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+            fail ("Should have thrown an Undefined parameter exception");
+        } catch (RuntimeException e) {
+            assertEquals(e.getMessage(), "Undefined parameter : param");
+        }
+        log.info("Done");
+    }
+
+    /* Test case 7   
+     *  Use a parameter in %declare that is defined in terms of other parameters
+     */
+    @Test
+    public void testSubstitutionWithinValue() throws Exception{
+        log.info("Starting test  testSubstitutionWithinValue()");
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputSubstitutionWithinValue.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = null;
+            String[] argFiles = {basedir+"/ConfFile1.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Parameter substitution within a value failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Parameter substitution within a value failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+        log.info("Done");
+    }
+
+
+    /* Test case 8
+     *  Use a parameter within a pig script, provide value for it through running a binary or script. 
+     *  The script itself takes an argument that is a parameter and should be resolved
+     */
+    @Test 
+    public void testSubstitutionWithinShellCommand() throws Exception{
+        log.info("Starting test testSubstitutionWithinShellCommand()");
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputSubstitutionWithinShellCommand.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = null; 
+            String[] argFiles = null;
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Parameter substitution with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+        log.info("Done");
+    }
+
+
+    /* Test case 9
+     *  Use parameters passed on the command line/file that are not resolved prior to the declare statement.
+     */
+    @Test
+    public void testCmdlineParamPriortoDeclare() throws Exception{
+        log.info("Starting test testCmdlineParamPriortoDeclare()");
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/input2.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"param='20080228'"}; 
+            String[] argFiles = null;
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Parameter substitution of command line arg. prior to declare stmt failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Parameter substitution of command line arg. prior to declare stmt failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+        log.info("Done");
+    } 
+
+
+    /* Test case 10
+     *  Use a command name in a %declare statement as a parameter
+     */
+    @Test
+    public void testCmdnameAsParamDeclare() throws Exception{
+        log.info("Starting test testCmdnameAsParamDeclare()");
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputCmdnameAsParamDeclare.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = null; 
+            String[] argFiles = null;
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Parameter substitution for a command with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Parameter substitution for a command with shell command failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+        log.info("Done");
+    }
+
+
+    /* Test case 11   
+     * Use the same parameter multiple times on the command line.
+     * Result : last value used and warning should be thrown
+     */
+    @Test
+    public void testMultipleCmdlineParam() throws Exception{
+        log.info("Starting test testCmdnameAsParamDeclare()");
+        try {     
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date='092487'","date='20080228'"};
+            String[] argFiles = null;
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+        log.info("Done");
+    }
+
+    /* Test case 12
+     * Read parameters from multiple files.
+     */
+    @Test
+    public void testFileParamsFromMultipleFiles() throws Exception{
+        log.info("Starting test testFileParamsFromMultipleFiles()");
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputMultipleParams.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = null;
+            String[] argFiles = {basedir+"/ConfFile1.txt" , basedir+"/ConfFile2.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+        log.info("Done");
+    }
+    
+    /* Test case 13
+     * Use the same parameter in multiple files.
+     * Result: last value used and warning should be thrown
+     */
+    @Test
+    public void testSameParamInMultipleFiles() throws Exception{
+        log.info("Starting test testSameParamInMultipleFiles()");
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputMultipleParams.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = null;
+            String[] argFiles = {basedir+"/ConfFile3.txt" , basedir+"/ConfFile2.txt", basedir+"/ConfFile1.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Same Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Same Parameter substitution from multiple config files failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+        log.info("Done");
+    }
+
+    /* Test case 14
+     * Use the same parameter multiple times in a single file.
+     * Result: last value used and warning should be thrown.
+     */
+    @Test
+    public void testMultipleParamsFromSingleFile() throws Exception{
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = null;
+            String[] argFiles = {basedir+"/ConfFileSameParamMultipleTimes.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+    }
+    
+    /* Test case 15,16   
+     *   Use an empty lines and Comment lines in the parameter file.
+     *  Result: Allowed
+     */
+    @Test
+    public void testEmptyCommentLineinConfigfile() throws Exception{
+        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+        pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
+        pigOStream = new FileWriter(basedir + "/output1.pig");
+
+        String[] arg = null;
+        String[] argFiles = {basedir+"/ConfFileWithEmptyComments.txt"};
+        try {
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+    }
+    
+    /* Test case 17   
+     *   Use a line in the file that is not empty or a comment but does not conform to param_name=param_value.
+     */
+    @Test
+    public void testInvalidLineinConfigfile() throws Exception{
+        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+        pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
+        pigOStream = new FileWriter(basedir + "/output1.pig");
+
+        String[] arg = null;
+        String[] argFiles = {basedir+"/ConfFileWithInvalidLines.txt"};
+        try {
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+            fail ("Should have thrown an exception");
+        } catch (ParseException e) {
+            assertTrue(e.getMessage().startsWith("Encountered \"is\" at line 2, column 6."));
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+    }
+    
+    
+    /* Test case 18,19
+     *   Check a parameter line of form param_name=param_value is allowed.
+     *   Check a parameter line of form param_name<white space>=<white space>param_value is allowed.
+     */
+    @Test
+    public void testValidLinesinConfigfile() throws Exception{
+        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+        pigIStream = new BufferedReader(new FileReader(basedir + "/inputMultipleParams.pig"));
+        pigOStream = new FileWriter(basedir + "/output1.pig");
+
+        String[] arg = null;
+        String[] argFiles = {basedir+"/ConfFileWithValidLines.txt"};
+        try {
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+    }
+    
+     /* Test case 20
+     * Use a combination of command line and file parameters.
+     */
+    @Test
+    public void testCmdlineFileCombo() throws Exception{
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputMultipleParams.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date='20080228'"};
+            String[] argFiles = {basedir+"/ConfFile2.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+    }
+
+    /* Test case 21
+     * Use a combination of command line and file parameters where there are duplicate parameters.
+     * Result: Command line parameters take precedence over files.
+     */
+    @Test
+    public void testCmdlineFileComboDuplicate() throws Exception{
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputMultipleParams.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date='20080228'"};
+            String[] argFiles = {basedir+"/ConfFileDuplicates.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+    }
+    
+
+    /* Test case 22
+     *  Use a combination of command line, file and declare.
+     */
+    @Test
+    public void testCmdlineFileDeclareCombo() throws Exception{
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputThreeParams.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date='20080228'"};
+            String[] argFiles = {basedir+"/ConfFile2.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+    }
+    
+    /* Test case 23
+     *  Use a combination of command line, file and declare where there are duplicate parameters.
+     *  Result: Declare has highest priority.
+     */
+    @Test
+    public void testCmdlineFileDeclareComboDuplicates() throws Exception{
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputThreeParams.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date='20080228'" , "tableName=\"skip this\""};
+            String[] argFiles = {basedir+"/ConfFile2.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+    }
+    
+    /* Test case 24
+     *   Use multiple declare statement that specify the same parameter.
+     *   Result: Scope of a parameter declared using declare is until the next declare statement that defines the same parameter.
+     */
+    @Test
+    public void testMultipleDeclareScope() throws Exception{
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputMultipleDeclares.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date='20080228'"};
+            String[] argFiles = null;
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+    }
+    
+    
+    /* Test case 25
+     *   Use %default to define param values
+     */
+    @Test
+    public void testDefaultParam() throws Exception{
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputDefault.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = null;
+            String[] argFiles = null;
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+    }
+    
+
+    /* Test case 26
+     *  Use a combination of file, command line, declare and default. Default has the lowest precedence.
+     */
+    @Test
+    public void testCmdlineFileDeclareDefaultComboDuplicates() throws Exception{
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputThreeParams.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date='20080228'" , "tableName=\"skip this\""};
+            String[] argFiles = {basedir+"/ConfFile2.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+    }
+    
+    
+    /* Test case 28
+     *  28. More than 1 parameter is present and needs to be substituted within a line e.g. A = load '/data/$name/$date';
+     */
+    @Test
+    public void testMultipleParamsinSingleLine() throws Exception{
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputThreeParams.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date='20080228'"};
+            String[] argFiles = {basedir+"/ConfFile2.txt"};
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+    }
+
+
+    /* Test case 29   
+     * Parameter is substituted within a literal e.g. store A into 'output/$name';
+     */
+    @Test
+    public void testSubstituteWithinLiteral() throws Exception{
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date='20080228'"};
+            String[] argFiles = null;
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+    }
+
+    
+    /* Test case 30   
+     * Make sure that escaped values are not substituted e.g. A = load '/data/\$name'
+     */
+    @Test
+    public void testEscaping() throws Exception{
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputEscape.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date='20080228'"};
+            String[] argFiles = null;
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult2.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+    }
+        
+    /* Test case 31   
+     * Use of inline command
+     */
+    @Test
+    public void testCmdlineParamWithInlineCmd() throws Exception{
+        log.info("Starting test testCmdlineParamWithInlineCmd() ...");
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = {"date=`perl -e 'print \"20080228\n20070101\"' | head -n 1`"};
+            String[] argFiles = null;
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+        log.info("Done");
+
+    }
+
+    /* Test case 32   
+     * No substitution
+     */
+    @Test
+    public void testNoVars() throws Exception{
+        log.info("Starting test testNoVars() ...");
+        try {
+            ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+            pigIStream = new BufferedReader(new FileReader(basedir + "/inputNoVars.pig"));
+            pigOStream = new FileWriter(basedir + "/output1.pig");
+
+            String[] arg = null;
+            String[] argFiles = null;
+            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+            FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+            pigExResultStream = new FileInputStream(basedir + "/inputNoVars.pig");
+            BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+            BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+            String exLine;
+            String resLine;
+            int lineNum=0;
+
+            while (true) {
+                lineNum++;
+                exLine = inExpected.readLine();
+                resLine = inResult.readLine();
+                if (exLine==null || resLine==null)
+                    break;
+                assertEquals("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+            }
+            if (!(exLine==null && resLine==null)) {
+                fail ("Command line parameter substitution failed. " + "Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+            }
+
+            inExpected.close();
+            inResult.close();
+        } catch (ParseException e) {
+            fail ("Got ParseException : " + e.getMessage());
+        } catch (RuntimeException e) {
+            fail ("Got RuntimeException : " + e.getMessage());
+        } catch (Error e) {
+            fail ("Got error : " + e.getMessage());
+        }
+
+        log.info("Done");
+
+    }
+}

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile1.txt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile1.txt?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile1.txt (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile1.txt Thu Jun 19 09:12:03 2008
@@ -0,0 +1 @@
+date="20080228"

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile2.txt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile2.txt?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile2.txt (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile2.txt Thu Jun 19 09:12:03 2008
@@ -0,0 +1 @@
+destination='\'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct\''

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile3.txt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile3.txt?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile3.txt (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFile3.txt Thu Jun 19 09:12:03 2008
@@ -0,0 +1 @@
+destination='this shud be skipped'
\ No newline at end of file

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileDuplicates.txt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileDuplicates.txt?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileDuplicates.txt (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileDuplicates.txt Thu Jun 19 09:12:03 2008
@@ -0,0 +1,2 @@
+destination='\'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct\''
+date = 'this is to be skipped'

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileSameParamMultipleTimes.txt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileSameParamMultipleTimes.txt?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileSameParamMultipleTimes.txt (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileSameParamMultipleTimes.txt Thu Jun 19 09:12:03 2008
@@ -0,0 +1,2 @@
+date = 'skip this'
+date = 20080228

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithEmptyComments.txt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithEmptyComments.txt?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithEmptyComments.txt (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithEmptyComments.txt Thu Jun 19 09:12:03 2008
@@ -0,0 +1,7 @@
+date = '20080228'
+
+#this is a comment line
+
+key1 = 'value1'
+
+#this is also a comment line

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithInvalidLines.txt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithInvalidLines.txt?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithInvalidLines.txt (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithInvalidLines.txt Thu Jun 19 09:12:03 2008
@@ -0,0 +1,2 @@
+date = '20080228'
+this is an invalid line

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithValidLines.txt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithValidLines.txt?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithValidLines.txt (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/ConfFileWithValidLines.txt Thu Jun 19 09:12:03 2008
@@ -0,0 +1,2 @@
+date = '20080228'
+destination=/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/ExpectedResult.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/ExpectedResult.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/ExpectedResult.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/ExpectedResult.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,11 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
\ No newline at end of file

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/ExpectedResult2.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/ExpectedResult2.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/ExpectedResult2.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/ExpectedResult2.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,11 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '$column1') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
\ No newline at end of file

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/generate_date.sh
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/generate_date.sh?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/generate_date.sh (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/generate_date.sh Thu Jun 19 09:12:03 2008
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+if  [ $# -eq 0 ]
+then
+        echo "20080228";
+elif [ $# -eq 1 ]
+then
+        #echo `date +%Y``date +%m``date +%d`;
+        echo $1;
+fi

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/input1.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/input1.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/input1.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/input1.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,11 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/input2.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/input2.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/input2.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/input2.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,12 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$param' using PigStorage('\x01');
+%declare param '2008.txt'
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/input3.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/input3.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/input3.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/input3.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,11 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/input4.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/input4.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/input4.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/input4.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,12 @@
+
+%declare date `sh test/org/apache/pig/test/data/generate_date.sh`
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/inputCmdlineParamPriortoDeclare.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/inputCmdlineParamPriortoDeclare.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/inputCmdlineParamPriortoDeclare.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/inputCmdlineParamPriortoDeclare.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,12 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$param' using PigStorage('\x01');
+%declare param '2008.txt'
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/inputCmdnameAsParamDeclare.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/inputCmdnameAsParamDeclare.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/inputCmdnameAsParamDeclare.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/inputCmdnameAsParamDeclare.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,14 @@
+
+%declare cmd 'test/org/apache/pig/test/data/generate_date'
+%declare date '20080228'
+%declare param `sh $cmd.sh $date`
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$param' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/inputDefault.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/inputDefault.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/inputDefault.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/inputDefault.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,12 @@
+%default date '20080228'
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/inputEscape.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/inputEscape.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/inputEscape.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/inputEscape.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,12 @@
+%default column1 'asdf'
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '\$column1') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/inputMultipleDeclares.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/inputMultipleDeclares.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/inputMultipleDeclares.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/inputMultipleDeclares.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,13 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+%declare destLocation '\'/user/kaleidoscope/pow_stats/$date/acct/InactiveAcct\''
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into $destLocation;
+%declare destLocation '\'/user/kaleidoscope/pow_stats/$date/acct_stats/InactiveAcctCount\''
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into $destLocation;

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/inputMultipleParams.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/inputMultipleParams.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/inputMultipleParams.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/inputMultipleParams.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,11 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into $destination;
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/inputNoVars.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/inputNoVars.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/inputNoVars.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/inputNoVars.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,6 @@
+
+register /home/y/lib/java/pigtest/testudf.jar;
+A = load '/user/pig/tests/data/singlefile/textdoc' using TextLoader();
+define X `perl -ne 'chomp $_; print "$_\n"'` output (stdout using org.apache.pig.test.udf.storefunc.StringStore());
+B = stream A through X;
+store B into '/user/pig/tests/results/olgan.1209067990/DefineClause_4.out';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubWithinLiteral.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubWithinLiteral.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubWithinLiteral.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubWithinLiteral.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,13 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+%declare destLocation '/user/kaleidoscope/pow_stats/$date/acct/InactiveAcct'
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into $destLocation;
+%declare destLocation '/user/kaleidoscope/pow_stats/$date/acct_stats/InactiveAcctCount'
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into $destLocation;

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,13 @@
+
+%DECLARE date '20080228'
+%DECLARE formatted_date `sh test/org/apache/pig/test/data/generate_date.sh $date`
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$formatted_date' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubstitutionWithinValue.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubstitutionWithinValue.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubstitutionWithinValue.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/inputSubstitutionWithinValue.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,14 @@
+
+%Declare var1 '1'
+%DECLARE var2 '6'
+%declare total '$var1$var2'
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
+bb = filter aa by (ARITY == '$total') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/inputThreeParams.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/inputThreeParams.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/inputThreeParams.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/inputThreeParams.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,13 @@
+%default tableName 'value to be skipped'
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+%declare tableName 'inactiveAccounts'
+--generate inactive accts
+$tableName = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store $tableName into $destination;
+grpInactiveAcct = group $tableName all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( $tableName ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/newinputS.txt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/newinputS.txt?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/newinputS.txt (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/newinputS.txt Thu Jun 19 09:12:03 2008
@@ -0,0 +1,10 @@
+key1=value1
+date=20080202
+mypath=/user/abhit/$date
+date2=20080202.txt
+path2='user/kal/$date/$date2'
+
+
+a 	=	2
+b	=	3
+c = $a + $b
\ No newline at end of file

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/output1.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/output1.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/output1.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/output1.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,6 @@
+
+register /home/y/lib/java/pigtest/testudf.jar;
+A = load '/user/pig/tests/data/singlefile/textdoc' using TextLoader();
+define X `perl -ne 'chomp $_; print "$_\n"'` output (stdout using org.apache.pig.test.udf.storefunc.StringStore());
+B = stream A through X;
+store B into '/user/pig/tests/results/olgan.1209067990/DefineClause_4.out';

Added: incubator/pig/branches/types/test/org/apache/pig/test/data/output3.pig
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/output3.pig?rev=669525&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/output3.pig (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/output3.pig Thu Jun 19 09:12:03 2008
@@ -0,0 +1,11 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';



Mime
View raw message