pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject svn commit: r1784224 [2/17] - in /pig/branches/spark: ./ bin/ conf/ contrib/piggybank/java/ contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/ contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelo...
Date Fri, 24 Feb 2017 03:34:40 GMT
Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/MaxTupleBy1stField.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/MaxTupleBy1stField.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/MaxTupleBy1stField.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/MaxTupleBy1stField.java Fri Feb 24 03:34:37 2017
@@ -21,7 +21,6 @@ package org.apache.pig.piggybank.evaluat
 import java.io.IOException;
 import java.util.Iterator;
 
-import org.apache.pig.Accumulator;
 import org.apache.pig.Algebraic;
 import org.apache.pig.EvalFunc;
 import org.apache.pig.backend.executionengine.ExecException;
@@ -44,7 +43,7 @@ import org.apache.pig.impl.logicalLayer.
  * 
  * @author Vadim Zaliva <lord@codemindes.com>
  */
-public class MaxTupleBy1stField extends EvalFunc<Tuple> implements Algebraic, Accumulator<Tuple>
+public class MaxTupleBy1stField extends EvalFunc<Tuple> implements Algebraic
 {
     /**
      * Indicates once for how many items progress hartbeat should be sent.
@@ -132,11 +131,6 @@ public class MaxTupleBy1stField extends
     protected static Tuple max(Tuple input, PigProgressable reporter) throws ExecException
     {
         DataBag values = (DataBag) input.get(0);
-        return max(values,reporter);
-    }
-
-    protected static Tuple max(DataBag values, PigProgressable reporter) throws ExecException
-    {
 
         // if we were handed an empty bag, return NULL
         // this is in compliance with SQL standard
@@ -189,44 +183,4 @@ public class MaxTupleBy1stField extends
         return Final.class.getName();
     }
 
-
-    /**
-     * Accumulator implementation
-     */
-
-    private Tuple intermediate = null;
-
-    /**
-     * Accumulate implementation - calls max() on the incoming tuple set including intermediate tuple if already exists
-     * @param b A tuple containing a single field, which is a bag.  The bag will contain the set
-     * @throws IOException
-     */
-    @Override
-    public void accumulate(Tuple b) throws IOException {
-        try{
-            DataBag values = BagFactory.getInstance().newDefaultBag();
-            values.addAll((DataBag) b.get(0));
-
-            if (intermediate != null) {
-                values.add(intermediate);
-            }
-            intermediate = max(values,reporter);
-
-        }catch (ExecException ee){
-            IOException oughtToBeEE = new IOException();
-            oughtToBeEE.initCause(ee);
-            throw oughtToBeEE;
-        }
-    }
-
-    @Override
-    public Tuple getValue() {
-        return intermediate;
-    }
-
-    @Override
-    public void cleanup() {
-        intermediate = null;
-    }
-
 }

Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/Over.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/Over.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/Over.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/Over.java Fri Feb 24 03:34:37 2017
@@ -23,13 +23,10 @@ import java.util.Iterator;
 import java.util.List;
 
 import org.apache.pig.EvalFunc;
+import org.apache.pig.FuncSpec;
 import org.apache.pig.PigException;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.builtin.AVG;
-import org.apache.pig.builtin.BigDecimalAvg;
-import org.apache.pig.builtin.BigDecimalMax;
-import org.apache.pig.builtin.BigDecimalMin;
-import org.apache.pig.builtin.BigDecimalSum;
 import org.apache.pig.builtin.COUNT;
 import org.apache.pig.builtin.DoubleAvg;
 import org.apache.pig.builtin.DoubleMax;
@@ -57,7 +54,6 @@ import org.apache.pig.data.Tuple;
 import org.apache.pig.data.TupleFactory;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
-import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
 
 /**
  * Given an aggregate function, a bag, and possibly a window definition,
@@ -77,27 +73,23 @@ import org.apache.pig.impl.logicalLayer.
  *           <li>sum(int)</li>
  *           <li>sum(long)</li>
  *           <li>sum(bytearray)</li>
- *           <li>sum(bigdecimal)</li>
  *           <li>avg(double)</li>
  *           <li>avg(float)</li>
  *           <li>avg(long)</li>
  *           <li>avg(int)</li>
  *           <li>avg(bytearray)</li>
- *           <li>avg(bigdecimal)</li>
  *           <li>min(double)</li>
  *           <li>min(float)</li>
  *           <li>min(long)</li>
  *           <li>min(int)</li>
  *           <li>min(chararray)</li>
  *           <li>min(bytearray)</li>
- *           <li>min(bigdecimal)</li>
  *           <li>max(double)</li>
  *           <li>max(float)</li>
  *           <li>max(long)</li>
  *           <li>max(int)</li>
  *           <li>max(chararray)</li>
  *           <li>max(bytearray)</li>
- *           <li>max(bigdecimal)</li>
  *           <li>row_number</li>
  *           <li>first_value</li>
  *           <li>last_value</li>
@@ -161,8 +153,7 @@ import org.apache.pig.impl.logicalLayer.
  * current row and 3 following) over T;</tt>
  *
  * <p>Over accepts a constructor argument specifying the name and type,
- * colon-separated, of its return schema. If the argument option is 'true' use the inner-search,
- * take the name and type of bag and return a schema with alias+'_over' and the same type</p>
+ * colon-separated, of its return schema.</p>
  *
  * <p><pre>
  * DEFINE IOver org.apache.pig.piggybank.evaluation.Over('state_rk:int');
@@ -197,14 +188,12 @@ public class Over extends EvalFunc<DataB
     private Object[] udfArgs;
     private byte   returnType;
     private String returnName;
-    private boolean searchInnerType;
 
     public Over() {
         initialized = false;
         udfArgs = null;
         func = null;
         returnType = DataType.UNKNOWN;
-        searchInnerType = false;
     }
 
     public Over(String typespec) {
@@ -213,16 +202,12 @@ public class Over extends EvalFunc<DataB
             String[] fn_tn = typespec.split(":", 2);
             this.returnName = fn_tn[0];
             this.returnType = DataType.findTypeByName(fn_tn[1]);
-        } else if(Boolean.parseBoolean(typespec)) {
-            searchInnerType = Boolean.parseBoolean(typespec);
-        }else{
+        } else {
             this.returnName = "result";
             this.returnType = DataType.findTypeByName(typespec);
-        }       
+        }
     }
 
-
-
     @Override
     public DataBag exec(Tuple input) throws IOException {
         if (input == null || input.size() < 2) {
@@ -270,42 +255,19 @@ public class Over extends EvalFunc<DataB
     @Override
     public Schema outputSchema(Schema inputSch) {
         try {
-            FieldSchema field;
-
-            if (searchInnerType) {
-                field = new FieldSchema(inputSch.getField(0));
-                while (searchInnerType) {
-                    if (field.schema != null
-                            && field.schema.getFields().size() > 1) {
-                        searchInnerType = false;
-                    } else {
-                        if (field.type == DataType.TUPLE
-                                || field.type == DataType.BAG) {
-                            field = new FieldSchema(field.schema.getField(0));
-                        } else {
-                            field.alias = field.alias + "_over";
-                            searchInnerType = false;
-                        }
-                    }
-                }
-
-                searchInnerType = true;
-            } else if (returnType == DataType.UNKNOWN) {
+            if (returnType == DataType.UNKNOWN) {
                 return Schema.generateNestedSchema(DataType.BAG, DataType.NULL);
             } else {
-                field = new Schema.FieldSchema(returnName, returnType);
+                Schema outputTupleSchema = new Schema(new Schema.FieldSchema(returnName, returnType));
+                return new Schema(new Schema.FieldSchema(
+                        getSchemaName(this.getClass().getName().toLowerCase(), inputSch),
+                            outputTupleSchema, 
+                            DataType.BAG));
             }
-
-            Schema outputTupleSchema = new Schema(field);
-            return new Schema(new Schema.FieldSchema(getSchemaName(this
-                    .getClass().getName().toLowerCase(), inputSch),
-                    outputTupleSchema, DataType.BAG));
-
         } catch (FrontendException fe) {
             throw new RuntimeException("Unable to create nested schema", fe);
         }
     }
-    
 
     private void init(Tuple input) throws IOException {
         initialized = true;
@@ -367,8 +329,6 @@ public class Over extends EvalFunc<DataB
             func = new LongSum();
         } else if ("sum(bytearray)".equalsIgnoreCase(agg)) {
             func = new SUM();
-        } else if ("sum(bigdecimal)".equalsIgnoreCase(agg)) {
-            func = new BigDecimalSum();
         } else if ("avg(double)".equalsIgnoreCase(agg)) {
             func = new DoubleAvg();
         } else if ("avg(float)".equalsIgnoreCase(agg)) {
@@ -379,8 +339,6 @@ public class Over extends EvalFunc<DataB
             func = new IntAvg();
         } else if ("avg(bytearray)".equalsIgnoreCase(agg)) {
             func = new AVG();
-        } else if ("avg(bigdecimal)".equalsIgnoreCase(agg)) {
-            func = new BigDecimalAvg();
         } else if ("min(double)".equalsIgnoreCase(agg)) {
             func = new DoubleMin();
         } else if ("min(float)".equalsIgnoreCase(agg)) {
@@ -393,8 +351,6 @@ public class Over extends EvalFunc<DataB
             func = new StringMin();
         } else if ("min(bytearray)".equalsIgnoreCase(agg)) {
             func = new MIN();
-        } else if ("min(bigdecimal)".equalsIgnoreCase(agg)) {
-            func = new BigDecimalMin();
         } else if ("max(double)".equalsIgnoreCase(agg)) {
             func = new DoubleMax();
         } else if ("max(float)".equalsIgnoreCase(agg)) {
@@ -407,8 +363,6 @@ public class Over extends EvalFunc<DataB
             func = new StringMax();
         } else if ("max(bytearray)".equalsIgnoreCase(agg)) {
             func = new MAX();
-        } else if ("max(bigdecimal)".equalsIgnoreCase(agg)) {
-            func = new BigDecimalMax();
         } else if ("row_number".equalsIgnoreCase(agg)) {
             func = new RowNumber();
         } else if ("first_value".equalsIgnoreCase(agg)) {

Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/SearchEngineExtractor.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/SearchEngineExtractor.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/SearchEngineExtractor.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/SearchEngineExtractor.java Fri Feb 24 03:34:37 2017
@@ -363,15 +363,6 @@ public class SearchEngineExtractor exten
         searchEngines.put("search.lycos.com", "Lycos");
         searchEngines.put("search.msn.co.uk", "MSN UK");
         searchEngines.put("search.msn.com", "MSN");
-        searchEngines.put("bing.com", "Bing");
-        searchEngines.put("ssl.bing.com", "Bing");
-        searchEngines.put("cn.bing.com", "Bing China");
-        searchEngines.put("br.bing.com", "Bing Brazil");
-        searchEngines.put("it.bing.com", "Bing Italy");
-        searchEngines.put("be.bing.com", "Bing Netherlands");
-        searchEngines.put("uk.bing.com", "Bing UK");
-        searchEngines.put("hk.bing.com", "Bing Hong Kong");
-        searchEngines.put("nz.bing.com", "Bing New Zeland");
         searchEngines.put("search.myway.com", "MyWay");
         searchEngines.put("search.mywebsearch.com", "My Web Search");
         searchEngines.put("search.ntlworld.com", "NTLWorld");

Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/xml/XPath.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/xml/XPath.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/xml/XPath.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/xml/XPath.java Fri Feb 24 03:34:37 2017
@@ -16,11 +16,8 @@ package org.apache.pig.piggybank.evaluat
 import java.io.IOException;
 import java.io.StringReader;
 import java.util.ArrayList;
-import java.util.Iterator;
 import java.util.List;
 
-import javax.xml.XMLConstants;
-import javax.xml.namespace.NamespaceContext;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.xpath.XPathFactory;
@@ -52,7 +49,8 @@ public class XPath extends EvalFunc<Stri
     
     private static boolean cache = true;
     private static boolean ignoreNamespace = true;
-
+    public static final String EMPTY_STRING = "";
+    
     /**
      * input should contain: 1) xml 2) xpath 
      *                       3) optional cache xml doc flag 
@@ -97,13 +95,8 @@ public class XPath extends EvalFunc<Stri
                 return null;
             }
             
-            if(input.size() > 2) {
+            if(input.size() > 2)
                 cache = (Boolean) input.get(2);
-            }
-
-            if (input.size() > 3) {
-                ignoreNamespace = (Boolean) input.get(3);
-            }
 
             if (!cache || xpath == null || !xml.equals(this.xml)) {
                 final InputSource source = new InputSource(new StringReader(xml));
@@ -111,7 +104,6 @@ public class XPath extends EvalFunc<Stri
                 this.xml = xml; // track the xml for subsequent calls to this udf
 
                 final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-                dbf.setNamespaceAware(!ignoreNamespace);
                 final DocumentBuilder db = dbf.newDocumentBuilder();
 
                 this.document = db.parse(source);
@@ -120,32 +112,14 @@ public class XPath extends EvalFunc<Stri
 
                 this.xpath = xpathFactory.newXPath();
 
-                if (!ignoreNamespace) {
-                    xpath.setNamespaceContext(new NamespaceContext() {
-                        @Override
-                        public String getNamespaceURI(String prefix) {
-                            if (prefix.equals(XMLConstants.DEFAULT_NS_PREFIX)) {
-                                return document.lookupNamespaceURI(null);
-                            } else {
-                                return document.lookupNamespaceURI(prefix);
-                            }
-                        }
-
-                        @Override
-                        public String getPrefix(String namespaceURI) {
-                            return document.lookupPrefix(namespaceURI);
-                        }
-
-                        @Override
-                        public Iterator getPrefixes(String namespaceURI) {
-                            return null;
-                        }
-                    });
-                }
             }
 
             String xpathString = (String) input.get(1);
 
+            if (ignoreNamespace) {
+                xpathString = createNameSpaceIgnoreXpathString(xpathString);
+            }
+
             final String value = xpath.evaluate(xpathString, document);
 
             return value;
@@ -191,6 +165,34 @@ public class XPath extends EvalFunc<Stri
         }
         return true;
     }
+    
+    
+    /**
+     * Returns a new the xPathString by adding additional parameters 
+     * in the existing xPathString for ignoring the namespace during compilation.
+     * 
+     * @param String xpathString
+     * @return String modified xpathString
+     */
+    private String createNameSpaceIgnoreXpathString(final String xpathString) {
+        final String QUERY_PREFIX = "//*";
+        final String LOCAL_PREFIX = "[local-name()='";
+        final String LOCAL_POSTFIX = "']";
+        final String SPLITTER = "/";
+
+        try {
+            String xpathStringWithLocalName = EMPTY_STRING;
+            String[] individualNodes = xpathString.split(SPLITTER);
+
+            for (String node : individualNodes) {
+                xpathStringWithLocalName = xpathStringWithLocalName.concat(QUERY_PREFIX + LOCAL_PREFIX + node
+                        + LOCAL_POSTFIX);
+            }
+            return xpathStringWithLocalName;
+        } catch (Exception ex) {
+            return xpathString;
+        }
+    }
 
     /**
      * Returns argument schemas of the UDF.

Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/CSVExcelStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/CSVExcelStorage.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/CSVExcelStorage.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/CSVExcelStorage.java Fri Feb 24 03:34:37 2017
@@ -580,7 +580,7 @@ public class CSVExcelStorage extends Pig
                 }
             } else if (b == DOUBLE_QUOTE) {
                 // Does a double quote immediately follow?                  
-                if ((i < recordLen-1) && (buf[i+1] == DOUBLE_QUOTE) && (fieldBuffer.position() != 0)) {
+                if ((i < recordLen-1) && (buf[i+1] == DOUBLE_QUOTE)) {
                     fieldBuffer.put(b);
                     nextTupleSkipChar = true;
                     continue;

Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java Fri Feb 24 03:34:37 2017
@@ -91,7 +91,6 @@ public class DBStorage extends StoreFunc
   /**
    * Write the tuple to Database directly here.
    */
-  @Override
   public void putNext(Tuple tuple) throws IOException {
     int sqlPos = 1;
     try {
@@ -374,9 +373,4 @@ public class DBStorage extends StoreFunc
       p.setProperty(SCHEMA_SIGNATURE, s.toString());
   }
 
-  @Override
-  public Boolean supportsParallelWriteToStoreLocation() {
-    return false;
-  }
-
 }

Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/IndexedStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/IndexedStorage.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/IndexedStorage.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/IndexedStorage.java Fri Feb 24 03:34:37 2017
@@ -60,6 +60,7 @@ import org.apache.pig.data.TupleFactory;
 import org.apache.pig.impl.util.StorageUtil;
 import org.apache.pig.data.DataType;
 import org.apache.pig.data.DataByteArray;
+import org.apache.pig.backend.hadoop.executionengine.shims.HadoopShims;
 
 /**
  * <code>IndexedStorage</code> is a form of <code>PigStorage</code> that supports a

Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/MultiStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/MultiStorage.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/MultiStorage.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/MultiStorage.java Fri Feb 24 03:34:37 2017
@@ -16,9 +16,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.text.NumberFormat;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
@@ -44,9 +42,6 @@ import org.apache.pig.backend.executione
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRConfiguration;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.util.StorageUtil;
-import org.apache.xml.utils.StringBufferPool;
-
-import com.google.common.base.Strings;
 
 /**
  * The UDF is useful for splitting the output data into a bunch of directories
@@ -78,21 +73,13 @@ import com.google.common.base.Strings;
  * If the output is compressed,then the sub directories and the output files will
  * be having the extension. Say for example in the above case if bz2 is used one file 
  * will look like ;/my/home/output.bz2/a1.bz2/a1-0000.bz2
- *
- * Key field can also be a comma separated list of indices e.g. '0,1' - in this case
- * storage will be multi-level:
- * /my/home/output/a1/b1/a1-b1-0000
- * /my/home/output/a1/b2/a1-b2-0000
- * There is also an option to leave key values out of storage, see isRemoveKeys.
  */
 public class MultiStorage extends StoreFunc {
 
-  private static final String KEYFIELD_DELIMETER = ",";
   private Path outputPath; // User specified output Path
-  private final List<Integer> splitFieldIndices= new ArrayList<Integer>(); // Indices of the key fields
+  private int splitFieldIndex = -1; // Index of the key field
   private String fieldDel; // delimiter of the output record.
   private Compression comp; // Compression type of output data.
-  private boolean isRemoveKeys = false;
   
   // Compression types supported by this store
   enum Compression {
@@ -108,14 +95,9 @@ public class MultiStorage extends StoreF
     this(parentPathStr, splitFieldIndex, compression, "\\t");
   }
 
-  public MultiStorage(String parentPathStr, String splitFieldIndex,
-      String compression, String fieldDel) {
-    this(parentPathStr, splitFieldIndex, compression, fieldDel, "false");
-  }
-
   /**
    * Constructor
-   *
+   * 
    * @param parentPathStr
    *          Parent output dir path (this will be specified in store statement,
    *            so MultiStorage don't use this parameter in reality. However, we don't
@@ -126,26 +108,18 @@ public class MultiStorage extends StoreF
    *          'bz2', 'bz', 'gz' or 'none'
    * @param fieldDel
    *          Output record field delimiter.
-   * @param isRemoveKeys
-   *          Removes key columns from result during write.
    */
   public MultiStorage(String parentPathStr, String splitFieldIndex,
-                      String compression, String fieldDel, String isRemoveKeys) {
-    this.isRemoveKeys = Boolean.parseBoolean(isRemoveKeys);
+      String compression, String fieldDel) {
     this.outputPath = new Path(parentPathStr);
-
-    String[] splitFieldIndices = splitFieldIndex.split(KEYFIELD_DELIMETER);
-    for (String splitFieldIndexString : splitFieldIndices){
-      this.splitFieldIndices.add(Integer.parseInt(splitFieldIndexString));
-    }
-
+    this.splitFieldIndex = Integer.parseInt(splitFieldIndex);
     this.fieldDel = fieldDel;
     try {
       this.comp = (compression == null) ? Compression.none : Compression
-              .valueOf(compression.toLowerCase());
+        .valueOf(compression.toLowerCase());
     } catch (IllegalArgumentException e) {
       System.err.println("Exception when converting compression string: "
-              + compression + " to enum. No compression will be used");
+          + compression + " to enum. No compression will be used");
       this.comp = Compression.none;
     }
   }
@@ -153,26 +127,22 @@ public class MultiStorage extends StoreF
   //--------------------------------------------------------------------------
   // Implementation of StoreFunc
 
-  private RecordWriter<List<String>, Tuple> writer;
+  private RecordWriter<String, Tuple> writer;
   
   @Override
   public void putNext(Tuple tuple) throws IOException {
-    for (int splitFieldIndex : this.splitFieldIndices) {
-      if (tuple.size() <= splitFieldIndex) {
-        throw new IOException("split field index:" + splitFieldIndex
-                + " >= tuple size:" + tuple.size());
-      }
+    if (tuple.size() <= splitFieldIndex) {
+      throw new IOException("split field index:" + this.splitFieldIndex
+          + " >= tuple size:" + tuple.size());
     }
-    List<String> fields = new ArrayList<String>();
-    for (int splitFieldIndex : this.splitFieldIndices){
-      try {
-        fields.add(String.valueOf(tuple.get(splitFieldIndex)));
-      } catch (ExecException exec) {
-        throw new IOException(exec);
-      }
+    Object field = null;
+    try {
+      field = tuple.get(splitFieldIndex);
+    } catch (ExecException exec) {
+      throw new IOException(exec);
     }
     try {
-      writer.write(fields, tuple);
+      writer.write(String.valueOf(field), tuple);
     } catch (InterruptedException e) {
       throw new IOException(e);
     }
@@ -183,9 +153,6 @@ public class MultiStorage extends StoreF
   public OutputFormat getOutputFormat() throws IOException {
       MultiStorageOutputFormat format = new MultiStorageOutputFormat();
       format.setKeyValueSeparator(fieldDel);
-      if (this.isRemoveKeys){
-        format.setSkipIndices(this.splitFieldIndices);
-      }
       return format;
   }
     
@@ -207,33 +174,27 @@ public class MultiStorage extends StoreF
       FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);
     }
   }
-
-  @Override
-  public Boolean supportsParallelWriteToStoreLocation() {
-    return false;
-  }
-
+ 
   //--------------------------------------------------------------------------
   // Implementation of OutputFormat
   
   public static class MultiStorageOutputFormat extends
-  TextOutputFormat<List<String>, Tuple> {
+  TextOutputFormat<String, Tuple> {
 
     private String keyValueSeparator = "\\t";
     private byte fieldDel = '\t';
-    private List<Integer> skipIndices = null;
-
+  
     @Override
-    public RecordWriter<List<String>, Tuple>
+    public RecordWriter<String, Tuple> 
     getRecordWriter(TaskAttemptContext context
                 ) throws IOException, InterruptedException {
     
       final TaskAttemptContext ctx = context;
         
-      return new RecordWriter<List<String>, Tuple>() {
+      return new RecordWriter<String, Tuple>() {
 
-        private Map<List<String>, MyLineRecordWriter> storeMap =
-              new HashMap<List<String>, MyLineRecordWriter>();
+        private Map<String, MyLineRecordWriter> storeMap = 
+              new HashMap<String, MyLineRecordWriter>();
           
         private static final int BUFFER_SIZE = 1024;
           
@@ -241,7 +202,7 @@ public class MultiStorage extends StoreF
               new ByteArrayOutputStream(BUFFER_SIZE);
                            
         @Override
-        public void write(List<String> key, Tuple val) throws IOException {
+        public void write(String key, Tuple val) throws IOException {                
           int sz = val.size();
           for (int i = 0; i < sz; i++) {
             Object field;
@@ -251,13 +212,9 @@ public class MultiStorage extends StoreF
               throw ee;
             }
 
-            boolean skipCurrentField = skipIndices != null && skipIndices.contains(i);
+            StorageUtil.putField(mOut, field);
 
-            if (!skipCurrentField) {
-              StorageUtil.putField(mOut, field);
-            }
-
-            if (i != sz - 1 && !skipCurrentField) {
+            if (i != sz - 1) {
               mOut.write(fieldDel);
             }
           }
@@ -274,17 +231,17 @@ public class MultiStorage extends StoreF
           }
         }
       
-        private MyLineRecordWriter getStore(List<String> fieldValues) throws IOException {
-          MyLineRecordWriter store = storeMap.get(fieldValues);
+        private MyLineRecordWriter getStore(String fieldValue) throws IOException {
+          MyLineRecordWriter store = storeMap.get(fieldValue);
           if (store == null) {                  
-            DataOutputStream os = createOutputStream(fieldValues);
+            DataOutputStream os = createOutputStream(fieldValue);
             store = new MyLineRecordWriter(os, keyValueSeparator);
-            storeMap.put(fieldValues, store);
+            storeMap.put(fieldValue, store);
           }
           return store;
         }
           
-        private DataOutputStream createOutputStream(List<String> fieldValues) throws IOException {
+        private DataOutputStream createOutputStream(String fieldValue) throws IOException {
           Configuration conf = ctx.getConfiguration();
           TaskID taskId = ctx.getTaskAttemptID().getTaskID();
           
@@ -302,21 +259,7 @@ public class MultiStorage extends StoreF
           NumberFormat nf = NumberFormat.getInstance();
           nf.setMinimumIntegerDigits(4);
 
-          StringBuffer pathStringBuffer = new StringBuffer();
-          for (String fieldValue : fieldValues){
-            String safeFieldValue = fieldValue.replaceAll("\\/","-");
-            pathStringBuffer.append(safeFieldValue);
-            pathStringBuffer.append("/");
-          }
-          pathStringBuffer.deleteCharAt(pathStringBuffer.length()-1);
-          String pathString = pathStringBuffer.toString();
-          String idString = pathString.replaceAll("\\/","-");
-
-          if (!Strings.isNullOrEmpty(extension)){
-            pathString = pathString.replaceAll("\\/",extension+"\\/");
-          }
-
-          Path path = new Path(pathString+extension, idString + '-'
+          Path path = new Path(fieldValue+extension, fieldValue + '-'
                 + nf.format(taskId.getId())+extension);
           Path workOutputPath = ((FileOutputCommitter)getOutputCommitter(ctx)).getWorkPath();
           Path file = new Path(workOutputPath, path);
@@ -336,12 +279,8 @@ public class MultiStorage extends StoreF
       keyValueSeparator = sep;
       fieldDel = StorageUtil.parseFieldDel(keyValueSeparator);  
     }
-
-    public void setSkipIndices(List<Integer> skipIndices) {
-      this.skipIndices = skipIndices;
-    }
-
-    //------------------------------------------------------------------------
+  
+  //------------------------------------------------------------------------
   //
   
     protected static class MyLineRecordWriter

Modified: pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/evaluation/TestOver.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/evaluation/TestOver.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/evaluation/TestOver.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/evaluation/TestOver.java Fri Feb 24 03:34:37 2017
@@ -18,11 +18,12 @@
 package org.apache.pig.piggybank.evaluation;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
-import java.math.BigDecimal;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Random;
 
 import org.apache.pig.backend.executionengine.ExecException;
@@ -33,6 +34,8 @@ import org.apache.pig.data.DataType;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.data.TupleFactory;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+
+import org.junit.Before;
 import org.junit.Test;
 
 public class TestOver {
@@ -63,25 +66,11 @@ public class TestOver {
         out = func.outputSchema(in);
         assertEquals("{org.apache.pig.piggybank.evaluation.over_3: {result: double}}", out.toString());
 
-        // bigdecimal
-        func = new Over("BIGDECIMAL");
-        in = Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER);
-        out = func.outputSchema(in);
-        assertEquals("{org.apache.pig.piggybank.evaluation.over_4: {result: bigdecimal}}", out.toString());
-        
         // named 
         func = new Over("bob:chararray");
         in = Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER);
         out = func.outputSchema(in);
-        assertEquals("{org.apache.pig.piggybank.evaluation.over_5: {bob: chararray}}", out.toString());
-        
-        
-        // Search inner alias and type
-        func = new Over("true");
-        in = Schema.generateNestedSchema(DataType.BAG, DataType.BIGDECIMAL);
-        in.getField(0).schema.getField(0).alias="test";
-        out = func.outputSchema(in);
-        assertEquals("{org.apache.pig.piggybank.evaluation.over_6: {test_over: bigdecimal}}", out.toString());
+        assertEquals("{org.apache.pig.piggybank.evaluation.over_4: {bob: chararray}}", out.toString());
     }
 
     @Test
@@ -408,28 +397,6 @@ public class TestOver {
             assertEquals(new Long(10), to.get(0));
         }
     }
-    
-    @Test
-    public void testSumBigDecimal() throws Exception {
-    	Over func = new Over();
-        DataBag inbag = BagFactory.getInstance().newDefaultBag();
-        for (int i = 0; i < 10; i++) {
-            Tuple t = TupleFactory.getInstance().newTuple(1);
-            t.set(0, new BigDecimal(1));
-            inbag.add(t);
-        }
-        Tuple t = TupleFactory.getInstance().newTuple(4);
-        t.set(0, inbag);
-        t.set(1, "sum(bigdecimal)");
-        t.set(2, -1);
-        t.set(3, -1);
-        DataBag outbag = func.exec(t);
-        assertEquals(10, outbag.size());
-        for (Tuple to : outbag) {
-            assertEquals(1, to.size());
-            assertEquals(new BigDecimal(10), to.get(0));
-        }
-    }
 
     @Test
     public void testAvgDouble() throws Exception {
@@ -542,29 +509,6 @@ public class TestOver {
     }
     
     @Test
-    public void testAvgBigDecimal() throws Exception {
-        Over func = new Over();
-        DataBag inbag = BagFactory.getInstance().newDefaultBag();
-        for (int i = 0; i < 10; i++) {
-            Tuple t = TupleFactory.getInstance().newTuple(1);
-            t.set(0, new BigDecimal(i));
-            inbag.add(t);
-        }
-        Tuple t = TupleFactory.getInstance().newTuple(4);
-        t.set(0, inbag);
-        t.set(1, "avg(bigdecimal)");
-        t.set(2, -1);
-        t.set(3, -1);
-        DataBag outbag = func.exec(t);
-        assertEquals(10, outbag.size());
-        for (Tuple to : outbag) {
-            assertEquals(1, to.size());
-            assertEquals(new BigDecimal(4.5), to.get(0));
-        }
-    }
-    
-    
-    @Test
     public void testMinDouble() throws Exception {
         Over func = new Over();
         DataBag inbag = BagFactory.getInstance().newDefaultBag();
@@ -683,26 +627,6 @@ public class TestOver {
             assertEquals("0", to.get(0));
         }
     }
-    
-    @Test
-    public void testMinBigDecimal() throws Exception {
-        Over func = new Over();
-        DataBag inbag = BagFactory.getInstance().newDefaultBag();
-        for (int i = 0; i < 10; i++) {
-            Tuple t = TupleFactory.getInstance().newTuple(1);
-            t.set(0,  new BigDecimal(i));
-            inbag.add(t);
-        }
-        Tuple t = TupleFactory.getInstance().newTuple(2);
-        t.set(0, inbag);
-        t.set(1, "min(bigdecimal)");
-        DataBag outbag = func.exec(t);
-        assertEquals(10, outbag.size());
-        for (Tuple to : outbag) {
-            assertEquals(1, to.size());
-            assertEquals(new BigDecimal(0), to.get(0));
-        }
-    }
 
     @Test
     public void testMaxDouble() throws Exception {
@@ -830,28 +754,6 @@ public class TestOver {
             assertEquals("9", to.get(0));
         }
     }
-    
-    @Test
-    public void testMaxBigDecimal() throws Exception {
-        Over func = new Over();
-        DataBag inbag = BagFactory.getInstance().newDefaultBag();
-        for (int i = 0; i < 10; i++) {
-            Tuple t = TupleFactory.getInstance().newTuple(1);
-            t.set(0, new BigDecimal(i));
-            inbag.add(t);
-        }
-        Tuple t = TupleFactory.getInstance().newTuple(2);
-        t.set(0, inbag);
-        t.set(1, "max(bigdecimal)");
-        DataBag outbag = func.exec(t);
-        assertEquals(10, outbag.size());
-        int count = 0;
-        for (Tuple to : outbag) {
-            assertEquals(1, to.size());
-            assertEquals(new BigDecimal(count++), to.get(0));
-        }
-    }
-	
 
     @Test
     public void testRowNumber() throws Exception {

Modified: pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/xml/XPathTest.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/xml/XPathTest.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/xml/XPathTest.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/xml/XPathTest.java Fri Feb 24 03:34:37 2017
@@ -151,95 +151,6 @@ public class XPathTest {
     }
 
     @Test
-    public void testExecTupleWithDontIgnoreNamespace() throws Exception {
-
-        final XPath xpath = new XPath();
-
-        final Tuple tuple = mock(Tuple.class);
-
-        when(tuple.get(0)).thenReturn("<?xml version='1.0'?>\n" +
-                "<foo:document xmlns:foo=\"http://apache.org/foo\" xmlns:bar=\"http://apache.org/bar\">" +
-                "<bar:element>MyBar</bar:element>" +
-                "</foo:document>");
-
-        when(tuple.size()).thenReturn(4);
-        when(tuple.get(2)).thenReturn(true);
-        when(tuple.get(3)).thenReturn(false);
-
-        when(tuple.get(1)).thenReturn("/foo:document/bar:element");
-        assertEquals("MyBar", xpath.exec(tuple));
-
-    }
-
-    @Test
-    public void testExecTupleWithDontIgnoreNamespace() throws Exception {
-
-        final XPath xpath = new XPath();
-
-        final Tuple tuple = mock(Tuple.class);
-
-        when(tuple.get(0)).thenReturn("<?xml version='1.0'?>\n" +
-                "<foo:document xmlns:foo=\"http://apache.org/foo\" xmlns:bar=\"http://apache.org/bar\">" +
-                "<bar:element>MyBar</bar:element>" +
-                "</foo:document>");
-
-        when(tuple.size()).thenReturn(4);
-        when(tuple.get(2)).thenReturn(true);
-        when(tuple.get(3)).thenReturn(false);
-
-        when(tuple.get(1)).thenReturn("/foo:document/bar:element");
-        assertEquals("MyBar", xpath.exec(tuple));
-
-    }
-
-    @Test
-    public void testExecTupleWithDontIgnoreNamespace() throws Exception {
-
-        final XPath xpath = new XPath();
-
-        final Tuple tuple = mock(Tuple.class);
-
-        when(tuple.get(0)).thenReturn("<?xml version='1.0'?>\n" +
-                "<foo:document xmlns:foo=\"http://apache.org/foo\" xmlns:bar=\"http://apache.org/bar\">" +
-                "<bar:element>MyBar</bar:element>" +
-                "</foo:document>");
-
-        when(tuple.size()).thenReturn(4);
-        when(tuple.get(2)).thenReturn(true);
-        when(tuple.get(3)).thenReturn(false);
-
-        when(tuple.get(1)).thenReturn("/foo:document/bar:element");
-        assertEquals("MyBar", xpath.exec(tuple));
-
-    }
-
-
-    @Test
-    public void testFunctionInXPath() throws Exception {
-
-        final XPath xpath = new XPath();
-
-        final Tuple tuple = mock(Tuple.class);
-
-        when(tuple.get(0)).thenReturn("<Aa name=\"test1\">" +
-                "<Bb Cc=\"1\"/>" +
-                "<Bb Cc=\"1\"/>" +
-                "<Bb Cc=\"1\"/>" +
-                "<Bb Cc=\"1\"/>" +
-                "<Dd>test2</Dd>" +
-                "</Aa>");
-
-        when(tuple.size()).thenReturn(4);
-        when(tuple.get(1)).thenReturn("sum(Aa/Bb/@Cc)");
-        when(tuple.get(2)).thenReturn(true);
-        when(tuple.get(3)).thenReturn(true);
-
-        assertEquals("4", xpath.exec(tuple));
-
-    }
-
-
-    @Test
     public void testExecTupleWithElementNodeWithComplexNameSpace() throws Exception {
 
         final XPath xpath = new XPath();
@@ -299,31 +210,7 @@ public class XPathTest {
         assertEquals("4 stars3.5 stars4 stars4.2 stars3.5 stars", xpath.exec(tuple));
 
     }
-
-    @Test
-    public void testFunctionInXPath() throws Exception {
-
-        final XPath xpath = new XPath();
-
-        final Tuple tuple = mock(Tuple.class);
-
-        when(tuple.get(0)).thenReturn("<Aa name=\"test1\">" +
-                "<Bb Cc=\"1\"/>" +
-                "<Bb Cc=\"1\"/>" +
-                "<Bb Cc=\"1\"/>" +
-                "<Bb Cc=\"1\"/>" +
-                "<Dd>test2</Dd>" +
-                "</Aa>");
-
-        when(tuple.size()).thenReturn(4);
-        when(tuple.get(1)).thenReturn("sum(Aa/Bb/@Cc)");
-        when(tuple.get(2)).thenReturn(true);
-        when(tuple.get(3)).thenReturn(true);
-
-        assertEquals("4", xpath.exec(tuple));
-
-    }
-
+    
     @Ignore //--optional test
     @Test 
     public void testCacheBenefit() throws Exception{

Modified: pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCSVExcelStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCSVExcelStorage.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCSVExcelStorage.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestCSVExcelStorage.java Fri Feb 24 03:34:37 2017
@@ -218,7 +218,7 @@ public class TestCSVExcelStorage  {
         Util.registerMultiLineQuery(pig, script);
         Iterator<Tuple> it = pig.openIterator("a");
         Assert.assertEquals(Util.createTuple(new String[] {"foo,\"bar\",baz"}), it.next());
-        Assert.assertEquals(Util.createTuple(new String[] {"\"\"\""}), it.next());
+        Assert.assertEquals(Util.createTuple(new String[] {"\"\"\"\""}), it.next());
     }
 
     // Handle newlines in fields

Modified: pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestLogFormatLoader.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestLogFormatLoader.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestLogFormatLoader.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestLogFormatLoader.java Fri Feb 24 03:34:37 2017
@@ -109,7 +109,7 @@ public class TestLogFormatLoader {
         Tuple actual = out.get(0);
         Tuple expected = tuple(
             "2001:980:91c0:1:8d31:a232:25e5:85d",
-            "05/Sep/2010:11:27:50 +0200",
+            "[05/Sep/2010:11:27:50 +0200]",
             "koken-pannen_303_hs-koken-pannen-afj-120601_B3_product_1_9200000002876066",
             map(
                 "promo"       , "koken-pannen_303_hs-koken-pannen-afj-120601_B3_product_1_9200000002876066",

Modified: pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorage.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorage.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorage.java Fri Feb 24 03:34:37 2017
@@ -18,41 +18,34 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
-import java.util.List;
-import java.util.Map;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
+import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
-import org.apache.pig.backend.executionengine.ExecJob;
-import org.apache.pig.test.MiniGenericCluster;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.test.MiniCluster;
 import org.apache.pig.test.Util;
-import org.apache.pig.tools.pigstats.JobStats;
-import org.apache.pig.tools.pigstats.OutputStats;
-import org.apache.pig.tools.pigstats.PigStats;
-import org.apache.pig.tools.pigstats.mapreduce.SimplePigStats;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import junit.framework.Assert;
+import junit.framework.TestCase;
 
-public class TestMultiStorage {
+public class TestMultiStorage extends TestCase {
   private static final String INPUT_FILE = "MultiStorageInput.txt";
 
   private PigServer pigServer;
   private PigServer pigServerLocal;
 
-  private static MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
+  private MiniCluster cluster = MiniCluster.buildCluster();
 
-  public TestMultiStorage() throws Exception {
-    pigServer = new PigServer(cluster.getExecType(), cluster.getProperties());
-    pigServerLocal = new PigServer(Util.getLocalTestMode());
+  public TestMultiStorage() throws ExecException, IOException {
+    pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+    pigServerLocal = new PigServer(ExecType.LOCAL);
   }
 
   public static final PathFilter hiddenPathFilter = new PathFilter() {
@@ -81,83 +74,59 @@ public class TestMultiStorage {
     Util.copyFromLocalToCluster(cluster, INPUT_FILE, INPUT_FILE);
   }
 
+  @Override
   @Before
   public void setUp() throws Exception {
     createFile();
     FileSystem fs = FileSystem.getLocal(new Configuration());
     Path localOut = new Path("local-out");
+    Path dummy = new Path("dummy");
     if (fs.exists(localOut)) {
       fs.delete(localOut, true);
     }
+    if (fs.exists(dummy)) {
+      fs.delete(dummy, true);
+    }
   }
 
+  @Override
   @After
   public void tearDown() throws Exception {
     new File(INPUT_FILE).delete();
     Util.deleteFile(cluster, INPUT_FILE);
-  }
-
-  @AfterClass
-  public static void shutdown() {
     cluster.shutDown();
   }
 
   enum Mode {
     local, cluster
-  }
+  };
 
   @Test
   public void testMultiStorage() throws IOException {
     final String LOAD = "A = LOAD '" + INPUT_FILE + "' as (id, name, n);";
     final String MULTI_STORE_CLUSTER = "STORE A INTO 'mr-out' USING "
         + "org.apache.pig.piggybank.storage.MultiStorage('mr-out', '1');";
-    final String MULTI_STORE_LOCAL = "STORE A INTO 'local-out' USING "
+    final String MULTI_STORE_LOCAL = "STORE A INTO 'dummy' USING "
         + "org.apache.pig.piggybank.storage.MultiStorage('local-out', '1');";
 
     System.out.print("Testing in LOCAL mode: ...");
-    testMultiStorage(Mode.local, "local-out", LOAD, MULTI_STORE_LOCAL);
+    //testMultiStorage(Mode.local, "local-out", LOAD, MULTI_STORE_LOCAL);
     System.out.println("Succeeded!");
-
+    
     System.out.print("Testing in CLUSTER mode: ...");
     testMultiStorage( Mode.cluster, "mr-out", LOAD, MULTI_STORE_CLUSTER);
     System.out.println("Succeeded!");
+    
+    
   }
 
-  @Test
-  public void testOutputStats() throws IOException {
-    FileSystem fs = cluster.getFileSystem();
-
-    pigServer.setBatchOn();
-    pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "' as (id, name, n);");
-    pigServer.registerQuery("B = FILTER A BY name == 'apple';");
-    pigServer.registerQuery("STORE A INTO 'out1' USING org.apache.pig.piggybank.storage.MultiStorage('out1', '1');"); //153 bytes
-    pigServer.registerQuery("STORE B INTO 'out2' USING org.apache.pig.piggybank.storage.MultiStorage('out2', '1');"); // 45 bytes
-
-    ExecJob job = pigServer.executeBatch().get(0);
-
-    PigStats stats = job.getStatistics();
-    PigStats.JobGraph jobGraph = stats.getJobGraph();
-    JobStats jobStats = (JobStats) jobGraph.getSinks().get(0);
-    Map<String, Long> multiStoreCounters = jobStats.getMultiStoreCounters();
-    List<OutputStats> outputStats = SimplePigStats.get().getOutputStats();
-    OutputStats outputStats1 = "out1".equals(outputStats.get(0).getName()) ? outputStats.get(0) : outputStats.get(1);
-    OutputStats outputStats2 = "out2".equals(outputStats.get(0).getName()) ? outputStats.get(0) : outputStats.get(1);
-
-    assertEquals(153 + 45, stats.getBytesWritten());
-    assertEquals(2, outputStats.size()); // 2 split conditions
-    assertEquals(153, outputStats1.getBytes());
-    assertEquals(45, outputStats2.getBytes());
-    assertEquals(9, outputStats1.getRecords());
-    assertEquals(3, outputStats2.getRecords());
-    assertEquals(3L, multiStoreCounters.get("Output records in _1_out2").longValue());
-    assertEquals(9L, multiStoreCounters.get("Output records in _0_out1").longValue());
-
-    fs.delete(new Path("out1"), true);
-    fs.delete(new Path("out2"), true);
-  }
-
-    /**
-   * The actual method that run the test in local or cluster mode.
+  /**
+   * The actual method that run the test in local or cluster mode. 
+   * 
+   * @param pigServer
+   * @param mode
+   * @param queries
+   * @throws IOException
    */
   private void testMultiStorage( Mode mode, String outPath,
       String... queries) throws IOException {
@@ -173,38 +142,42 @@ public class TestMultiStorage {
   /**
    * Test if records are split into directories corresponding to split field
    * values
+   * 
+   * @param mode
+   * @throws IOException
    */
   private void verifyResults(Mode mode, String outPath) throws IOException {
     FileSystem fs = (Mode.local == mode ? FileSystem
         .getLocal(new Configuration()) : cluster.getFileSystem());
     Path output = new Path(outPath);
-    assertTrue("Output dir does not exists!", fs.exists(output)
+    Assert.assertTrue("Output dir does not exists!", fs.exists(output)
         && fs.getFileStatus(output).isDir());
 
     Path[] paths = FileUtil.stat2Paths(fs.listStatus(output, hiddenPathFilter));
-    assertTrue("Split field dirs not found!", paths != null);
+    Assert.assertTrue("Split field dirs not found!", paths != null);
 
     for (Path path : paths) {
       String splitField = path.getName();
       Path[] files = FileUtil.stat2Paths(fs.listStatus(path, hiddenPathFilter));
-      assertTrue("No files found for path: " + path.toUri().getPath(),
+      Assert.assertTrue("No files found for path: " + path.toUri().getPath(),
           files != null);
       for (Path filePath : files) {
-        assertTrue("This shouldn't be a directory", fs.isFile(filePath));
+        Assert.assertTrue("This shouldn't be a directory", fs.isFile(filePath));
+        
         BufferedReader reader = new BufferedReader(new InputStreamReader(fs
                 .open(filePath)));
         String line = "";
         int count = 0;
         while ((line = reader.readLine()) != null) {
           String[] fields = line.split("\\t");
-          assertEquals(fields.length, 3);
-          assertEquals("Unexpected field value in the output record",
+          Assert.assertEquals(fields.length, 3);
+          Assert.assertEquals("Unexpected field value in the output record",
                 splitField, fields[1]);
           count++;
           System.out.println("field: " + fields[1]);
-        }
+        }        
         reader.close();
-        assertEquals(count, 3);
+        Assert.assertEquals(count, 3);
       }
     }
   }

Modified: pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorageCompression.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorageCompression.java?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorageCompression.java (original)
+++ pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestMultiStorageCompression.java Fri Feb 24 03:34:37 2017
@@ -21,14 +21,11 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.Set;
 
 import junit.framework.TestCase;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.compress.BZip2Codec;
@@ -40,10 +37,6 @@ import org.apache.pig.backend.executione
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.test.Util;
 
-import com.google.common.collect.Sets;
-
-import org.junit.Assert;
-
 public class TestMultiStorageCompression extends TestCase {
 
    private static String patternString = "(\\d+)!+(\\w+)~+(\\w+)";
@@ -66,8 +59,8 @@ public class TestMultiStorageCompression
       filesToDelete.add(outputPath);
 
       try {
-         runQuery(outputPath, "0", type);
-         verifyResults(type, outputPath);
+         runQuery(outputPath, type);
+         verifyResults(type, filesToDelete, outputPath);
       } finally {
          cleanUpDirs(filesToDelete);
       }
@@ -84,22 +77,22 @@ public class TestMultiStorageCompression
       filesToDelete.add(outputPath);
 
       try {
-         runQuery(outputPath, "0", type);
-         verifyResults(type, outputPath);
+         runQuery(outputPath, type);
+         verifyResults(type, filesToDelete, outputPath);
       } finally {
          cleanUpDirs(filesToDelete);
       }
    }
 
-   private void cleanUpDirs(List<String> filesToDelete) throws IOException {
+   private void cleanUpDirs(List<String> filesToDelete) {
       // Delete files recursively
       Collections.reverse(filesToDelete);
       for (String string : filesToDelete)
-         FileUtils.deleteDirectory(new File(string));
+         new File(string).delete();
    }
 
 
-   private void verifyResults(String type,
+   private void verifyResults(String type, List<String> filesToDelete,
          String outputPath) throws IOException, FileNotFoundException {
       // Verify the output
       File outputDir = new File(outputPath);
@@ -121,10 +114,12 @@ public class TestMultiStorageCompression
              continue;
          String topFolder = outputPath + File.separator + indexFolder;
          File indexFolderFile = new File(topFolder);
+         filesToDelete.add(topFolder);
          String[] list = indexFolderFile.list();
          for (String outputFile : list) {
 
             String file = topFolder + File.separator + outputFile;
+            filesToDelete.add(file);
 
             // Skip off any file starting with .
             if (outputFile.startsWith("."))
@@ -164,7 +159,7 @@ public class TestMultiStorageCompression
       }
    }
 
-   private void runQuery(String outputPath, String keyColIndices, String compressionType)
+   private void runQuery(String outputPath, String compressionType)
          throws Exception, ExecException, IOException, FrontendException {
 
       // create a data file
@@ -177,7 +172,7 @@ public class TestMultiStorageCompression
 
       String query2 = "STORE A INTO '" + Util.encodeEscape(outputPath)
             + "' USING org.apache.pig.piggybank.storage.MultiStorage" + "('"
-            + Util.encodeEscape(outputPath) + "','"+keyColIndices+"', '" + compressionType + "', '\\t');";
+            + Util.encodeEscape(outputPath) + "','0', '" + compressionType + "', '\\t');";
 
       // Run Pig
       pig.setBatchOn();
@@ -187,32 +182,5 @@ public class TestMultiStorageCompression
       pig.executeBatch();
    }
 
-   public void testMultiStorageShouldSupportMultiLevelAndGz() throws Exception {
-      String type = "gz";
-      String outputDir = "output001.multi." + type;
-      List<String> filesToDelete = new ArrayList<String>();
-
-      String tmpDir = System.getProperty("java.io.tmpdir");
-      String outputPath = tmpDir + File.separator + outputDir;
-
-      filesToDelete.add(outputPath);
-      try {
-         runQuery(outputPath, "1,0", type);
-         Collection<File> fileList = FileUtils.listFiles(new File(outputPath),null,true);
-         Set<String> expectedPaths = Sets.newHashSet( "output001.multi.gz/a.gz/f1.gz/a-f1-0,000.gz",
-                                                      "output001.multi.gz/b.gz/f2.gz/b-f2-0,000.gz",
-                                                      "output001.multi.gz/c.gz/f3.gz/c-f3-0,000.gz",
-                                                      "output001.multi.gz/d.gz/f4.gz/d-f4-0,000.gz");
-         for (File file : fileList){
-            String foundPath = file.getAbsolutePath().substring(file.getAbsolutePath().indexOf(outputDir));
-            if (expectedPaths.contains(foundPath)){
-               expectedPaths.remove(foundPath);
-            }
-         }
-         Assert.assertTrue(expectedPaths.isEmpty());
-      } finally {
-         cleanUpDirs(filesToDelete);
-      }
-   }
 
 }

Modified: pig/branches/spark/ivy.xml
URL: http://svn.apache.org/viewvc/pig/branches/spark/ivy.xml?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/ivy.xml (original)
+++ pig/branches/spark/ivy.xml Fri Feb 24 03:34:37 2017
@@ -38,8 +38,10 @@
     <conf name="jdiff" visibility="private"/>
     <conf name="checkstyle" visibility="private"/>
     <conf name="buildJar" extends="compile,test" visibility="private"/>
-    <conf name="hadoop2" visibility="private"/>
-    <conf name="hbase1" visibility="private"/>
+    <conf name="hadoop20" visibility="private"/>
+    <conf name="hadoop23" visibility="private"/>
+    <conf name="hbase94" visibility="private"/>
+    <conf name="hbase95" visibility="private"/>
     <conf name="spark" visibility="private" />
   </configurations>
   <publications>
@@ -59,17 +61,17 @@
     <dependency org="commons-beanutils" name="commons-beanutils-core" rev="${commons-beanutils.version}"
       conf="checkstyle->master"/>
     <dependency org="xmlenc" name="xmlenc" rev="${xmlenc.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="com.sun.jersey" name="jersey-bundle" rev="${jersey.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="com.sun.jersey" name="jersey-server" rev="${jersey.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="com.sun.jersey.contribs" name="jersey-guice" rev="${jersey.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="commons-codec" name="commons-codec" rev="${commons-codec.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="commons-httpclient" name="commons-httpclient" rev="${commons-httpclient.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="commons-el" name="commons-el" rev="${commons-el.version}"
       conf="compile->master"/>
     <dependency org="commons-io" name="commons-io" rev="${commons-io.version}"
@@ -87,86 +89,92 @@
     <dependency org="nl.basjes.parse" name="parser-core" rev="${basjes-httpdlog-pigloader.version}"
       conf="compile->master"/>
     <dependency org="commons-configuration" name="commons-configuration" rev="${commons-configuration.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="commons-collections" name="commons-collections" rev="${commons-collections.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="javax.servlet" name="servlet-api" rev="${servlet-api.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="javax.ws.rs" name="jsr311-api" rev="${jsr311-api.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="com.google.protobuf" name="protobuf-java" rev="${protobuf-java.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="javax.inject" name="javax.inject" rev="${javax-inject.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="javax.xml.bind" name="jaxb-api" rev="${jaxb-api.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="com.sun.xml.bind" name="jaxb-impl" rev="${jaxb-impl.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/> 
     <dependency org="com.google.inject" name="guice" rev="${guice.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="com.google.inject.extensions" name="guice-servlet" rev="${guice-servlet.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="aopalliance" name="aopalliance" rev="${aopalliance.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="org.mortbay.jetty" name="jsp-2.1" rev="${jasper.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="org.mortbay.jetty" name="jsp-api-2.1" rev="${jasper.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="log4j" name="log4j" rev="${log4j.version}"
       conf="compile->master"/>
-    <dependency org="org.apache.hadoop" name="hadoop-annotations"
-      rev="${hadoop-common.version}" conf="hadoop2->master"/>
+    <dependency org="com.sun.jersey" name="jersey-core" rev="${jersey-core.version}"
+      conf="hadoop20->default"/>
+    <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop-core.version}"
+      conf="hadoop20->default"/>
+    <dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop-test.version}"
+      conf="hadoop20->default"/>
+    <dependency org="org.apache.hadoop" name="hadoop-annotations" 
+      rev="${hadoop-common.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-auth" 
-      rev="${hadoop-common.version}" conf="hadoop2->master"/>
+      rev="${hadoop-common.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-common" 
-      rev="${hadoop-common.version}" conf="hadoop2->master">
+      rev="${hadoop-common.version}" conf="hadoop23->master">
       <artifact name="hadoop-common" ext="jar" />
       <artifact name="hadoop-common" type="tests" ext="jar" m:classifier="tests" />
     </dependency>
     <dependency org="org.apache.hadoop" name="hadoop-hdfs"
-      rev="${hadoop-hdfs.version}" conf="hadoop2->master">
+      rev="${hadoop-hdfs.version}" conf="hadoop23->master">
       <artifact name="hadoop-hdfs" ext="jar" />
       <artifact name="hadoop-hdfs" type="tests" ext="jar" m:classifier="tests" />
     </dependency>
     <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core" rev="${hadoop-mapreduce.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-jobclient" rev="${hadoop-mapreduce.version}"
-      conf="hadoop2->master">
+      conf="hadoop23->master">
         <artifact name="hadoop-mapreduce-client-jobclient" ext="jar" />
         <artifact name="hadoop-mapreduce-client-jobclient" type="tests" ext="jar" m:classifier="tests"/>
         <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
         <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
     </dependency>
     <dependency org="org.apache.hadoop" name="hadoop-yarn-server-tests" rev="${hadoop-mapreduce.version}"
-      conf="hadoop2->master">
+      conf="hadoop23->master">
       <artifact name="hadoop-yarn-server-tests" type="jar" m:classifier="tests"/>
     </dependency>
     <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-app" rev="${hadoop-mapreduce.version}"
-      conf="hadoop2->master" />
+      conf="hadoop23->master" />
     <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-shuffle" rev="${hadoop-mapreduce.version}"
-      conf="hadoop2->master" />
+      conf="hadoop23->master" />
     <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-common" 
-      rev="${hadoop-mapreduce.version}" conf="hadoop2->master"/>
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-yarn-api" 
-      rev="${hadoop-mapreduce.version}" conf="hadoop2->master"/>
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-yarn-common" 
-      rev="${hadoop-mapreduce.version}" conf="hadoop2->master"/>
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-yarn-server" 
-      rev="${hadoop-mapreduce.version}" conf="hadoop2->master"/>
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-yarn-server-web-proxy" 
-      rev="${hadoop-mapreduce.version}" conf="hadoop2->master"/>
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-yarn-server-common" 
-      rev="${hadoop-mapreduce.version}" conf="hadoop2->master"/>
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-yarn-server-nodemanager" 
-      rev="${hadoop-mapreduce.version}" conf="hadoop2->master"/>
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-yarn-server-resourcemanager" 
-      rev="${hadoop-mapreduce.version}" conf="hadoop2->master"/>
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-yarn-client" 
-      rev="${hadoop-mapreduce.version}" conf="hadoop2->master"/>
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-yarn-server-applicationhistoryservice" 
-      rev="${hadoop-mapreduce.version}" conf="hadoop2->master"/>
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-hs" 
-      rev="${hadoop-mapreduce.version}" conf="hadoop2->master"/>
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
     <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}"
       conf="compile->master">
       <artifact name="jetty" ext="jar" />
@@ -185,7 +193,13 @@
       <exclude org="org.codehaus.jackson" module="jackson-mapper-asl"/>
     </dependency>
     <dependency org="org.apache.avro" name="avro-mapred" rev="${avro.version}"
-      conf="hadoop2->default;checkstyle->master">
+      conf="hadoop20->default;checkstyle->master">
+      <exclude org="org.codehaus.jackson" module="jackson-core-asl"/>
+      <exclude org="org.codehaus.jackson" module="jackson-mapper-asl"/>
+      <exclude org="io.netty" module="netty"/>
+    </dependency>
+    <dependency org="org.apache.avro" name="avro-mapred" rev="${avro.version}"
+      conf="hadoop23->default;checkstyle->master">
       <artifact name="avro-mapred" type="jar" m:classifier="hadoop2"/>
       <exclude org="org.codehaus.jackson" module="jackson-core-asl"/>
       <exclude org="org.codehaus.jackson" module="jackson-mapper-asl"/>
@@ -247,14 +261,37 @@
     <dependency org="org.antlr" name="ST4" rev="${stringtemplate.version}" conf="compile->default"/>
     <dependency org="org.apache.zookeeper" name="zookeeper" rev="${zookeeper.version}" conf="compile->master"/>
     <dependency org="io.netty" name="netty" rev="${netty.version}" conf="test->master"/>
-    <dependency org="io.netty" name="netty-all" rev="${netty-all.version}" conf="test->master" />
     <dependency org="dk.brics.automaton" name="automaton" rev="1.11-8" conf="compile->default"/>
 
     <dependency org="org.jruby" name="jruby-complete" rev="${jruby.version}" conf="compile->master"/>
     <dependency org="asm" name="asm" rev="${asm.version}" conf="compile->default"/>
 
+    <!-- HBase dependency in format for releases up to 0.94 (including) -->
+    <dependency org="org.apache.hbase" name="hbase" rev="${hbase94.version}" conf="hbase94->master">
+      <artifact name="hbase" type="jar"/>
+      <artifact name="hbase" type="test-jar" ext="jar" m:classifier="tests"/>
+      <exclude org="org.apache.thrift" module="thrift"/>
+      <exclude org="org.apache.hadoop" module="hadoop-core"/>
+      <exclude org="org.apache.ant" module="ant" />
+      <exclude org="org.slf4j" module="slf4j"/>
+      <exclude org="org.slf4j" module="slf4j-api"/>
+      <exclude org="org.slf4j" module="slf4j-log4j12" />
+      <exclude org="org.slf4j" module="log4j12"/>
+      <exclude org="org.slf4j" module="log4j-over-slf4j"/>
+      <exclude org="stax" module="stax-api" />
+      <exclude org="javax.xml.bind" module="jaxb-api" />
+      <exclude org="javax.ws.rs" module="jsr311-api" />
+      <exclude org="tomcat" module="jasper-runtime"/>
+      <exclude org="tomcat" module="jasper-compiler"/>
+      <exclude org="com.google.protobuf" module="protobuf-java"/>
+      <exclude org="com.sun.jersey" module="jersey-core"/>
+      <exclude org="com.sun.jersey" module="jersey-server"/>
+      <exclude org="com.sun.jersey" module="jersey-json"/>
+      <exclude org="asm" module="asm"/>
+    </dependency>
+
     <!-- HBase dependency in format for releases higher or equal to 0.95 -->
-    <dependency org="org.apache.hbase" name="hbase-client" rev="${hbase1.version}" conf="hbase1->master">
+    <dependency org="org.apache.hbase" name="hbase-client" rev="${hbase95.version}" conf="hbase95->master">
       <artifact name="hbase-client" type="jar"/>
       <artifact name="hbase-client" type="test-jar" ext="jar" m:classifier="tests"/>
       <exclude org="org.slf4j" module="slf4j-api"/>
@@ -270,7 +307,7 @@
       <exclude org="asm" module="asm"/>
     </dependency>
 
-    <dependency org="org.apache.hbase" name="hbase-common" rev="${hbase1.version}" conf="hbase1->master">
+    <dependency org="org.apache.hbase" name="hbase-common" rev="${hbase95.version}" conf="hbase95->master">
       <artifact name="hbase-common" type="jar"/>
       <artifact name="hbase-common" type="test-jar" ext="jar" m:classifier="tests"/>
       <exclude org="org.apache.hadoop" module="hadoop-core"/>
@@ -285,7 +322,7 @@
       <exclude org="asm" module="asm"/>
     </dependency>
 
-    <dependency org="org.apache.hbase" name="hbase-server" rev="${hbase1.version}" conf="hbase1->master">
+    <dependency org="org.apache.hbase" name="hbase-server" rev="${hbase95.version}" conf="hbase95->master">
       <artifact name="hbase-server" type="jar"/>
       <artifact name="hbase-server" type="test-jar" ext="jar" m:classifier="tests"/>
       <exclude org="org.apache.hadoop" module="hadoop-core"/>
@@ -302,20 +339,20 @@
       <exclude org="asm" module="asm"/>
     </dependency>
 
-    <dependency org="org.apache.hbase" name="hbase-protocol" rev="${hbase1.version}" conf="hbase1->master">
+    <dependency org="org.apache.hbase" name="hbase-protocol" rev="${hbase95.version}" conf="hbase95->master">
       <artifact name="hbase-protocol" type="jar"/>
       <artifact name="hbase-protocol" type="test-jar" ext="jar" m:classifier="tests"/>
       <exclude org="com.google.protobuf" module="protobuf-java"/>
     </dependency>
 
-    <dependency org="org.apache.hbase" name="hbase-hadoop-compat" rev="${hbase1.version}" conf="hbase1->master">
+    <dependency org="org.apache.hbase" name="hbase-hadoop-compat" rev="${hbase95.version}" conf="hbase95->master">
       <artifact name="hbase-hadoop-compat" type="jar"/>
       <artifact name="hbase-hadoop-compat" type="test-jar" ext="jar" m:classifier="tests"/>
     </dependency>
 
-    <dependency org="org.apache.hbase" name="hbase-hadoop2-compat" rev="${hbase1.version}" conf="hbase1->master">
-      <artifact name="hbase-hadoop2-compat" type="jar"/>
-      <artifact name="hbase-hadoop2-compat" type="test-jar" ext="jar" m:classifier="tests"/>
+    <dependency org="org.apache.hbase" name="hbase-${hbase.hadoop.version}-compat" rev="${hbase95.version}" conf="hbase95->master">
+      <artifact name="hbase-${hbase.hadoop.version}-compat" type="jar"/>
+      <artifact name="hbase-${hbase.hadoop.version}-compat" type="test-jar" ext="jar" m:classifier="tests"/>
       <exclude org="org.apache.hadoop" module="hadoop-core"/>
       <exclude org="org.slf4j" module="slf4j-api"/>
       <exclude org="stax" module="stax-api" />
@@ -328,17 +365,20 @@
       <exclude org="asm" module="asm"/>
     </dependency>
 
-    <dependency org="org.htrace" name="htrace-core" rev="3.0.4" conf="hadoop2->master"/>
-    <dependency org="org.apache.htrace" name="htrace-core" rev="${htrace.version}" conf="hadoop2->master"/>
+    <dependency org="org.htrace" name="htrace-core" rev="3.0.4" conf="hadoop23->master"/>
+    <dependency org="org.apache.htrace" name="htrace-core" rev="${htrace.version}" conf="hadoop23->master"/>
     <dependency org="org.fusesource.leveldbjni" name="leveldbjni-all" rev="${leveldbjni.version}"
-      conf="hadoop2->master"/>
-    <dependency org="org.cloudera.htrace" name="htrace-core" rev="2.00" conf="hbase1->master">
+      conf="hadoop23->master"/>
+    <dependency org="org.cloudera.htrace" name="htrace-core" rev="2.00" conf="hbase95->master">
       <artifact name="htrace-core" type="jar"/>
     </dependency>
-    <dependency org="com.lmax" name="disruptor" rev="3.3.0" conf="hbase1->master"/>
+    <dependency org="com.lmax" name="disruptor" rev="3.3.0" conf="hbase95->master"/>
 
     <!-- for TestHBaseStorage -->
-    <dependency org="org.apache.hbase" name="hbase-procedure" rev="${hbase1.version}" conf="test->master"/>
+    <dependency org="com.github.stephenc.high-scale-lib" name="high-scale-lib" rev="${high-scale-lib.version}"
+       conf="test->default"/>
+    <dependency org="com.google.protobuf" name="protobuf-java" rev="${protobuf-java.version}"
+       conf="test->default"/>
     <dependency org="com.yammer.metrics" name="metrics-core" rev="${metrics-core.version}"
        conf="test->default"/>
 
@@ -388,10 +428,10 @@
       conf="compile->master" />
     <dependency org="org.apache.hive.shims" name="hive-shims-common" rev="${hive.version}" changing="true"
       conf="compile->master" />
-    <dependency org="org.apache.hive" name="hive-contrib" rev="${hive.version}" changing="true"
-                conf="test->master" />
     <dependency org="org.apache.hive.shims" name="hive-shims-0.23" rev="${hive.version}" changing="true"
-      conf="hadoop2->master" />
+      conf="hadoop23->master" />
+    <dependency org="org.apache.hive.shims" name="hive-shims-0.20S" rev="${hive.version}" changing="true"
+      conf="hadoop20->master" />
     <dependency org="org.iq80.snappy" name="snappy" rev="${snappy.version}"
       conf="test->master" />
     <dependency org="com.esotericsoftware.kryo" name="kryo" rev="${kryo.version}"
@@ -426,31 +466,31 @@
 
     <!-- for Tez integration -->
     <dependency org="org.apache.tez" name="tez" rev="${tez.version}"
-       conf="hadoop2->master"/>
+       conf="hadoop23->master"/>
     <dependency org="org.apache.tez" name="tez-common" rev="${tez.version}"
-       conf="hadoop2->master"/>
+       conf="hadoop23->master"/>
     <dependency org="org.apache.tez" name="tez-api" rev="${tez.version}"
-       conf="hadoop2->master"/>
+       conf="hadoop23->master"/>
     <dependency org="org.apache.tez" name="tez-dag" rev="${tez.version}"
-       conf="hadoop2->master"/>
+       conf="hadoop23->master"/>
     <dependency org="org.apache.tez" name="tez-runtime-internals" rev="${tez.version}"
-       conf="hadoop2->master"/>
+       conf="hadoop23->master"/>
     <dependency org="org.apache.tez" name="tez-runtime-library" rev="${tez.version}"
-       conf="hadoop2->master"/>
+       conf="hadoop23->master"/>
     <dependency org="org.apache.tez" name="tez-mapreduce" rev="${tez.version}"
-       conf="hadoop2->master"/>
+       conf="hadoop23->master"/>
     <dependency org="org.apache.tez" name="tez-yarn-timeline-history-with-acls" rev="${tez.version}"
-       conf="hadoop2->master"/>
+       conf="hadoop23->master"/>
     <dependency org="org.apache.commons" name="commons-collections4" rev="${commons-collections4.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="org.codehaus.jettison" name="jettison" rev="${jettison.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="org.apache.commons" name="commons-math3" rev="${commons-math3.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="org.apache.curator" name="curator-framework" rev="${curator.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
     <dependency org="org.apache.curator" name="curator-client" rev="${curator.version}"
-      conf="hadoop2->master"/>
+      conf="hadoop23->master"/>
   </dependencies>
 </ivy-module>
 

Modified: pig/branches/spark/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/pig/branches/spark/ivy/libraries.properties?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/ivy/libraries.properties (original)
+++ pig/branches/spark/ivy/libraries.properties Fri Feb 24 03:34:37 2017
@@ -40,10 +40,14 @@ ivy.version=2.2.0
 jasper.version=6.1.14
 groovy.version=2.4.5
 guava.version=11.0
-hadoop-common.version=2.7.3
-hadoop-hdfs.version=2.7.3
-hadoop-mapreduce.version=2.7.3
-hbase1.version=1.2.4
+jersey-core.version=1.8
+hadoop-core.version=1.0.4
+hadoop-test.version=1.0.4
+hadoop-common.version=2.6.0
+hadoop-hdfs.version=2.6.0
+hadoop-mapreduce.version=2.6.0
+hbase94.version=0.94.1
+hbase95.version=0.98.12-${hbase.hadoop.version}
 hsqldb.version=1.8.0.10
 hive.version=1.2.1
 httpcomponents.version=4.1
@@ -57,20 +61,19 @@ jdeb.version=0.8
 jdiff.version=1.0.9
 jettison.version=1.3.4
 jetty.version=6.1.26
-jline.version=2.11
-joda-time.version=2.9.3
+jline.version=1.0
+joda-time.version=2.8.2
 jopt.version=4.1
 json-simple.version=1.1
 junit.version=4.11
 jruby.version=1.6.7
-jython.version=2.7.0
-kryo.version=2.22
+jython.version=2.5.3
+kryo.version=2.21
 rhino.version=1.7R2
 antlr.version=3.4
 stringtemplate.version=4.0.4
 log4j.version=1.2.16
 netty.version=3.6.6.Final
-netty-all.version=4.0.23.Final
 rats-lib.version=0.5.1
 slf4j-api.version=1.6.1
 slf4j-log4j12.version=1.6.1
@@ -82,6 +85,7 @@ zookeeper.version=3.4.5
 servlet.version=4.0.6
 servlet-api.version=2.5
 protobuf-java.version=2.5.0
+high-scale-lib.version=1.1.1
 metrics-core.version=2.1.2
 guice.version=3.0
 guice-servlet.version=3.0

Modified: pig/branches/spark/ivy/pig-template.xml
URL: http://svn.apache.org/viewvc/pig/branches/spark/ivy/pig-template.xml?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/ivy/pig-template.xml (original)
+++ pig/branches/spark/ivy/pig-template.xml Fri Feb 24 03:34:37 2017
@@ -46,7 +46,7 @@
     <dependency>
       <groupId>commons-net</groupId>
       <artifactId>commons-net</artifactId>
-      <version>3.1</version>
+      <version>1.4.1</version>
     </dependency>
     <dependency>
       <groupId>org.mortbay.jetty</groupId>
@@ -59,6 +59,16 @@
       <version>6.1.26</version>
     </dependency>
     <dependency>
+      <groupId>tomcat</groupId>
+      <artifactId>jasper-runtime</artifactId>
+      <version>5.5.12</version>
+    </dependency>
+    <dependency>
+      <groupId>tomcat</groupId>
+      <artifactId>jasper-compiler</artifactId>
+      <version>5.5.12</version>
+    </dependency>
+    <dependency>
       <groupId>org.mortbay.jetty</groupId>
       <artifactId>jsp-api-2.1</artifactId>
       <version>6.1.14</version>
@@ -76,17 +86,27 @@
     <dependency>
       <groupId>net.java.dev.jets3t</groupId>
       <artifactId>jets3t</artifactId>
-      <version>0.9.0</version>
+      <version>0.7.1</version>
     </dependency>
     <dependency>
-      <groupId>javax.servlet</groupId>
-      <artifactId>servlet-api</artifactId>
-      <version>2.5</version>
+      <groupId>commons-net</groupId>
+      <artifactId>commons-net</artifactId>
+      <version>1.4.1</version>
+    </dependency>
+    <dependency>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>servlet-api-2.5</artifactId>
+      <version>6.1.14</version>
+    </dependency>
+    <dependency>
+      <groupId>net.sf.kosmosfs</groupId>
+      <artifactId>kfs</artifactId>
+      <version>0.3</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
-      <version>4.11</version>
+      <version>4.8.1</version>
     </dependency>
     <dependency>
       <groupId>hsqldb</groupId>
@@ -94,9 +114,14 @@
       <version>1.8.0.10</version>
     </dependency>
     <dependency>
+      <groupId>oro</groupId>
+      <artifactId>oro</artifactId>
+      <version>2.0.8</version>
+    </dependency>
+    <dependency>
       <groupId>jline</groupId>
       <artifactId>jline</artifactId>
-      <version>2.11</version>
+      <version>1.0</version>
     </dependency>
     <dependency>
       <groupId>org.antlr</groupId>
@@ -121,22 +146,22 @@
     <dependency>
       <groupId>org.codehaus.jackson</groupId>
       <artifactId>jackson-mapper-asl</artifactId>
-      <version>1.9.13</version>
+      <version>1.8.8</version>
     </dependency>
     <dependency>
       <groupId>org.codehaus.jackson</groupId>
       <artifactId>jackson-core-asl</artifactId>
-      <version>1.9.13</version>
+      <version>1.8.8</version>
     </dependency>
     <dependency>
       <groupId>joda-time</groupId>
       <artifactId>joda-time</artifactId>
-      <version>2.9.3</version>
+      <version>2.1</version>
     </dependency>
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>
-      <version>1.7.5</version>
+      <version>1.7.4</version>
       <exclusions>
         <exclusion>
           <!-- Don't pull in Avro's (later) version of Jetty.-->
@@ -158,7 +183,7 @@
     <dependency>
       <groupId>org.codehaus.groovy</groupId>
       <artifactId>groovy-all</artifactId>
-      <version>2.4.5</version>
+      <version>1.8.6</version>
     </dependency>
   </dependencies>
 </project>

Modified: pig/branches/spark/ivy/piggybank-template.xml
URL: http://svn.apache.org/viewvc/pig/branches/spark/ivy/piggybank-template.xml?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/ivy/piggybank-template.xml (original)
+++ pig/branches/spark/ivy/piggybank-template.xml Fri Feb 24 03:34:37 2017
@@ -51,7 +51,7 @@
     <dependency>
       <groupId>joda-time</groupId>
       <artifactId>joda-time</artifactId>
-      <version>2.9.3</version>
+      <version>2.1</version>
     </dependency>
     <dependency>
       <groupId>com.googlecode.json-simple</groupId>
@@ -61,7 +61,7 @@
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro</artifactId>
-      <version>1.7.5</version>
+      <version>1.7.4</version>
       <exclusions>
         <exclusion>
           <!-- Don't pull in Avro's (later) version of Jetty.-->
@@ -78,7 +78,7 @@
     <dependency>
       <groupId>nl.basjes.parse.httpdlog</groupId>
       <artifactId>httpdlog-pigloader</artifactId>
-      <version>2.4</version>
+      <version>2.2</version>
     </dependency>
     <dependency>
       <groupId>org.apache.pig</groupId>

Modified: pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType?rev=1784224&r1=1784223&r2=1784224&view=diff
==============================================================================
--- pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType (original)
+++ pig/branches/spark/src/META-INF/services/org.apache.pig.ExecType Fri Feb 24 03:34:37 2017
@@ -13,7 +13,7 @@
 #
 org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LocalExecType
 org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRExecType
-org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType
-org.apache.pig.backend.hadoop.executionengine.tez.TezExecType
+#org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType
+#org.apache.pig.backend.hadoop.executionengine.tez.TezExecType
 org.apache.pig.backend.hadoop.executionengine.spark.SparkExecType
 org.apache.pig.backend.hadoop.executionengine.spark.SparkLocalExecType



Mime
View raw message