pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pradeep...@apache.org
Subject svn commit: r904607 - in /hadoop/pig/branches/load-store-redesign/src/org/apache/pig: StoreFunc.java backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java impl/logicalLayer/LOStore.java impl/logicalLayer/parser/QueryParser.jjt
Date Fri, 29 Jan 2010 19:40:18 GMT
Author: pradeepkth
Date: Fri Jan 29 19:40:17 2010
New Revision: 904607

URL: http://svn.apache.org/viewvc?rev=904607&view=rev
Log:
PIG-1090: additional commit to enable storeFunc interacting with metadata systems (daijy via
pradeepkth)

Modified:
    hadoop/pig/branches/load-store-redesign/src/org/apache/pig/StoreFunc.java
    hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
    hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOStore.java
    hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt

Modified: hadoop/pig/branches/load-store-redesign/src/org/apache/pig/StoreFunc.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/StoreFunc.java?rev=904607&r1=904606&r2=904607&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/org/apache/pig/StoreFunc.java (original)
+++ hadoop/pig/branches/load-store-redesign/src/org/apache/pig/StoreFunc.java Fri Jan 29 19:40:17
2010
@@ -73,7 +73,12 @@
      * <b>store A into 'bla'</b>
      * then 'bla' is the location.  This location should be either a file name
      * or a URI.  If it does not have a URI scheme Pig will assume it is a 
-     * filename.  This will be called multiple times during execution on the backend.
+     * filename.  
+     * This method will be called in the frontend and backend multiple times. Implementations
+     * should bear in mind that this method is called multiple times and should
+     * ensure there are no inconsistent side effects due to the multiple calls.
+     * 
+
      * @param location Location indicated in store statement.
      * @param job The {@link Job} object
      * @throws IOException if the location is not valid.

Modified: hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=904607&r1=904606&r2=904607&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
(original)
+++ hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Fri Jan 29 19:40:17 2010
@@ -446,6 +446,7 @@
                 StoreFunc sFunc = st.getStoreFunc();
                 if (st.getSchema()!=null)
                     sFunc.checkSchema(new ResourceSchema(st.getSchema(), st.getSortInfo()));
+                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
             }
 
             for (POStore st: reduceStores) {
@@ -453,6 +454,7 @@
                 StoreFunc sFunc = st.getStoreFunc();
                 if (st.getSchema()!=null)
                     sFunc.checkSchema(new ResourceSchema(st.getSchema(), st.getSortInfo()));
+                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
             }
 
             // the OutputFormat we report to Hadoop is always PigOutputFormat

Modified: hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOStore.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOStore.java?rev=904607&r1=904606&r2=904607&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOStore.java
(original)
+++ hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/LOStore.java
Fri Jan 29 19:40:17 2010
@@ -24,6 +24,7 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.pig.FuncSpec;
 import org.apache.pig.StoreFunc; 
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.io.FileSpec;
@@ -74,7 +75,7 @@
         try { 
              mStoreFunc = (StoreFunc) PigContext.instantiateFuncFromSpec(outputFileSpec.getFuncSpec());
              this.mAlias = alias;
-             this.signature = constructSignature(mAlias, mOutputFile.getFileName(), mOutputFile.getFuncSpec().getCtorArgs());
+             this.signature = constructSignature(mAlias, outputFileSpec.getFileName(), mOutputFile.getFuncSpec());
              mStoreFunc.setStoreFuncUDFContextSignature(this.signature);
         } catch (Exception e) { 
             IOException ioe = new IOException(e.getMessage()); 
@@ -83,16 +84,8 @@
         }
     }
     
-    private String constructSignature(String alias, String filename, String[] args) {
-        String s = alias+"_"+filename+"_";
-        if (args!=null) {
-            for (int i=0;i<args.length;i++) {
-                s = s+args[i];
-                if (i!=args.length-1)
-                    s = s+"_";
-            }
-        }
-        return s;
+    public static String constructSignature(String alias, String filename, FuncSpec funcSpec)
{
+        return alias+"_"+filename+"_"+funcSpec.toString();
     }
 
     public FileSpec getOutputFile() {
@@ -219,9 +212,8 @@
     @Override
     public void setAlias(String newAlias) {
         super.setAlias(newAlias);
-        mStoreFunc.setStoreFuncUDFContextSignature(getAlias()+"_"+mOutputFile.getFileName());
-        signature = constructSignature(mAlias, mOutputFile.getFileName(), mOutputFile.getFuncSpec().getCtorArgs());
-        signature = getAlias()+"_"+mOutputFile.getFileName();
+        signature = constructSignature(mAlias, mOutputFile.getFileName(), mOutputFile.getFuncSpec());
+        mStoreFunc.setStoreFuncUDFContextSignature(signature);
     }
     
     public String getSignature() {

Modified: hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=904607&r1=904606&r2=904607&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
(original)
+++ hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
Fri Jan 29 19:40:17 2010
@@ -619,7 +619,10 @@
         }
         return true;    
     }
-
+    
+    static String constructFileNameSignature(String fileName, FuncSpec funcSpec) {
+        return fileName+"_"+funcSpec.toString();
+    }
 }
 
 
@@ -1270,10 +1273,10 @@
             // so translating 'foo' to its absolute path based on that dir would give incorrect
             // results - hence we store the translations into a map during the first parse
for
             // use during the reparse.
-            String absolutePath = fileNameMap.get(filename);
+            String absolutePath = fileNameMap.get(constructFileNameSignature(filename, funcSpec));
             if (absolutePath == null) {
                 absolutePath = loFunc.relativeToAbsolutePath(filename, getCurrentDir(pigContext));
-                fileNameMap.put(filename, absolutePath);
+                fileNameMap.put(constructFileNameSignature(filename, funcSpec), absolutePath);
             }
             lo = new LOLoad(lp, new OperatorKey(scope, getNextId()), new FileSpec(absolutePath,
funcSpec),
                   ConfigurationUtil.toConfiguration(pigContext.getProperties()));
@@ -2411,12 +2414,13 @@
 
         Object obj = PigContext.instantiateFuncFromSpec(funcSpec);
         StoreFunc stoFunc = (StoreFunc)obj;
+        stoFunc.setStoreFuncUDFContextSignature(LOStore.constructSignature(t.image, fileName,
funcSpec));
          
         // see the comments in LoadClause for reasons to cache absolutePath
-        String absolutePath = fileNameMap.get(fileName);
+        String absolutePath = fileNameMap.get(constructFileNameSignature(fileName, funcSpec));
         if (absolutePath == null) {
             absolutePath = stoFunc.relToAbsPathForStoreLocation(fileName, getCurrentDir(pigContext));
-            fileNameMap.put(fileName, absolutePath);   
+            fileNameMap.put(constructFileNameSignature(fileName, funcSpec), absolutePath);
  
         }
         LogicalOperator store = new LOStore(lp, new OperatorKey(scope, getNextId()),
                                             new FileSpec(absolutePath, funcSpec), t.image);
@@ -3954,4 +3958,4 @@
 		log.trace("Exiting AliasFieldOrSpec");
 		return item;
 	}
-}
+}
\ No newline at end of file



Mime
View raw message