pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sz...@apache.org
Subject svn commit: r1820631 - in /pig/trunk: CHANGES.txt src/org/apache/pig/newplan/logical/relational/LOStore.java test/org/apache/pig/test/TestSchema.java
Date Tue, 09 Jan 2018 10:03:07 GMT
Author: szita
Date: Tue Jan  9 10:03:07 2018
New Revision: 1820631

URL: http://svn.apache.org/viewvc?rev=1820631&view=rev
Log:
PIG-5325: Schema disambiguation can't be turned off for nested schemas (szita)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java
    pig/trunk/test/org/apache/pig/test/TestSchema.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1820631&r1=1820630&r2=1820631&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jan  9 10:03:07 2018
@@ -64,6 +64,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-5325: Schema disambiguation can't be turned off for nested schemas (szita)
+
 PIG-5311: POReservoirSample fails for more than Integer.MAX_VALUE records (rohini)
 
 PIG-3864: ToDate(userstring, format, timezone) computes DateTime with strange handling of
Daylight Saving Time with location based timezones (daijy via rohini)

Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java?rev=1820631&r1=1820630&r2=1820631&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStore.java Tue Jan  9 10:03:07
2018
@@ -63,16 +63,31 @@ public class LOStore extends LogicalRela
     public LogicalSchema getSchema() throws FrontendException {
         schema = ((LogicalRelationalOperator)plan.getPredecessors(this).get(0)).getSchema();
 
-        if (!disambiguationEnabled && schema != null && schema.getFields()
!= null) {
+        if (!disambiguationEnabled) {
             //If requested try and remove parent alias substring including colon(s)
+            removeDisambiguation(schema);
+        }
+
+        return schema;
+    }
+
+    /**
+     * Removes schema disambiguation parts (parent alias and :) from field aliases
+     * @param schema
+     * @return
+     */
+    private static LogicalSchema removeDisambiguation(LogicalSchema schema) {
+        if (schema != null && schema.getFields() != null) {
             for (LogicalSchema.LogicalFieldSchema field : schema.getFields()) {
+                if (field.schema != null) {
+                    removeDisambiguation(field.schema);
+                }
                 if (field.alias == null || !field.alias.contains(":")) {
                     continue;
                 }
                 field.alias = field.alias.substring(field.alias.lastIndexOf(":") + 1);
             }
         }
-
         return schema;
     }
 

Modified: pig/trunk/test/org/apache/pig/test/TestSchema.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSchema.java?rev=1820631&r1=1820630&r2=1820631&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSchema.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSchema.java Tue Jan  9 10:03:07 2018
@@ -990,6 +990,33 @@ public class TestSchema {
     }
 
     @Test
+    public void testDisabledDisambiguationContainsNoColonsForNestedSchema() throws IOException
{
+        resetDisambiguationTestPropertyOverride();
+
+        String inputFileName = "testPrepend-nested-input.txt";
+        String[] inputData = new String[]{"apple\t1\tred", "orange\t2\torange", "kiwi\t3\tgreen",
"orange\t4\torange"};
+        Util.createInputFile(cluster, inputFileName, inputData);
+
+        String script = "A = LOAD '" + inputFileName + "' AS (fruit:chararray, foo:int, color:
chararray);" +
+                "B = LOAD '" + inputFileName + "' AS (id:chararray, bar:int);" +
+                "C = JOIN A by fruit, B by id;" +
+                "D = GROUP C by fruit;" +
+                "E = LOAD '" + inputFileName + "' AS (name:chararray, qwe:int);" +
+                "F = JOIN E by name, D by group;";
+
+        Util.registerMultiLineQuery(pigServer, script);
+
+        //Prepending should happen with default settings
+        assertEquals("{E::name: chararray,E::qwe: int,D::group: chararray,D::C: {(A::fruit:
chararray,A::foo: int,A::color: chararray,B::id: chararray,B::bar: int)}}", pigServer.dumpSchema("F").toString());
+
+        //Override prepend property setting (check for flatten, join)
+        pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_STORE_SCHEMA_DISAMBIGUATE,
"false");
+        assertEquals("{name: chararray,qwe: int,group: chararray,C: {(fruit: chararray,foo:
int,color: chararray,id:" +
+                " chararray,bar: int)}}", pigServer.dumpSchema("F").toString());
+        assertTrue(pigServer.openIterator("F").hasNext());
+    }
+
+    @Test
     public void testEnabledDisambiguationPassesForDupeAliases() throws IOException {
         resetDisambiguationTestPropertyOverride();
 



Mime
View raw message