gora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alfonsonishik...@apache.org
Subject svn commit: r1455623 - in /gora/branches/GORA_174/gora-core/src: examples/avro/ examples/java/org/apache/gora/examples/ examples/java/org/apache/gora/examples/generated/ examples/java/org/apache/gora/examples/mapreduce/ main/java/org/apache/gora/compil...
Date Tue, 12 Mar 2013 17:07:03 GMT
Author: alfonsonishikawa
Date: Tue Mar 12 17:07:03 2013
New Revision: 1455623

URL: http://svn.apache.org/r1455623
Log:
Applied working patch GORA-174 [0] version 12/Mar/13 17:55

[0] https://issues.apache.org/jira/browse/GORA-174

Modified:
    gora/branches/GORA_174/gora-core/src/examples/avro/webpage.json
    gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/WebPageDataCreator.java
    gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java
    gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java
    gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java
    gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java
    gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/WordCount.java
    gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/compiler/GoraCompiler.java
    gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java
    gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
    gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java

Modified: gora/branches/GORA_174/gora-core/src/examples/avro/webpage.json
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/avro/webpage.json?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/avro/webpage.json (original)
+++ gora/branches/GORA_174/gora-core/src/examples/avro/webpage.json Tue Mar 12 17:07:03 2013
@@ -4,7 +4,7 @@
   "namespace": "org.apache.gora.examples.generated",
   "fields" : [
     {"name": "url", "type": "string"},
-    {"name": "content", "type": "bytes"},
+    {"name": "content", "type": ["null","bytes"]},
     {"name": "parsedContent", "type": {"type":"array", "items": "string"}},
     {"name": "outlinks", "type": {"type":"map", "values":"string"}},
     {"name": "metadata", "type": {

Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/WebPageDataCreator.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/WebPageDataCreator.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/WebPageDataCreator.java
(original)
+++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/WebPageDataCreator.java
Tue Mar 12 17:07:03 2013
@@ -61,7 +61,7 @@ public class WebPageDataCreator {
   }
   
   public static final String[] CONTENTS = {
-    "foo baz bar",
+    null,
     "foo",
     "foo1 bar1 baz1",
     "a b c d e",
@@ -116,11 +116,12 @@ public class WebPageDataCreator {
 	    for(int i=0; i<URLS.length; i++) {
 	      page = new WebPage();
 	      page.setUrl(new Utf8(URLS[i]));
-	      page.setContent(ByteBuffer.wrap(CONTENTS[i].getBytes()));
-	      for(String token : CONTENTS[i].split(" ")) {
-	        page.addToParsedContent(new Utf8(token));  
-	      }
-	      
+	      if (CONTENTS[i]!=null){
+	        page.setContent(ByteBuffer.wrap(CONTENTS[i].getBytes()));
+	        for(String token : CONTENTS[i].split(" ")) {
+	    	  page.addToParsedContent(new Utf8(token));  
+	        }
+          }
 	      for(int j=0; j<LINKS[i].length; j++) {
 	        page.putToOutlinks(new Utf8(URLS[LINKS[i][j]]), new Utf8(ANCHORS[i][j]));
 	      }

Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java
(original)
+++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Employee.java
Tue Mar 12 17:07:03 2013
@@ -1,3 +1,21 @@
+/**
+ *Licensed to the Apache Software Foundation (ASF) under one
+ *or more contributor license agreements.  See the NOTICE file
+ *distributed with this work for additional information
+ *regarding copyright ownership.  The ASF licenses this file
+ *to you under the Apache License, Version 2.0 (the"
+ *License"); you may not use this file except in compliance
+ *with the License.  You may obtain a copy of the License at
+ *
+  * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *Unless required by applicable law or agreed to in writing, software
+ *distributed under the License is distributed on an "AS IS" BASIS,
+ *WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *See the License for the specific language governing permissions and
+ *limitations under the License.
+ */
+
 package org.apache.gora.examples.generated;
 
 import java.nio.ByteBuffer;
@@ -10,6 +28,7 @@ import org.apache.avro.Protocol;
 import org.apache.avro.util.Utf8;
 import org.apache.avro.ipc.AvroRemoteException;
 import org.apache.avro.generic.GenericArray;
+import org.apache.avro.specific.FixedSize;
 import org.apache.avro.specific.SpecificExceptionBase;
 import org.apache.avro.specific.SpecificRecordBase;
 import org.apache.avro.specific.SpecificRecord;

Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java
(original)
+++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/Metadata.java
Tue Mar 12 17:07:03 2013
@@ -1,3 +1,21 @@
+/**
+ *Licensed to the Apache Software Foundation (ASF) under one
+ *or more contributor license agreements.  See the NOTICE file
+ *distributed with this work for additional information
+ *regarding copyright ownership.  The ASF licenses this file
+ *to you under the Apache License, Version 2.0 (the"
+ *License"); you may not use this file except in compliance
+ *with the License.  You may obtain a copy of the License at
+ *
+  * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *Unless required by applicable law or agreed to in writing, software
+ *distributed under the License is distributed on an "AS IS" BASIS,
+ *WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *See the License for the specific language governing permissions and
+ *limitations under the License.
+ */
+
 package org.apache.gora.examples.generated;
 
 import java.nio.ByteBuffer;
@@ -10,6 +28,7 @@ import org.apache.avro.Protocol;
 import org.apache.avro.util.Utf8;
 import org.apache.avro.ipc.AvroRemoteException;
 import org.apache.avro.generic.GenericArray;
+import org.apache.avro.specific.FixedSize;
 import org.apache.avro.specific.SpecificExceptionBase;
 import org.apache.avro.specific.SpecificRecordBase;
 import org.apache.avro.specific.SpecificRecord;

Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java
(original)
+++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/TokenDatum.java
Tue Mar 12 17:07:03 2013
@@ -1,3 +1,21 @@
+/**
+ *Licensed to the Apache Software Foundation (ASF) under one
+ *or more contributor license agreements.  See the NOTICE file
+ *distributed with this work for additional information
+ *regarding copyright ownership.  The ASF licenses this file
+ *to you under the Apache License, Version 2.0 (the"
+ *License"); you may not use this file except in compliance
+ *with the License.  You may obtain a copy of the License at
+ *
+  * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *Unless required by applicable law or agreed to in writing, software
+ *distributed under the License is distributed on an "AS IS" BASIS,
+ *WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *See the License for the specific language governing permissions and
+ *limitations under the License.
+ */
+
 package org.apache.gora.examples.generated;
 
 import java.nio.ByteBuffer;
@@ -10,6 +28,7 @@ import org.apache.avro.Protocol;
 import org.apache.avro.util.Utf8;
 import org.apache.avro.ipc.AvroRemoteException;
 import org.apache.avro.generic.GenericArray;
+import org.apache.avro.specific.FixedSize;
 import org.apache.avro.specific.SpecificExceptionBase;
 import org.apache.avro.specific.SpecificRecordBase;
 import org.apache.avro.specific.SpecificRecord;

Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java
(original)
+++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/generated/WebPage.java
Tue Mar 12 17:07:03 2013
@@ -1,3 +1,21 @@
+/**
+ *Licensed to the Apache Software Foundation (ASF) under one
+ *or more contributor license agreements.  See the NOTICE file
+ *distributed with this work for additional information
+ *regarding copyright ownership.  The ASF licenses this file
+ *to you under the Apache License, Version 2.0 (the"
+ *License"); you may not use this file except in compliance
+ *with the License.  You may obtain a copy of the License at
+ *
+  * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ *Unless required by applicable law or agreed to in writing, software
+ *distributed under the License is distributed on an "AS IS" BASIS,
+ *WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *See the License for the specific language governing permissions and
+ *limitations under the License.
+ */
+
 package org.apache.gora.examples.generated;
 
 import java.nio.ByteBuffer;
@@ -10,6 +28,7 @@ import org.apache.avro.Protocol;
 import org.apache.avro.util.Utf8;
 import org.apache.avro.ipc.AvroRemoteException;
 import org.apache.avro.generic.GenericArray;
+import org.apache.avro.specific.FixedSize;
 import org.apache.avro.specific.SpecificExceptionBase;
 import org.apache.avro.specific.SpecificRecordBase;
 import org.apache.avro.specific.SpecificRecord;
@@ -22,7 +41,7 @@ import org.apache.gora.persistency.ListG
 
 @SuppressWarnings("all")
 public class WebPage extends PersistentBase {
-  public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"WebPage\",\"namespace\":\"org.apache.gora.examples.generated\",\"fields\":[{\"name\":\"url\",\"type\":\"string\"},{\"name\":\"content\",\"type\":\"bytes\"},{\"name\":\"parsedContent\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"metadata\",\"type\":{\"type\":\"record\",\"name\":\"Metadata\",\"fields\":[{\"name\":\"version\",\"type\":\"int\"},{\"name\":\"data\",\"type\":{\"type\":\"map\",\"values\":\"string\"}}]}}]}");
+  public static final Schema _SCHEMA = Schema.parse("{\"type\":\"record\",\"name\":\"WebPage\",\"namespace\":\"org.apache.gora.examples.generated\",\"fields\":[{\"name\":\"url\",\"type\":\"string\"},{\"name\":\"content\",\"type\":[\"null\",\"bytes\"]},{\"name\":\"parsedContent\",\"type\":{\"type\":\"array\",\"items\":\"string\"}},{\"name\":\"outlinks\",\"type\":{\"type\":\"map\",\"values\":\"string\"}},{\"name\":\"metadata\",\"type\":{\"type\":\"record\",\"name\":\"Metadata\",\"fields\":[{\"name\":\"version\",\"type\":\"int\"},{\"name\":\"data\",\"type\":{\"type\":\"map\",\"values\":\"string\"}}]}}]}");
   public static enum Field {
     URL(0,"url"),
     CONTENT(1,"content"),

Modified: gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/WordCount.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/WordCount.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/WordCount.java
(original)
+++ gora/branches/GORA_174/gora-core/src/examples/java/org/apache/gora/examples/mapreduce/WordCount.java
Tue Mar 12 17:07:03 2013
@@ -65,13 +65,15 @@ public class WordCount extends Configure
     protected void map(String key, WebPage page, Context context) 
       throws IOException ,InterruptedException {
       
-      //Get the content from a WebPage as obtained from the DataStore
-      String content =  new String(page.getContent().array());
-      
-      StringTokenizer itr = new StringTokenizer(content);
-      while (itr.hasMoreTokens()) {
-        word.set(itr.nextToken());
-        context.write(word, one);
+      if (page.getContent() != null) {
+        // Get the content from a WebPage as obtained from the DataStore
+        String content = new String(page.getContent().array());
+
+        StringTokenizer itr = new StringTokenizer(content);
+        while (itr.hasMoreTokens()) {
+          word.set(itr.nextToken());
+          context.write(word, one);
+        }
       }
     };
   }

Modified: gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/compiler/GoraCompiler.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/compiler/GoraCompiler.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/compiler/GoraCompiler.java
(original)
+++ gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/compiler/GoraCompiler.java
Tue Mar 12 17:07:03 2013
@@ -374,6 +374,27 @@ public class GoraCompiler {
             line(2, "getStateManager().setDirty(this, "+i+");");
             line(2, "return "+field.name()+".remove(key);");
             line(1, "}");
+            break;
+          case UNION:
+            fieldType = type(fieldSchema);
+            //Create get method: public <unbox(field.schema())> get<camelKey>()
+            line(1, "public "+unbox(field.schema())+" get" +camelKey+"() {");
+            line(2, "return ("+unbox(field.schema())+") get("+i+");");
+            line(1, "}");
+            
+            //Create set methods: public void set<camelKey>(<subschema.fieldType>
value)
+            for (Schema s : fieldSchema.getTypes()) {
+              if (s.getType().equals(Schema.Type.NULL)) continue ;
+              String unionFieldType = type(s);
+              line(1, "public void set"+camelKey+"("+unionFieldType+" value) {");
+              line(2, "put("+i+", value);");
+              line(1, "}");
+            }
+            break;
+          case NULL:
+            throw new RuntimeException("Unexpected NULL field: "+field);
+          default:
+            throw new RuntimeException("Unknown field: "+field);
           }
           i++;
         }

Modified: gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java
(original)
+++ gora/branches/GORA_174/gora-core/src/main/java/org/apache/gora/persistency/impl/PersistentBase.java
Tue Mar 12 17:07:03 2013
@@ -24,6 +24,7 @@ import java.util.Map;
 
 import org.apache.avro.Schema.Field;
 import org.apache.avro.Schema.Type;
+import org.apache.avro.generic.GenericData;
 import org.apache.avro.specific.SpecificRecord;
 import org.apache.gora.avro.PersistentDatumReader;
 import org.apache.gora.persistency.ListGenericArray;
@@ -253,12 +254,33 @@ public abstract class PersistentBase imp
     return result;
   }
 
+  /**
+   * Computes a (record's) field's hash code.
+   * @param i Index of the field in the actual
+   * @param field
+   * @return
+   */
   private int getFieldHashCode(int i, Field field) {
     Object o = get(i);
     if(o == null)
       return 0;
 
-    if(field.schema().getType() == Type.BYTES) {
+    // XXX Union special case: in a field being union we have to check the
+    // inner schemas for Type.BYTES special case, but because it is not a
+    // field we check it this way. Too simple case to create another
+    // private method
+    boolean isUnionField = false ;
+    int unionIndex = -1 ;
+    
+    if (field.schema().getType() == Type.UNION) {
+      isUnionField = true ;
+      unionIndex = GenericData.get().resolveUnion(field.schema(), o);
+    }
+    
+    if(field.schema().getType() == Type.BYTES
+       || (isUnionField
+           && field.schema().getTypes().get(unionIndex).getType() == Type.BYTES))
{
+      // ByteBuffer.hashCode() depends on internal 'position' index, but we must ignore that.
       return getByteBufferHashCode((ByteBuffer)o);
     }
 

Modified: gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
(original)
+++ gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/mapreduce/MapReduceTestUtils.java
Tue Mar 12 17:07:03 2013
@@ -78,11 +78,13 @@ public class MapReduceTestUtils {
     //assert results
     HashMap<String, Integer> actualCounts = new HashMap<String, Integer>();
     for(String content : WebPageDataCreator.CONTENTS) {
-      for(String token:content.split(" ")) {
-        Integer count = actualCounts.get(token);
-        if(count == null) 
-          count = 0;
-        actualCounts.put(token, ++count);
+      if (content != null) {
+        for(String token:content.split(" ")) {
+          Integer count = actualCounts.get(token);
+          if(count == null) 
+            count = 0;
+          actualCounts.put(token, ++count);
+        }
       }
     }
     for(Map.Entry<String, Integer> entry:actualCounts.entrySet()) {

Modified: gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
URL: http://svn.apache.org/viewvc/gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java?rev=1455623&r1=1455622&r2=1455623&view=diff
==============================================================================
--- gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
(original)
+++ gora/branches/GORA_174/gora-core/src/test/java/org/apache/gora/store/DataStoreTestUtil.java
Tue Mar 12 17:07:03 2013
@@ -336,27 +336,35 @@ public class DataStoreTestUtil {
     Assert.assertNotNull(page);
 
     Assert.assertEquals(URLS[i], page.getUrl().toString());
-    Assert.assertTrue("content error:" + new String( toByteArray(page.getContent()) ) +
+    // 'content' is optional
+    if (page.getContent() != null) {
+      Assert.assertTrue("content error:" + new String( toByteArray(page.getContent()) ) +
         " actual=" + CONTENTS[i] + " i=" + i
-    , Arrays.equals( toByteArray(page.getContent() )
+        , Arrays.equals( toByteArray(page.getContent() )
         , CONTENTS[i].getBytes()));
-
-    GenericArray<Utf8> parsedContent = page.getParsedContent();
-    Assert.assertNotNull(parsedContent);
-    Assert.assertTrue(parsedContent.size() > 0);
-
-    int j=0;
-    String[] tokens = CONTENTS[i].split(" ");
-    for(Utf8 token : parsedContent) {
-      Assert.assertEquals(tokens[j++], token.toString());
+      GenericArray<Utf8> parsedContent = page.getParsedContent();
+      Assert.assertNotNull(parsedContent);
+      Assert.assertTrue(parsedContent.size() > 0);
+    
+      int j=0;
+      String[] tokens = CONTENTS[i].split(" ");
+      for(Utf8 token : parsedContent) {
+        Assert.assertEquals(tokens[j++], token.toString());
+      }
+    } else {
+      // when page.getContent() is null
+      Assert.assertTrue(CONTENTS[i] == null) ;
+      GenericArray<Utf8> parsedContent = page.getParsedContent();
+      Assert.assertNotNull(parsedContent);
+      Assert.assertTrue(parsedContent.size() == 0);
     }
 
     if(LINKS[i].length > 0) {
       Assert.assertNotNull(page.getOutlinks());
       Assert.assertTrue(page.getOutlinks().size() > 0);
-      for(j=0; j<LINKS[i].length; j++) {
-        Assert.assertEquals(ANCHORS[i][j],
-            page.getFromOutlinks(new Utf8(URLS[LINKS[i][j]])).toString());
+      for(int k=0; k<LINKS[i].length; k++) {
+        Assert.assertEquals(ANCHORS[i][k],
+          page.getFromOutlinks(new Utf8(URLS[LINKS[i][k]])).toString());
       }
     } else {
       Assert.assertTrue(page.getOutlinks() == null || page.getOutlinks().isEmpty());



Mime
View raw message