pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r591143 [4/4] - in /incubator/pig/branches/types: src/org/apache/pig/ src/org/apache/pig/builtin/ src/org/apache/pig/data/ src/org/apache/pig/impl/ src/org/apache/pig/impl/builtin/ src/org/apache/pig/impl/eval/ src/org/apache/pig/impl/eval/...
Date Thu, 01 Nov 2007 20:48:22 GMT
Added: incubator/pig/branches/types/test/org/apache/pig/test/TestDataUnknown.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestDataUnknown.java?rev=591143&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestDataUnknown.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestDataUnknown.java Thu Nov  1
13:48:16 2007
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileInputStream;
+
+import org.junit.Test;
+
+import org.apache.pig.data.*;
+
+/**
+ * This class will exercise the unknown data type.
+ * 
+ * @author gates
+ */
+public class TestDataUnknown extends junit.framework.TestCase
+{
+
+public void testDefaultConstructor() throws Exception
+{
+	DataUnknown unk = new DataUnknown();
+
+	assertEquals("getType", Datum.DataType.UNKNOWN, unk.getType());
+	assertFalse("is null", unk.isNull());
+
+	assertEquals("Default constructor size before", 0L, unk.size());
+	assertNull("Default constructor get before", unk.get());
+	String s = "hello world";
+	unk.set(s.getBytes());
+	assertEquals("Default constructor size after", 11L, unk.size());
+	String n = new String(unk.get());
+	assertEquals("Default constructor get after", "hello world", n);
+}
+
+public void testByteConstructor() throws Exception
+{
+	String s = "hello world";
+	DataUnknown unk = new DataUnknown(s.getBytes());
+	assertEquals("Byte constructor size before", 11L, unk.size());
+	String n = new String(unk.get());
+	assertEquals("Byte constructor get before", "hello world", n);
+
+	s = "goodbye world";
+	unk.set(s.getBytes());
+	assertEquals("Byte constructor after size", 13L, unk.size());
+	n = new String(unk.get());
+	assertEquals("Byte constructor after get", "goodbye world", n);
+}
+
+public void testToString() throws Exception
+{
+	String s = "hello world";
+	DataUnknown unk = new DataUnknown(s.getBytes());
+	assertEquals("toString", s, unk.toString());
+}
+
+public void testHashCode() throws Exception
+{
+	String s = "hello world";
+	DataUnknown unk1 = new DataUnknown(s.getBytes());
+	DataUnknown unk2 = new DataUnknown(s.getBytes());
+	assertEquals("same data", unk1.hashCode(), unk2.hashCode());
+	s = "goodbye world";
+	DataUnknown unk3 = new DataUnknown(s.getBytes());
+	assertFalse("different data", unk1.hashCode() == unk3.hashCode()); 
+}
+
+public void testEquals() throws Exception
+{
+	String s = "hello world";
+	DataUnknown unk1 = new DataUnknown(s.getBytes());
+	DataUnknown unk2 = new DataUnknown(s.getBytes());
+
+	assertFalse("different object", unk1.equals(s));
+
+	assertTrue("same data", unk1.equals(unk2));
+
+	s = "goodbye world";
+	DataUnknown unk3 = new DataUnknown(s.getBytes());
+	assertFalse("different data", unk1.equals(unk3));
+}
+
+public void testCompareTo() throws Exception
+{
+	String s = "hello world";
+	DataUnknown unk1 = new DataUnknown(s.getBytes());
+	DataUnknown unk2 = new DataUnknown(s.getBytes());
+
+	assertEquals("different object less than", -1, unk1.compareTo(s));
+
+	DataBag bag = new DataBag(Datum.DataType.INT);
+	assertTrue("greater than bag", unk1.compareTo(bag) > 0);
+	Tuple t = new Tuple();
+	assertTrue("greater than tuple", unk1.compareTo(t) > 0);
+	DataMap map = new DataMap();
+	assertTrue("greater than map", unk1.compareTo(map) > 0);
+	DataInteger i = new DataInteger();
+	assertTrue("greater than integer", unk1.compareTo(i) > 0);
+	DataLong l = new DataLong();
+	assertTrue("greater than long", unk1.compareTo(l) > 0);
+	DataFloat f = new DataFloat();
+	assertTrue("greater than float", unk1.compareTo(f) > 0);
+	DataDouble d = new DataDouble();
+	assertTrue("greater than double", unk1.compareTo(d) > 0);
+	DataCharArrayUtf16 utf16 = new DataCharArrayUtf16();
+	assertTrue("greater than char array", unk1.compareTo(utf16) > 0);
+
+	assertEquals("same data equal", 0,  unk1.compareTo(unk2));
+
+	s = "hello dollie";
+	unk2 = new DataUnknown(s.getBytes());
+	assertEquals("greater than unknown with greater lexigraphic value", 1, unk1.compareTo(unk2));
+
+	s = "hello zylophone";
+	unk2 = new DataUnknown(s.getBytes());
+	assertEquals("less than unknown with lower lexigraphic value", -1, unk1.compareTo(unk2));
+
+	s = "hello world, goodbye moon";
+	unk2 = new DataUnknown(s.getBytes());
+	assertEquals("less than longer unknown", -1, unk1.compareTo(unk2));
+
+	s = "hello worl";
+	unk2 = new DataUnknown(s.getBytes());
+	assertEquals("greater than shorter unknown", 1, unk1.compareTo(unk2));
+}
+
+
+public void testWriteRead() throws Exception
+{
+	String s = "hello world";
+	DataUnknown before = new DataUnknown(s.getBytes());
+	File file = null;
+	file = File.createTempFile("DataUnknown", "put");
+	FileOutputStream fos = new FileOutputStream(file);
+	DataOutput out = new DataOutputStream(fos);
+	before.write(out);
+	fos.close();
+
+	FileInputStream fis = new FileInputStream(file);
+	DataInput in = new DataInputStream(fis);
+	Datum a = DatumImpl.readDatum(in);
+
+	assertTrue("isa DataUnknown", a instanceof DataUnknown);
+
+	DataUnknown after = (DataUnknown)a;
+		
+	byte[] beforeBytes = before.get();
+	byte[] afterBytes = after.get();
+
+	assertEquals("length", beforeBytes.length, afterBytes.length);
+	boolean same = true;
+	for (int i = 0; i < beforeBytes.length; i++) {
+		same &= beforeBytes[i] == afterBytes[i];
+	}
+	assertTrue("byte values", same);
+	file.delete();
+}
+
+}
+
+
+ 

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java?rev=591143&r1=591142&r2=591143&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java Thu Nov  1
13:48:16 2007
@@ -15,59 +15,59 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.pig.test;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.StringTokenizer;
-
-import org.junit.Test;
-
-import org.apache.pig.EvalFunc;
-import org.apache.pig.PigServer;
-import org.apache.pig.builtin.BinStorage;
-import org.apache.pig.builtin.PigStorage;
-import org.apache.pig.builtin.TextLoader;
-import org.apache.pig.data.DataBag;
-import org.apache.pig.data.DataMap;
-import org.apache.pig.data.Tuple;
-import org.apache.pig.impl.io.FileLocalizer;
-import org.apache.pig.impl.io.PigFile;
-
-import junit.framework.TestCase;
-
-public class TestEvalPipeline extends TestCase {
-	
-	String initString = "mapreduce";
-	
-	
-	static public class MyBagFunction extends EvalFunc<DataBag>{
-		@Override
-		public void exec(Tuple input, DataBag output) throws IOException {
-			output.add(new Tuple("a"));
-			output.add(new Tuple("a"));
-			output.add(new Tuple("a"));
-			
-		}
-	}
-	
-	private File createFile(String[] data) throws Exception{
-		File f = File.createTempFile("tmp", "");
-		PrintWriter pw = new PrintWriter(f);
-		for (int i=0; i<data.length; i++){
-			pw.println(data[i]);
-		}
-		pw.close();
-		return f;
+package org.apache.pig.test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.StringTokenizer;
+
+import org.junit.Test;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.PigServer;
+import org.apache.pig.builtin.BinStorage;
+import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.builtin.TextLoader;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataMap;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.impl.io.PigFile;
+
+import junit.framework.TestCase;
+
+public class TestEvalPipeline extends TestCase {
+	
+	String initString = "mapreduce";
+	
+	
+	static public class MyBagFunction extends EvalFunc<DataBag>{
+		@Override
+		public void exec(Tuple input, DataBag output) throws IOException {
+			output.add(new Tuple("a"));
+			output.add(new Tuple("a"));
+			output.add(new Tuple("a"));
+			
+		}
+	}
+	
+	private File createFile(String[] data) throws Exception{
+		File f = File.createTempFile("tmp", "");
+		PrintWriter pw = new PrintWriter(f);
+		for (int i=0; i<data.length; i++){
+			pw.println(data[i]);
+		}
+		pw.close();
+		return f;
 	}
 	
 	@Test
@@ -84,230 +84,232 @@
 			assertEquals(iter.next().getAtomField(0).numval(), 0.0);
 		}
 		
-	}
-	
-	@Test
-	public void testJoin() throws Exception{
-		PigServer pigServer = new PigServer(initString);
-		
-		File f1 = createFile(new String[]{"a:1","b:1","a:1"});
-		File f2 = createFile(new String[]{"b","b","a"});
-		
-		pigServer.registerQuery("a = load 'file:" + f1 + "' using " + PigStorage.class.getName()
+ "(':');");
-		pigServer.registerQuery("b = load 'file:" + f2 + "';");
-		pigServer.registerQuery("c = cogroup a by $0, b by $0;");		
-		pigServer.registerQuery("d = foreach c generate flatten($1),flatten($2);");
-		
-		Iterator<Tuple> iter = pigServer.openIterator("d");
-		int count = 0;
-		while(iter.hasNext()){
-			Tuple t = iter.next();
-			assertTrue(t.getAtomField(0).strval().equals(t.getAtomField(2).strval()));
-			count++;
-		}
-		assertEquals(count, 4);
-	}
-	
-	@Test
-	public void testDriverMethod() throws Exception{
-		PigServer pigServer = new PigServer(initString);
-		File f = File.createTempFile("tmp", "");
-		PrintWriter pw = new PrintWriter(f);
-		pw.println("a");
-		pw.println("a");
-		pw.close();
-		pigServer.registerQuery("a = foreach (load 'file:" + f + "') generate '1', flatten(" +
MyBagFunction.class.getName() + "(*));");
-		pigServer.registerQuery("b = foreach a generate $0, flatten($1);");
-		Iterator<Tuple> iter = pigServer.openIterator("a");
-		int count = 0;
-		while(iter.hasNext()){
-			Tuple t = iter.next();
-			assertTrue(t.getAtomField(0).strval().equals("1"));
-			assertTrue(t.getAtomField(1).strval().equals("a"));
-			count++;
-		}
-		assertEquals(count, 6);
-		f.delete();
-	}
-	
-	
-	@Test
-	public void testMapLookup() throws IOException{
-		PigServer pigServer = new PigServer(initString);
-		DataBag b = new DataBag();
-		DataMap colors = new DataMap();
-		colors.put("apple","red");
-		colors.put("orange","orange");
-		
-		DataMap weights = new DataMap();
-		weights.put("apple","0.1");
-		weights.put("orange","0.3");
-		
-		Tuple t = new Tuple();
-		t.appendField(colors);
-		t.appendField(weights);
-		b.add(t);
-		
-		String fileName = "file:"+File.createTempFile("tmp", "");
-		PigFile f = new PigFile(fileName);
-		f.store(b, new BinStorage(), pigServer.getPigContext());
-		
-		
-		pigServer.registerQuery("a = load '" + fileName + "' using BinStorage();");
-		pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
-		Iterator<Tuple> iter = pigServer.openIterator("b");
-		t = iter.next();
-		assertEquals(t.getAtomField(0).strval(), "red");
-		assertEquals(t.getAtomField(1).numval(), 0.3);
-		assertFalse(iter.hasNext());
-	}
-	
-	
-	static public class TitleNGrams extends EvalFunc<DataBag> {
-		
-		@Override
-		public void exec(Tuple input, DataBag output) throws IOException {	
-		    String str = input.getAtomField(0).strval();
-			
-			String title = str;
-
-			if (title != null) {
-				List<String> nGrams = makeNGrams(title);
-				
-				for (Iterator<String> it = nGrams.iterator(); it.hasNext(); ) {
-					Tuple t = new Tuple(1);
-					t.setField(0, it.next());
-					output.add(t);
-				}
-			}
-	    }
-		
-		
-		List<String> makeNGrams(String str) {
-			List<String> tokens = new ArrayList<String>();
-			
-			StringTokenizer st = new StringTokenizer(str);
-			while (st.hasMoreTokens())
-				tokens.add(st.nextToken());
-			
-			return nGramHelper(tokens, new ArrayList<String>());
-		}
-		
-		ArrayList<String> nGramHelper(List<String> str, ArrayList<String> nGrams)
{
-			if (str.size() == 0)
-				return nGrams;
-			
-			for (int i = 0; i < str.size(); i++)
-				nGrams.add(makeString(str.subList(0, i+1)));
-			
-			return nGramHelper(str.subList(1, str.size()), nGrams);
-		}
-		
-		String makeString(List<String> list) {
-			StringBuffer sb = new StringBuffer();
-			for (Iterator<String> it = list.iterator(); it.hasNext(); ) {
-				sb.append(it.next());
-				if (it.hasNext())
-					sb.append(" ");
-			}
-			return sb.toString();
-		}
-	}
-
-	
-	
-	
-	@Test
-	public void testBagFunctionWithFlattening() throws Exception{
-		PigServer pigServer = new PigServer(initString);
-		File queryLogFile = createFile(
-					new String[]{ 
-						"stanford\tdeer\tsighting",
-						"bush\tpresident",
-						"stanford\tbush",
-						"conference\tyahoo",
-						"world\tcup\tcricket",
-						"bush\twins",
-						"stanford\tpresident",
-					}
-				);
-				
-		File newsFile = createFile(
-					new String[]{
-						"deer seen at stanford",
-						"george bush visits stanford", 
-						"yahoo hosting a conference in the bay area", 
-						"who will win the world cup"
-					}
-				);	
-		
-		Map<String, Integer> expectedResults = new HashMap<String, Integer>();
-		expectedResults.put("bush", 2);
-		expectedResults.put("stanford", 3);
-		expectedResults.put("world", 1);
-		expectedResults.put("conference", 1);
-		
-		pigServer.registerQuery("newsArticles = LOAD 'file:" + newsFile + "' USING " + TextLoader.class.getName()
+ "();");
-	    pigServer.registerQuery("queryLog = LOAD 'file:" + queryLogFile + "';");
-
-	    pigServer.registerQuery("titleNGrams = FOREACH newsArticles GENERATE flatten(" + TitleNGrams.class.getName()
+ "(*));");
-	    pigServer.registerQuery("cogrouped = COGROUP titleNGrams BY $0 INNER, queryLog BY $0
INNER;");
-	    pigServer.registerQuery("answer = FOREACH cogrouped GENERATE COUNT(queryLog),group;");
-		
-	    Iterator<Tuple> iter = pigServer.openIterator("answer");
-	    while(iter.hasNext()){
-	    	Tuple t = iter.next();
-	    	assertEquals(expectedResults.get(t.getAtomField(1).strval()).doubleValue(),t.getAtomField(0).numval().doubleValue());
-	    }
-	}
-	
-
-	
-	@Test
-	public void testSort() throws Exception{
-		testSortDistinct(false);
-	}
-	
-
-	@Test
-	public void testDistinct() throws Exception{
-		testSortDistinct(true);
-	}
-
-	private void testSortDistinct(boolean eliminateDuplicates) throws Exception{
-		int LOOP_SIZE = 1024*16;
-		File tmpFile = File.createTempFile("test", "txt");
-        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
-        Random r = new Random();
-        for(int i = 0; i < LOOP_SIZE; i++) {
-            ps.println(r.nextInt(LOOP_SIZE/2) + "\t" + i);
-        }
-        ps.close(); 
-		
-		PigServer pig = new PigServer(initString);
-        String tmpOutputFile = FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString();
-		pig.registerQuery("A = LOAD 'file:" + tmpFile + "';");
-		if (eliminateDuplicates){
-			pig.registerQuery("B = DISTINCT (FOREACH A GENERATE $0) PARALLEL 10;");
-		}else{
-			pig.registerQuery("B = ORDER A BY $0 PARALLEL 10;");
-		}
-		pig.store("B", tmpOutputFile);
-		
-		pig.registerQuery("A = load '" + tmpOutputFile + "';");
-		Iterator<Tuple> iter = pig.openIterator("A");
-		int last = -1;
-		while (iter.hasNext()){
-			Tuple t = iter.next();
-			if (eliminateDuplicates){
-				assertTrue(last < t.getAtomField(0).numval().intValue());
-			}else{
-				assertTrue(last <= t.getAtomField(0).numval().intValue());
-				assertEquals(t.arity(), 2);
-			}
-		}
-		
-	}
-	
-
-}
+	}
+	
+	@Test
+	public void testJoin() throws Exception{
+		PigServer pigServer = new PigServer(initString);
+		
+		File f1 = createFile(new String[]{"a:1","b:1","a:1"});
+		File f2 = createFile(new String[]{"b","b","a"});
+		
+		pigServer.registerQuery("a = load 'file:" + f1 + "' using " + PigStorage.class.getName()
+ "(':');");
+		pigServer.registerQuery("b = load 'file:" + f2 + "';");
+		pigServer.registerQuery("c = cogroup a by $0, b by $0;");		
+		pigServer.registerQuery("d = foreach c generate flatten($1),flatten($2);");
+		
+		Iterator<Tuple> iter = pigServer.openIterator("d");
+		int count = 0;
+		while(iter.hasNext()){
+			Tuple t = iter.next();
+			assertTrue(t.getAtomField(0).strval().equals(t.getAtomField(2).strval()));
+			count++;
+		}
+		assertEquals(count, 4);
+	}
+	
+	@Test
+	public void testDriverMethod() throws Exception{
+		PigServer pigServer = new PigServer(initString);
+		File f = File.createTempFile("tmp", "");
+		PrintWriter pw = new PrintWriter(f);
+		pw.println("a");
+		pw.println("a");
+		pw.close();
+		pigServer.registerQuery("a = foreach (load 'file:" + f + "') generate '1', flatten(" +
MyBagFunction.class.getName() + "(*));");
+		pigServer.registerQuery("b = foreach a generate $0, flatten($1);");
+		Iterator<Tuple> iter = pigServer.openIterator("a");
+		int count = 0;
+		while(iter.hasNext()){
+			Tuple t = iter.next();
+			assertTrue(t.getAtomField(0).strval().equals("1"));
+			assertTrue(t.getAtomField(1).strval().equals("a"));
+			count++;
+		}
+		assertEquals(count, 6);
+		f.delete();
+	}
+	
+	
+	/* Replaced by TestDataMap.java
+	@Test
+	public void testMapLookup() throws IOException{
+		PigServer pigServer = new PigServer(initString);
+		DataBag b = new DataBag();
+		DataMap colors = new DataMap();
+		colors.put("apple","red");
+		colors.put("orange","orange");
+		
+		DataMap weights = new DataMap();
+		weights.put("apple","0.1");
+		weights.put("orange","0.3");
+		
+		Tuple t = new Tuple();
+		t.appendField(colors);
+		t.appendField(weights);
+		b.add(t);
+		
+		String fileName = "file:"+File.createTempFile("tmp", "");
+		PigFile f = new PigFile(fileName);
+		f.store(b, new BinStorage(), pigServer.getPigContext());
+		
+		
+		pigServer.registerQuery("a = load '" + fileName + "' using BinStorage();");
+		pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
+		Iterator<Tuple> iter = pigServer.openIterator("b");
+		t = iter.next();
+		assertEquals(t.getAtomField(0).strval(), "red");
+		assertEquals(t.getAtomField(1).numval(), 0.3);
+		assertFalse(iter.hasNext());
+	}
+	*/
+	
+	
+	static public class TitleNGrams extends EvalFunc<DataBag> {
+		
+		@Override
+		public void exec(Tuple input, DataBag output) throws IOException {	
+		    String str = input.getAtomField(0).strval();
+			
+			String title = str;
+
+			if (title != null) {
+				List<String> nGrams = makeNGrams(title);
+				
+				for (Iterator<String> it = nGrams.iterator(); it.hasNext(); ) {
+					Tuple t = new Tuple(1);
+					t.setField(0, it.next());
+					output.add(t);
+				}
+			}
+	    }
+		
+		
+		List<String> makeNGrams(String str) {
+			List<String> tokens = new ArrayList<String>();
+			
+			StringTokenizer st = new StringTokenizer(str);
+			while (st.hasMoreTokens())
+				tokens.add(st.nextToken());
+			
+			return nGramHelper(tokens, new ArrayList<String>());
+		}
+		
+		ArrayList<String> nGramHelper(List<String> str, ArrayList<String> nGrams)
{
+			if (str.size() == 0)
+				return nGrams;
+			
+			for (int i = 0; i < str.size(); i++)
+				nGrams.add(makeString(str.subList(0, i+1)));
+			
+			return nGramHelper(str.subList(1, str.size()), nGrams);
+		}
+		
+		String makeString(List<String> list) {
+			StringBuffer sb = new StringBuffer();
+			for (Iterator<String> it = list.iterator(); it.hasNext(); ) {
+				sb.append(it.next());
+				if (it.hasNext())
+					sb.append(" ");
+			}
+			return sb.toString();
+		}
+	}
+
+	
+	
+	
+	@Test
+	public void testBagFunctionWithFlattening() throws Exception{
+		PigServer pigServer = new PigServer(initString);
+		File queryLogFile = createFile(
+					new String[]{ 
+						"stanford\tdeer\tsighting",
+						"bush\tpresident",
+						"stanford\tbush",
+						"conference\tyahoo",
+						"world\tcup\tcricket",
+						"bush\twins",
+						"stanford\tpresident",
+					}
+				);
+				
+		File newsFile = createFile(
+					new String[]{
+						"deer seen at stanford",
+						"george bush visits stanford", 
+						"yahoo hosting a conference in the bay area", 
+						"who will win the world cup"
+					}
+				);	
+		
+		Map<String, Integer> expectedResults = new HashMap<String, Integer>();
+		expectedResults.put("bush", 2);
+		expectedResults.put("stanford", 3);
+		expectedResults.put("world", 1);
+		expectedResults.put("conference", 1);
+		
+		pigServer.registerQuery("newsArticles = LOAD 'file:" + newsFile + "' USING " + TextLoader.class.getName()
+ "();");
+	    pigServer.registerQuery("queryLog = LOAD 'file:" + queryLogFile + "';");
+
+	    pigServer.registerQuery("titleNGrams = FOREACH newsArticles GENERATE flatten(" + TitleNGrams.class.getName()
+ "(*));");
+	    pigServer.registerQuery("cogrouped = COGROUP titleNGrams BY $0 INNER, queryLog BY $0
INNER;");
+	    pigServer.registerQuery("answer = FOREACH cogrouped GENERATE COUNT(queryLog),group;");
+		
+	    Iterator<Tuple> iter = pigServer.openIterator("answer");
+	    while(iter.hasNext()){
+	    	Tuple t = iter.next();
+	    	assertEquals(expectedResults.get(t.getAtomField(1).strval()).doubleValue(),t.getAtomField(0).numval().doubleValue());
+	    }
+	}
+	
+
+	
+	@Test
+	public void testSort() throws Exception{
+		testSortDistinct(false);
+	}
+	
+
+	@Test
+	public void testDistinct() throws Exception{
+		testSortDistinct(true);
+	}
+
+	private void testSortDistinct(boolean eliminateDuplicates) throws Exception{
+		int LOOP_SIZE = 1024*16;
+		File tmpFile = File.createTempFile("test", "txt");
+        PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
+        Random r = new Random();
+        for(int i = 0; i < LOOP_SIZE; i++) {
+            ps.println(r.nextInt(LOOP_SIZE/2) + "\t" + i);
+        }
+        ps.close(); 
+		
+		PigServer pig = new PigServer(initString);
+        String tmpOutputFile = FileLocalizer.getTemporaryPath(null, pig.getPigContext()).toString();
+		pig.registerQuery("A = LOAD 'file:" + tmpFile + "';");
+		if (eliminateDuplicates){
+			pig.registerQuery("B = DISTINCT (FOREACH A GENERATE $0) PARALLEL 10;");
+		}else{
+			pig.registerQuery("B = ORDER A BY $0 PARALLEL 10;");
+		}
+		pig.store("B", tmpOutputFile);
+		
+		pig.registerQuery("A = load '" + tmpOutputFile + "';");
+		Iterator<Tuple> iter = pig.openIterator("A");
+		int last = -1;
+		while (iter.hasNext()){
+			Tuple t = iter.next();
+			if (eliminateDuplicates){
+				assertTrue(last < t.getAtomField(0).numval().intValue());
+			}else{
+				assertTrue(last <= t.getAtomField(0).numval().intValue());
+				assertEquals(t.arity(), 2);
+			}
+		}
+		
+	}
+	
+
+}

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpNumeric.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpNumeric.java?rev=591143&r1=591142&r2=591143&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpNumeric.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpNumeric.java Thu Nov
 1 13:48:16 2007
@@ -27,6 +27,7 @@
 import org.apache.pig.PigServer;
 import org.apache.pig.builtin.PigStorage;
 import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.util.PigLogger;
 
 import junit.framework.TestCase;
 
@@ -38,6 +39,7 @@
     
     @Test
     public void testNumericEq() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -69,6 +71,7 @@
 
     @Test
     public void testNumericNeq() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -96,6 +99,7 @@
 
     @Test
     public void testNumericGt() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -124,6 +128,7 @@
 
     @Test
     public void testBinCond() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -153,6 +158,7 @@
     
     @Test
     public void testNestedBinCond() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -181,6 +187,7 @@
     
     @Test 
     public void testNumericLt() throws Exception {
+		PigLogger.setAppenderForJunit();
     	PigServer pig = new PigServer(initString);
     	File tmpFile = File.createTempFile("test", "txt");
     	PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -211,6 +218,7 @@
     
     @Test
     public void testNumericGte() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -241,6 +249,7 @@
 
     @Test
     public void testNumericLte() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpString.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpString.java?rev=591143&r1=591142&r2=591143&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpString.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestFilterOpString.java Thu Nov
 1 13:48:16 2007
@@ -27,6 +27,7 @@
 import org.apache.pig.PigServer;
 import org.apache.pig.builtin.PigStorage;
 import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.util.PigLogger;
 
 import junit.framework.TestCase;
 
@@ -37,6 +38,7 @@
     
     @Test
     public void testStringEq() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -65,6 +67,7 @@
     
     @Test
     public void testStringNeq() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -93,6 +96,7 @@
 
     @Test
     public void testStringGt() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -123,6 +127,7 @@
 
     @Test
     public void testStringGte() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -154,6 +159,7 @@
 
     @Test
     public void testStringLt() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
@@ -183,6 +189,7 @@
 
     @Test
     public void testStringLte() throws Exception {
+		PigLogger.setAppenderForJunit();
         PigServer pig = new PigServer(initString);
         File tmpFile = File.createTempFile("test", "txt");
         PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestMapReduce.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestMapReduce.java?rev=591143&r1=591142&r2=591143&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestMapReduce.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestMapReduce.java Thu Nov  1 13:48:16
2007
@@ -78,9 +78,9 @@
     	}
         @Override
 		public void exec(Tuple input, DataBag output) throws IOException {
-            Iterator<Tuple> it = (input.getBagField(0)).content();
+            Iterator<Datum> it = (input.getBagField(0)).content();
             while(it.hasNext()) {
-                Tuple t = it.next();
+                Tuple t = (Tuple)it.next();
                 Tuple newT = new Tuple(2);
                 newT.setField(0, field0);
                 newT.setField(1, t.getField(0).toString());

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestPigFile.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestPigFile.java?rev=591143&r1=591142&r2=591143&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestPigFile.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestPigFile.java Thu Nov  1 13:48:16
2007
@@ -44,7 +44,7 @@
 
 public class TestPigFile extends TestCase {
 
-    DataBag bag          = new DataBag();
+    DataBag bag          = new DataBag(Datum.DataType.TUPLE);
     Random rand = new Random();
     
     @Override
@@ -91,11 +91,11 @@
 
         assertTrue(bag.cardinality() == loaded.cardinality());
 
-        Iterator<Tuple> it1 = bag.content();
-        Iterator<Tuple> it2 = loaded.content();
+        Iterator<Datum> it1 = bag.content();
+        Iterator<Datum> it2 = loaded.content();
         while (it1.hasNext() && it2.hasNext()) {
-            Tuple f1 = it1.next();
-            Tuple f2 = it2.next();
+            Tuple f1 = (Tuple)it1.next();
+            Tuple f2 = (Tuple)it2.next();
             assertTrue(f1.equals(f2));
         }
         assertFalse(it1.hasNext() || it2.hasNext());
@@ -131,7 +131,7 @@
     
     private DataBag getRandomBag(int maxCardinality, int nestingLevel) throws IOException{
     	int cardinality = rand.nextInt(maxCardinality)+1;
-    	DataBag b = new DataBag();
+    	DataBag b = new DataBag(Datum.DataType.TUPLE);
     	for (int i=0; i<cardinality; i++){
     		Tuple t = getRandomTuple(nestingLevel+1); 
     		b.add(t);
@@ -170,11 +170,12 @@
 
         assertTrue(bag.cardinality() == loaded.cardinality());
 
-        Iterator<Tuple> it1 = bag.content();
-        Iterator<Tuple> it2 = loaded.content();
-        while (it1.hasNext() && it2.hasNext()) {
-            Tuple f1 = it1.next();
-            Tuple f2 = it2.next();
+        Iterator<Datum> it1 = bag.content();
+        Iterator<Datum> it2 = loaded.content();
+        //while (it1.hasNext() && it2.hasNext()) {
+        for (int i = 0; it1.hasNext() && it2.hasNext(); i++) {
+            Tuple f1 = (Tuple)it1.next();
+            Tuple f2 = (Tuple)it2.next();
             assertTrue(f1.equals(f2));
         }
         assertFalse(it1.hasNext() || it2.hasNext());

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestPigSplit.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestPigSplit.java?rev=591143&r1=591142&r2=591143&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestPigSplit.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestPigSplit.java Thu Nov  1 13:48:16
2007
@@ -28,6 +28,7 @@
 
 import org.apache.pig.PigServer;
 import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.util.PigLogger;
 
 import junit.framework.TestCase;
 
@@ -37,6 +38,7 @@
 	@Override
 	protected void setUp() throws Exception {
 		super.setUp();
+		PigLogger.setAppenderForJunit();
 		pig = new PigServer();
 	}
 	

Added: incubator/pig/branches/types/test/org/apache/pig/test/TestTuple.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTuple.java?rev=591143&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestTuple.java (added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestTuple.java Thu Nov  1 13:48:16
2007
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * tstributed with this work for adttional information
+ * regartng copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * tstributed under the License is tstributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileInputStream;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.junit.Test;
+
+import org.apache.pig.data.*;
+
+/**
+ * This class will exercise the tuple data type.
+ * 
+ * @author gates
+ */
+public class TestTuple extends junit.framework.TestCase
+{
+
+public void testDefaultConstructor() throws Exception
+{
+	Tuple t = new Tuple();
+
+	assertEquals("getType", Datum.DataType.TUPLE, t.getType());
+	assertFalse("is null", t.isNull());
+
+	assertEquals("default construct size", 0L, t.size());
+
+	Datum d = t.getField(0);
+	assertTrue("unknown field is null", d.isNull());
+
+	DataInteger di = new DataInteger(3);
+	t.appendField(new DataInteger(3));
+
+	assertEquals("default constructor size after append", 1L, t.size());
+	d = t.getField(0);
+	assertTrue("default constructor after append", di.equals(d));
+
+	t.setField(0, new DataInteger(4));
+	assertEquals("default constructor size after set", 1L, t.size());
+	d = t.getField(0);
+	assertFalse("default constructor after set", di.equals(d));
+}
+
+public void testLongConstructor() throws Exception
+{
+	Tuple t = new Tuple(3);
+
+	assertEquals("long construct size", 3L, t.size());
+
+	Datum d = t.getField(2);
+	assertTrue("unset field is null", d.isNull());
+
+	d = t.getField(3);
+	assertTrue("field past end is null", d.isNull());
+
+
+	t.setField(0, new DataInteger(10));
+	t.setField(1, new DataLong(11L));
+	t.setField(2, new DataFloat(9.9f));
+
+	assertEquals("long constructor size after set", 3L, t.size());
+	d = t.getField(0);
+	assertEquals("long constructor after set, col 0", 10,
+		((DataInteger)d).get());
+	d = t.getField(1);
+	assertEquals("long constructor after set, col 1", 11L,
+		((DataLong)d).get());
+	d = t.getField(2);
+	assertEquals("long constructor after set, col 2", 9.9f,
+		((DataFloat)d).get());
+
+	t.appendField(new DataInteger(4));
+	assertEquals("long constructor size after append", 4L, t.size());
+	d = t.getField(3);
+	assertEquals("long constructor after append, col 3", 4,
+		((DataInteger)d).get());
+}
+
+public void testListConstructor() throws Exception
+{
+	List<Datum> list = new ArrayList<Datum>();
+	list.add(new DataInteger(10));
+	list.add(new DataLong(11L));
+	list.add(new DataFloat(9.9f));
+
+	Tuple t = new Tuple(list);
+
+	assertEquals("list construct size", 3L, t.size());
+
+	Datum d = t.getField(2);
+	assertFalse("set field is not null", d.isNull());
+
+	d = t.getField(3);
+	assertTrue("field past end is null", d.isNull());
+
+
+	d = t.getField(0);
+	assertEquals("list constructor after set, col 0", 10,
+		((DataInteger)d).get());
+	d = t.getField(1);
+	assertEquals("list constructor after set, col 1", 11L,
+		((DataLong)d).get());
+	d = t.getField(2);
+	assertEquals("list constructor after set, col 2", 9.9f,
+		((DataFloat)d).get());
+
+	t.appendField(new DataInteger(4));
+	assertEquals("list constructor size after append", 4L, t.size());
+	d = t.getField(3);
+	assertEquals("list constructor after append, col 3", 4,
+		((DataInteger)d).get());
+}
+
+
+public void testToString() throws Exception
+{
+	Tuple t = giveMeOneOfEach();
+
+	assertEquals("toString",
+		"(hello world, 1, 2, 3.1415, 4.23223, mary had a little lamb, jack be nimble, [aaa#1],
(99), {8})", t.toString());
+}
+
+public void testHashCode() throws Exception
+{
+	Tuple t1 = new Tuple(2);
+	String s = new String("hello world");
+	t1.setField(0, new DataUnknown(s.getBytes()));
+	t1.setField(1, new DataInteger(1));
+
+	Tuple t2 = new Tuple();
+	t2.appendField(new DataUnknown(s.getBytes()));
+	t2.appendField(new DataInteger(1));
+
+	assertEquals("same data", t1.hashCode(), t2.hashCode());
+
+	Tuple t3 = new Tuple(3);
+	t1.setField(0, new DataUnknown(s.getBytes()));
+	t1.setField(1, new DataInteger(1));
+	assertFalse("different size", t1.hashCode() == t3.hashCode()); 
+
+	Tuple t4 = new Tuple(2);
+	t1.setField(0, new DataUnknown(s.getBytes()));
+	t1.setField(1, new DataInteger(2));
+	assertFalse("same size, different data", t1.hashCode() == t3.hashCode()); 
+}
+
+public void testEquals() throws Exception
+{
+	Tuple t1 = new Tuple();
+	Tuple t2 = new Tuple();
+
+	t1.appendField(new DataInteger(3));
+	t2.appendField(new DataInteger(3));
+
+	assertFalse("different object", t1.equals(new String()));
+
+	assertTrue("same data", t1.equals(t2));
+
+	t2 = new Tuple();
+	t2.appendField(new DataInteger(4));
+	assertFalse("different data", t1.equals(t2));
+
+	t2 = new Tuple();
+	t2.appendField(new DataInteger(3));
+	t2.appendField(new DataInteger(3));
+	assertFalse("different size", t1.equals(t2));
+}
+
+public void testCompareTo() throws Exception
+{
+	Tuple t1 = new Tuple();
+	Tuple t2 = new Tuple();
+
+	t1.appendField(new DataInteger(3));
+	t2.appendField(new DataInteger(3));
+
+	assertEquals("different object less than", -1, t1.compareTo(new String()));
+
+	DataBag bag = new DataBag(Datum.DataType.INT);
+	assertTrue("greater than bag", t1.compareTo(bag) > 0);
+	DataMap map = new DataMap();
+	assertTrue("less than map", t1.compareTo(map) < 0);
+	DataLong l = new DataLong();
+	assertTrue("less than long", t1.compareTo(l) < 0);
+	DataFloat f = new DataFloat();
+	assertTrue("less than float", t1.compareTo(f) < 0);
+	DataDouble d = new DataDouble();
+	assertTrue("less than double", t1.compareTo(d) < 0);
+	DataUnknown unk = new DataUnknown();
+	assertTrue("less than unknown", t1.compareTo(unk) < 0);
+	DataCharArrayUtf16 utf16 = new DataCharArrayUtf16();
+	assertTrue("less than utf16", t1.compareTo(utf16) < 0);
+
+	assertEquals("same data equal", 0,  t1.compareTo(t2));
+
+	t2 = new Tuple();
+	t2.appendField(new DataInteger(2));
+	assertEquals("greater than tuple with lesser value", 1, t1.compareTo(t2));
+
+	t2 = new Tuple();
+	t2.appendField(new DataInteger(4));
+	assertEquals("less than tuple with greater value", -1, t1.compareTo(t2));
+
+	t2 = new Tuple();
+	t2.appendField(new DataInteger(3));
+	t2.appendField(new DataInteger(4));
+	assertEquals("less than bigger tuple", -1, t1.compareTo(t2));
+
+	t2 = new Tuple();
+	assertEquals("greater than smaller tuple", 1, t1.compareTo(t2));
+}
+
+
+public void testWriteRead() throws Exception
+{
+	Tuple before = giveMeOneOfEach();
+
+	File file = null;
+	file = File.createTempFile("Tuple", "put");
+	FileOutputStream fos = new FileOutputStream(file);
+	DataOutput out = new DataOutputStream(fos);
+	before.write(out);
+	fos.close();
+
+	FileInputStream fis = new FileInputStream(file);
+	DataInput in = new DataInputStream(fis);
+	Datum a = DatumImpl.readDatum(in);
+
+	assertTrue("isa Tuple", a instanceof Tuple);
+
+	Tuple after = (Tuple)a;
+		
+	String s = new String("hello world");
+	Datum f1 = after.getField(0);
+	assertTrue("isa Unknown", f1 instanceof DataUnknown);
+	byte[] beforeBytes = s.getBytes();
+	byte[] afterBytes = ((DataUnknown)f1).get();
+	assertEquals("length", beforeBytes.length, afterBytes.length);
+	boolean same = true;
+	for (int i = 0; i < beforeBytes.length && same; i++) {
+		same &= beforeBytes[i] == afterBytes[i];
+	}
+	assertTrue("after write/read unknown", same);
+
+	Datum f2 = after.getField(1);
+	assertTrue("isa Integer", f2 instanceof DataInteger);
+	assertEquals("after read/write integer", 1, ((DataInteger)f2).get());
+
+	Datum f3 = after.getField(2);
+	assertTrue("isa Long", f3 instanceof DataLong);
+	assertEquals("after read/write long", 2L, ((DataLong)f3).get());
+
+	Datum f4 = after.getField(3);
+	assertTrue("isa Float", f4 instanceof DataFloat);
+	assertEquals("after read/write float", 3.1415f, ((DataFloat)f4).get());
+
+	Datum f5 = after.getField(4);
+	assertTrue("isa Double", f5 instanceof DataDouble);
+	assertEquals("after read/write double", 4.23223, ((DataDouble)f5).get());
+
+	s = new String("mary had a little lamb");
+	Datum f6 = after.getField(5);
+	assertTrue("isa None", f6 instanceof DataCharArrayNone);
+	beforeBytes = s.getBytes();
+	afterBytes = ((DataCharArrayNone)f6).get();
+	assertEquals("length", beforeBytes.length, afterBytes.length);
+	same = true;
+	for (int i = 0; i < beforeBytes.length && same; i++) {
+		same &= beforeBytes[i] == afterBytes[i];
+	}
+	assertTrue("after write/read none", same);
+
+	Datum f7 = after.getField(6);
+	assertTrue("isa Utf16", f7 instanceof DataCharArrayUtf16);
+	assertEquals("after read/write utf16", "jack be nimble",
+		((DataCharArrayUtf16)f7).get());
+
+	Datum f8 = after.getField(7);
+	assertTrue("isa map", f8 instanceof DataMap);
+	assertEquals("after read/write size of map", 1L, f8.size());
+	Datum val = ((DataMap)f8).get(new DataCharArrayUtf16("aaa"));
+	assertFalse("val of map after read/write", val.isNull());
+	assertTrue("val is a long after read/write", val instanceof DataLong);
+	assertEquals("val is 1 after read/write", 1L, ((DataLong)val).get());
+
+	Datum f9 = after.getField(8);
+	assertTrue("isa tuple", f9 instanceof Tuple);
+	assertEquals("after read/write size of tuple", 1L, f9.size());
+	val = ((Tuple)f9).getField(0);
+	assertFalse("val of tuple after read/write", val.isNull());
+	assertTrue("val is an int after read/write", val instanceof DataInteger);
+	assertEquals("val is 99 after read/write", 99, ((DataInteger)val).get());
+
+	Datum f10 = after.getField(9);
+	assertTrue("isa bag", f10 instanceof DataBag);
+	assertEquals("after read/write size of bag", 1L, f10.size());
+	Iterator<Datum> i = ((DataBag)f10).content();
+	while (i.hasNext()) {
+		DataInteger di = (DataInteger)i.next();
+		assertEquals("after read/write bag element", 8, di.get());
+	}
+
+	file.delete();
+}
+
+private Tuple giveMeOneOfEach()
+{
+	// Create a tuple with one of each data type in it.
+	Tuple t = new Tuple();
+	String s = new String("hello world");
+	t.appendField(new DataUnknown(s.getBytes()));
+	t.appendField(new DataInteger(1));
+	t.appendField(new DataLong(2L));
+	t.appendField(new DataFloat(3.1415f));
+	t.appendField(new DataDouble(4.23223));
+	s = new String("mary had a little lamb");
+	t.appendField(new DataCharArrayNone(s.getBytes()));
+	t.appendField(new DataCharArrayUtf16("jack be nimble"));
+
+	DataMap map = new DataMap();
+	DataCharArrayUtf16 key = new DataCharArrayUtf16("aaa");
+	Datum val = new DataLong(1L);
+	map.put(key, val);
+	t.appendField(map);
+
+	Tuple tsub = new Tuple(new DataInteger(99));
+	t.appendField(tsub);
+
+	DataBag bag = new DataBag(Datum.DataType.INT);
+	bag.add(new DataInteger(8));
+	t.appendField(bag);
+
+	return t;
+}
+
+}
+
+
+ 

Modified: incubator/pig/branches/types/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/Util.java?rev=591143&r1=591142&r2=591143&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/Util.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/Util.java Thu Nov  1 13:48:16 2007
@@ -21,13 +21,15 @@
 
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.Tuple;
+import org.apache.pig.data.Datum;
+import org.apache.pig.data.DataAtom;
 
 public class Util {
     // Helper Functions
     // =================
     static public Tuple loadFlatTuple(Tuple t, int[] input) throws IOException {
         for (int i = 0; i < input.length; i++) {
-            t.setField(i, input[i]);
+            t.setField(i, new DataAtom(new Integer(input[i]).toString()));
         }
         return t;
     }
@@ -40,7 +42,7 @@
     }
 
     static public Tuple loadNestTuple(Tuple t, int[] input) throws IOException {
-        DataBag bag = new DataBag();
+        DataBag bag = new DataBag(Datum.DataType.TUPLE);
         for(int i = 0; i < input.length; i++) {
             Tuple f = new Tuple(1);
             f.setField(0, input[i]);
@@ -52,7 +54,7 @@
 
     static public Tuple loadNestTuple(Tuple t, int[][] input) throws IOException {
         for (int i = 0; i < input.length; i++) {
-            DataBag bag = new DataBag();
+            DataBag bag = new DataBag(Datum.DataType.TUPLE);
             Tuple f = loadFlatTuple(new Tuple(input[i].length), input[i]);
             bag.add(f);
             t.setField(i, bag);
@@ -62,7 +64,7 @@
 
     static public Tuple loadTuple(Tuple t, String[][] input) throws IOException {
         for (int i = 0; i < input.length; i++) {
-            DataBag bag = new DataBag();
+            DataBag bag = new DataBag(Datum.DataType.TUPLE);
             Tuple f = loadTuple(new Tuple(input[i].length), input[i]);
             bag.add(f);
             t.setField(i, bag);



Mime
View raw message