Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 42677187FA for ; Mon, 23 Nov 2015 02:14:47 +0000 (UTC) Received: (qmail 64834 invoked by uid 500); 23 Nov 2015 02:14:47 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 64773 invoked by uid 500); 23 Nov 2015 02:14:47 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 64764 invoked by uid 99); 23 Nov 2015 02:14:46 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 23 Nov 2015 02:14:46 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id CDE04DFBD5; Mon, 23 Nov 2015 02:14:46 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wheat9@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HADOOP-10465. Fix use of generics within SortedMapWritable. Contributed by Bertrand Dechoux. Date: Mon, 23 Nov 2015 02:14:46 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk aac260faa -> dc46c46b9 HADOOP-10465. Fix use of generics within SortedMapWritable. Contributed by Bertrand Dechoux. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc46c46b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc46c46b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc46c46b Branch: refs/heads/trunk Commit: dc46c46b9123ddb37b972971d3a4f7424165795c Parents: aac260f Author: Haohui Mai Authored: Sun Nov 22 18:10:08 2015 -0800 Committer: Haohui Mai Committed: Sun Nov 22 18:10:08 2015 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++ .../org/apache/hadoop/io/SortedMapWritable.java | 47 +++++++++----------- .../apache/hadoop/io/TestSortedMapWritable.java | 37 +++++++-------- .../typedbytes/TypedBytesWritableInput.java | 13 +++--- .../typedbytes/TypedBytesWritableOutput.java | 6 +-- 5 files changed, 53 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc46c46b/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 08c7ae8..08a8bf4 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -978,6 +978,9 @@ Release 2.8.0 - UNRELEASED HADOOP-10555. Add offset support to MurmurHash. (Sergey Shelukhin via wheat9) + HADOOP-10465. Fix use of generics within SortedMapWritable. + (Bertrand Dechoux via wheat9) + OPTIMIZATIONS HADOOP-11785. Reduce the number of listStatus operation in distcp http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc46c46b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java index 3b51fc2..3ed2a1d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SortedMapWritable.java @@ -36,15 +36,15 @@ import org.apache.hadoop.util.ReflectionUtils; */ @InterfaceAudience.Public @InterfaceStability.Stable -public class SortedMapWritable extends AbstractMapWritable - implements SortedMap { +public class SortedMapWritable> extends AbstractMapWritable + implements SortedMap { - private SortedMap instance; + private SortedMap instance; /** default constructor. */ public SortedMapWritable() { super(); - this.instance = new TreeMap(); + this.instance = new TreeMap(); } /** @@ -52,45 +52,39 @@ public class SortedMapWritable extends AbstractMapWritable * * @param other the map to copy from */ - public SortedMapWritable(SortedMapWritable other) { + public SortedMapWritable(SortedMapWritable other) { this(); copy(other); } @Override - public Comparator comparator() { + public Comparator comparator() { // Returning null means we use the natural ordering of the keys return null; } @Override - public WritableComparable firstKey() { + public K firstKey() { return instance.firstKey(); } @Override - public SortedMap - headMap(WritableComparable toKey) { - + public SortedMap headMap(K toKey) { return instance.headMap(toKey); } @Override - public WritableComparable lastKey() { + public K lastKey() { return instance.lastKey(); } @Override - public SortedMap - subMap(WritableComparable fromKey, WritableComparable toKey) { - + public SortedMap subMap(K fromKey, K toKey) { return instance.subMap(fromKey, toKey); } @Override - public SortedMap - tailMap(WritableComparable fromKey) { - + public SortedMap tailMap(K fromKey) { return instance.tailMap(fromKey); } @@ -110,7 +104,7 @@ public class SortedMapWritable extends AbstractMapWritable } @Override - public Set> entrySet() { + public Set> entrySet() { return instance.entrySet(); } @@ -125,22 +119,21 @@ public class SortedMapWritable extends AbstractMapWritable } @Override - public Set keySet() { + public Set keySet() { return instance.keySet(); } @Override - public Writable put(WritableComparable key, Writable value) { + public Writable put(K key, Writable value) { addToMap(key.getClass()); addToMap(value.getClass()); return instance.put(key, value); } @Override - public void putAll(Map t) { - for (Map.Entry e: + public void putAll(Map t) { + for (Map.Entry e: t.entrySet()) { - put(e.getKey(), e.getValue()); } } @@ -172,8 +165,8 @@ public class SortedMapWritable extends AbstractMapWritable // Then read each key/value pair for (int i = 0; i < entries; i++) { - WritableComparable key = - (WritableComparable) ReflectionUtils.newInstance(getClass( + K key = + (K) ReflectionUtils.newInstance(getClass( in.readByte()), getConf()); key.readFields(in); @@ -196,7 +189,7 @@ public class SortedMapWritable extends AbstractMapWritable // Then write out each key/value pair - for (Map.Entry e: instance.entrySet()) { + for (Map.Entry e: instance.entrySet()) { out.writeByte(getId(e.getKey().getClass())); e.getKey().write(out); out.writeByte(getId(e.getValue().getClass())); @@ -211,7 +204,7 @@ public class SortedMapWritable extends AbstractMapWritable } if (obj instanceof SortedMapWritable) { - Map map = (Map) obj; + Map map = (Map) obj; if (size() != map.size()) { return false; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc46c46b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java index 5ed1db2..8c98808 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSortedMapWritable.java @@ -45,7 +45,7 @@ public class TestSortedMapWritable { new BytesWritable("value3".getBytes()) }; - SortedMapWritable inMap = new SortedMapWritable(); + SortedMapWritable inMap = new SortedMapWritable(); for (int i = 0; i < keys.length; i++) { inMap.put(keys[i], values[i]); } @@ -53,13 +53,14 @@ public class TestSortedMapWritable { assertEquals(0, inMap.firstKey().compareTo(keys[0])); assertEquals(0, inMap.lastKey().compareTo(keys[2])); - SortedMapWritable outMap = new SortedMapWritable(inMap); + SortedMapWritable outMap = new SortedMapWritable(inMap); assertEquals(inMap.size(), outMap.size()); - for (Map.Entry e: inMap.entrySet()) { + for (Map.Entry e: inMap.entrySet()) { assertTrue(outMap.containsKey(e.getKey())); - assertEquals(0, ((WritableComparable) outMap.get(e.getKey())).compareTo( - e.getValue())); + WritableComparable> aValue = (WritableComparable>) outMap.get(e.getKey()); + WritableComparable> bValue = (WritableComparable>) e.getValue(); + assertEquals(0, aValue.compareTo(bValue)); } // Now for something a little harder... @@ -69,24 +70,24 @@ public class TestSortedMapWritable { new Text("map2") }; - SortedMapWritable mapOfMaps = new SortedMapWritable(); + SortedMapWritable mapOfMaps = new SortedMapWritable(); mapOfMaps.put(maps[0], inMap); mapOfMaps.put(maps[1], outMap); - SortedMapWritable copyOfMapOfMaps = new SortedMapWritable(mapOfMaps); + SortedMapWritable copyOfMapOfMaps = new SortedMapWritable(mapOfMaps); for (int i = 0; i < maps.length; i++) { assertTrue(copyOfMapOfMaps.containsKey(maps[i])); - SortedMapWritable a = (SortedMapWritable) mapOfMaps.get(maps[i]); - SortedMapWritable b = (SortedMapWritable) copyOfMapOfMaps.get(maps[i]); + SortedMapWritable a = (SortedMapWritable) mapOfMaps.get(maps[i]); + SortedMapWritable b = (SortedMapWritable) copyOfMapOfMaps.get(maps[i]); assertEquals(a.size(), b.size()); for (Writable key: a.keySet()) { assertTrue(b.containsKey(key)); // This will work because we know what we put into each set - WritableComparable aValue = (WritableComparable) a.get(key); - WritableComparable bValue = (WritableComparable) b.get(key); + WritableComparable> aValue = (WritableComparable>) a.get(key); + WritableComparable> bValue = (WritableComparable>) b.get(key); assertEquals(0, aValue.compareTo(bValue)); } } @@ -98,11 +99,11 @@ public class TestSortedMapWritable { @Test @SuppressWarnings("deprecation") public void testForeignClass() { - SortedMapWritable inMap = new SortedMapWritable(); + SortedMapWritable inMap = new SortedMapWritable(); inMap.put(new Text("key"), new UTF8("value")); inMap.put(new Text("key2"), new UTF8("value2")); - SortedMapWritable outMap = new SortedMapWritable(inMap); - SortedMapWritable copyOfCopy = new SortedMapWritable(outMap); + SortedMapWritable outMap = new SortedMapWritable(inMap); + SortedMapWritable copyOfCopy = new SortedMapWritable(outMap); assertEquals(1, copyOfCopy.getNewClasses()); } @@ -112,8 +113,8 @@ public class TestSortedMapWritable { @Test public void testEqualsAndHashCode() { String failureReason; - SortedMapWritable mapA = new SortedMapWritable(); - SortedMapWritable mapB = new SortedMapWritable(); + SortedMapWritable mapA = new SortedMapWritable(); + SortedMapWritable mapB = new SortedMapWritable(); // Sanity checks failureReason = "SortedMapWritable couldn't be initialized. Got null reference"; @@ -167,8 +168,8 @@ public class TestSortedMapWritable { @Test(timeout = 1000) public void testPutAll() { - SortedMapWritable map1 = new SortedMapWritable(); - SortedMapWritable map2 = new SortedMapWritable(); + SortedMapWritable map1 = new SortedMapWritable(); + SortedMapWritable map2 = new SortedMapWritable(); map1.put(new Text("key"), new Text("value")); map2.putAll(map1); http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc46c46b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableInput.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableInput.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableInput.java index a7ca12c..70b2100 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableInput.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableInput.java @@ -330,22 +330,25 @@ public class TypedBytesWritableInput implements Configurable { return readMap(null); } - public SortedMapWritable readSortedMap(SortedMapWritable mw) + public > + SortedMapWritable readSortedMap(SortedMapWritable mw) throws IOException { if (mw == null) { - mw = new SortedMapWritable(); + mw = new SortedMapWritable(); } int length = in.readMapHeader(); for (int i = 0; i < length; i++) { - WritableComparable key = (WritableComparable) read(); + @SuppressWarnings("unchecked") + K key = (K) read(); Writable value = read(); mw.put(key, value); } return mw; } - public SortedMapWritable readSortedMap() throws IOException { - return readSortedMap(null); + public > SortedMapWritable + readSortedMap() throws IOException { + return readSortedMap((SortedMapWritable)null); } public Writable readWritable(Writable writable) throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc46c46b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableOutput.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableOutput.java b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableOutput.java index 8dd0472..e7afe93 100644 --- a/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableOutput.java +++ b/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/typedbytes/TypedBytesWritableOutput.java @@ -131,7 +131,7 @@ public class TypedBytesWritableOutput { } else if (w instanceof MapWritable) { writeMap((MapWritable) w); } else if (w instanceof SortedMapWritable) { - writeSortedMap((SortedMapWritable) w); + writeSortedMap((SortedMapWritable) w); } else if (w instanceof Record) { writeRecord((Record) w); } else { @@ -200,9 +200,9 @@ public class TypedBytesWritableOutput { } } - public void writeSortedMap(SortedMapWritable smw) throws IOException { + public void writeSortedMap(SortedMapWritable smw) throws IOException { out.writeMapHeader(smw.size()); - for (Map.Entry entry : smw.entrySet()) { + for (Map.Entry, Writable> entry : smw.entrySet()) { write(entry.getKey()); write(entry.getValue()); }