tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject tez git commit: TEZ-1861. Fix failing test: TestOnFileSortedOutput. (sseth) (cherry picked from commit e8d7dc9dc1463d5bb0e36144b0e9d0f9bab599ac)
Date Tue, 16 Dec 2014 21:54:32 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 8ddfd18c9 -> 880df51b0


TEZ-1861. Fix failing test: TestOnFileSortedOutput. (sseth)
(cherry picked from commit e8d7dc9dc1463d5bb0e36144b0e9d0f9bab599ac)

Conflicts:
	CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/880df51b
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/880df51b
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/880df51b

Branch: refs/heads/branch-0.5
Commit: 880df51b016bf94927f8213078e48a6847efaa49
Parents: 8ddfd18
Author: Siddharth Seth <sseth@apache.org>
Authored: Tue Dec 16 13:53:09 2014 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Tue Dec 16 13:54:22 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |   3 +-
 .../sort/impl/dflt/TestDefaultSorter.java       | 122 +++++++++++++++++++
 .../library/output/TestOnFileSortedOutput.java  |  27 ----
 3 files changed, 124 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/880df51b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 609df83..760fb60 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,8 +4,9 @@ Apache Tez Change Log
 Release 0.5.4: Unreleased
 
 ALL CHANGES:
-  TEZ-1800. Integer overflow in ExternalSorter.getInitialMemoryRequirement()
+  TEZ-1861. Fix failing test: TestOnFileSortedOutput.
   TEZ-1836. Provide better error messages when tez.runtime.io.sort.mb, spill percentage is
incorrectly configured.
+  TEZ-1800. Integer overflow in ExternalSorter.getInitialMemoryRequirement()
 
 Release 0.5.3: 2014-12-10
 

http://git-wip-us.apache.org/repos/asf/tez/blob/880df51b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
new file mode 100644
index 0000000..16dca55
--- /dev/null
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.runtime.library.common.sort.impl.dflt;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
+import org.apache.tez.common.TezRuntimeFrameworkConfigs;
+import org.apache.tez.common.TezUtils;
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.runtime.api.MemoryUpdateCallback;
+import org.apache.tez.runtime.api.OutputContext;
+import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
+import org.apache.tez.runtime.library.common.MemoryUpdateCallbackHandler;
+import org.apache.tez.runtime.library.common.shuffle.ShuffleUtils;
+import org.apache.tez.runtime.library.partitioner.HashPartitioner;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class TestDefaultSorter {
+
+  private Configuration conf;
+  private Path workingDir;
+  private static final int PORT = 80;
+  private static final String UniqueID = "UUID";
+
+  @Before
+  public void setup() throws IOException {
+    conf = new Configuration();
+    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_THREADS, 1); // DefaultSorter
+
+    workingDir = new Path(".", this.getClass().getName());
+    String localDirs = workingDir.toString();
+    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS, Text.class.getName());
+    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS, Text.class.getName());
+    conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS,
+        HashPartitioner.class.getName());
+    conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, localDirs);
+  }
+
+  @Test(timeout = 5000)
+  public void testSortSpillPercent() throws Exception {
+    OutputContext context = createTezOutputContext();
+
+    conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, 0.0f);
+    try {
+      new DefaultSorter(context, conf, 10, 2048);
+      fail();
+    } catch(IllegalArgumentException e) {
+      assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT));
+    }
+
+    conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, 1.1f);
+    try {
+      new DefaultSorter(context, conf, 10, 2048);
+      fail();
+    } catch(IllegalArgumentException e) {
+      assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT));
+    }
+  }
+
+  private OutputContext createTezOutputContext() throws IOException {
+    String[] workingDirs = { workingDir.toString() };
+    UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
+    DataOutputBuffer serviceProviderMetaData = new DataOutputBuffer();
+    serviceProviderMetaData.writeInt(PORT);
+
+    TezCounters counters = new TezCounters();
+
+    OutputContext context = mock(OutputContext.class);
+    doReturn(counters).when(context).getCounters();
+    doReturn(workingDirs).when(context).getWorkDirs();
+    doReturn(payLoad).when(context).getUserPayload();
+    doReturn(5 * 1024 * 1024l).when(context).getTotalMemoryAvailableToTask();
+    doReturn(UniqueID).when(context).getUniqueIdentifier();
+    doReturn("v1").when(context).getDestinationVertexName();
+    doReturn(ByteBuffer.wrap(serviceProviderMetaData.getData())).when(context)
+        .getServiceProviderMetaData
+            (ShuffleUtils.SHUFFLE_HANDLER_SERVICE_ID);
+    doAnswer(new Answer() {
+      @Override public Object answer(InvocationOnMock invocation) throws Throwable {
+        long requestedSize = (Long) invocation.getArguments()[0];
+        MemoryUpdateCallbackHandler callback = (MemoryUpdateCallbackHandler) invocation
+            .getArguments()[1];
+        callback.memoryAssigned(requestedSize);
+        return null;
+      }
+    }).when(context).requestInitialMemory(anyLong(), any(MemoryUpdateCallback.class));
+    return context;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/tez/blob/880df51b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
index afd0090..905fdb9 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
@@ -184,33 +184,6 @@ public class TestOnFileSortedOutput {
   }
 
   @Test
-  public void testSortSpillPercent() throws Exception {
-    OutputContext context = createTezOutputContext();
-    conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, 0.0f);
-    UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
-    doReturn(payLoad).when(context).getUserPayload();
-    sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
-    try {
-      sortedOutput.initialize();
-      sortedOutput.start();
-      fail();
-    } catch(IllegalArgumentException e) {
-      assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT));
-    }
-    conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, 1.1f);
-    payLoad = TezUtils.createUserPayloadFromConf(conf);
-    doReturn(payLoad).when(context).getUserPayload();
-    sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
-    try {
-      sortedOutput.initialize();
-      sortedOutput.start();
-      fail();
-    } catch(IllegalArgumentException e) {
-      assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT));
-    }
-  }
-
-  @Test
   public void baseTest() throws Exception {
     startSortedOutput(partitions);
 


Mime
View raw message