zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From felixche...@apache.org
Subject zeppelin git commit: ZEPPELIN-1866. No meaningful exception for pig interpreter under tez engine
Date Thu, 05 Jan 2017 06:46:38 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/master a43815352 -> 1e7c05373


ZEPPELIN-1866. No meaningful exception for pig interpreter under tez engine

### What is this PR for?
This PR is to fix the bug that there's no meaningful exception when using tez engine.

### What type of PR is it?
[Bug Fix | Improvement ]

### Todos
* [ ] - Task

### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-1866

### How should this be tested?
Unit test is added and also manually test it.

### Screenshots (if appropriate)
![image](https://cloud.githubusercontent.com/assets/164491/21519099/9eb518d6-cd25-11e6-8bd9-38161173cb64.png)

### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No

Author: Jeff Zhang <zjffdu@apache.org>

Closes #1809 from zjffdu/ZEPPELIN-1866 and squashes the following commits:

34901ef [Jeff Zhang] ZEPPELIN-1866. No meaningful exception for pig interpreter under tez
engine


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/1e7c0537
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/1e7c0537
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/1e7c0537

Branch: refs/heads/master
Commit: 1e7c0537390f8630788db2006a5bafe359ef5a15
Parents: a438153
Author: Jeff Zhang <zjffdu@apache.org>
Authored: Wed Dec 28 17:44:47 2016 +0800
Committer: Felix Cheung <felixcheung@apache.org>
Committed: Wed Jan 4 22:46:23 2017 -0800

----------------------------------------------------------------------
 pig/pom.xml                                     |   7 +
 .../java/org/apache/zeppelin/pig/PigUtils.java  |   9 ++
 .../apache/zeppelin/pig/PigInterpreterTest.java |  13 +-
 .../zeppelin/pig/PigInterpreterTezTest.java     | 152 +++++++++++++++++++
 pig/src/test/resources/core-site.xml            |   3 +
 5 files changed, 177 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e7c0537/pig/pom.xml
----------------------------------------------------------------------
diff --git a/pig/pom.xml b/pig/pom.xml
index f8050cd..1dbfa14 100644
--- a/pig/pom.xml
+++ b/pig/pom.xml
@@ -177,6 +177,13 @@
                     </execution>
                 </executions>
             </plugin>
+
+            <plugin>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <forkMode>always</forkMode>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 </project>

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e7c0537/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
----------------------------------------------------------------------
diff --git a/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java b/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
index d444e02..3398281 100644
--- a/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
+++ b/pig/src/main/java/org/apache/zeppelin/pig/PigUtils.java
@@ -171,6 +171,15 @@ public class PigUtils {
   private static String extractFromTezPigStats(TezPigScriptStats stats) {
 
     try {
+      if (stats.getReturnCode() == PigRunner.ReturnCode.UNKNOWN) {
+        LOGGER.warn("unknown return code, can't display the results");
+        return null;
+      }
+      if (stats.getPigContext() == null) {
+        LOGGER.warn("unknown exec type, don't display the results");
+        return null;
+      }
+
       Field userIdField = PigStats.class.getDeclaredField("userId");
       userIdField.setAccessible(true);
       String userId = (String) (userIdField.get(stats));

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e7c0537/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
----------------------------------------------------------------------
diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
index 2f1aab2..efcbb58 100644
--- a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
+++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTest.java
@@ -41,10 +41,10 @@ public class PigInterpreterTest {
   private PigInterpreter pigInterpreter;
   private InterpreterContext context;
 
-  @Before
-  public void setUp() {
+  private void setUpLocal(boolean includeJobStats) {
     Properties properties = new Properties();
     properties.put("zeppelin.pig.execType", "local");
+    properties.put("zeppelin.pig.includeJobStats", includeJobStats + "");
     pigInterpreter = new PigInterpreter(properties);
     pigInterpreter.open();
     context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null,
null, null, null,
@@ -58,6 +58,8 @@ public class PigInterpreterTest {
 
   @Test
   public void testBasics() throws IOException {
+    setUpLocal(false);
+
     String content = "1\tandy\n"
             + "2\tpeter\n";
     File tmpFile = File.createTempFile("zeppelin", "test");
@@ -101,11 +103,7 @@ public class PigInterpreterTest {
 
   @Test
   public void testIncludeJobStats() throws IOException {
-    Properties properties = new Properties();
-    properties.put("zeppelin.pig.execType", "local");
-    properties.put("zeppelin.pig.includeJobStats", "true");
-    pigInterpreter = new PigInterpreter(properties);
-    pigInterpreter.open();
+    setUpLocal(true);
 
     String content = "1\tandy\n"
             + "2\tpeter\n";
@@ -152,4 +150,5 @@ public class PigInterpreterTest {
     assertTrue(result.message().get(0).getData().contains("Counters:"));
     assertTrue(result.message().get(0).getData().contains("Input path does not exist"));
   }
+
 }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e7c0537/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java
----------------------------------------------------------------------
diff --git a/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java
new file mode 100644
index 0000000..e742fd8
--- /dev/null
+++ b/pig/src/test/java/org/apache/zeppelin/pig/PigInterpreterTezTest.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.pig;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResult.Code;
+import org.apache.zeppelin.interpreter.InterpreterResult.Type;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Properties;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class PigInterpreterTezTest {
+
+  private PigInterpreter pigInterpreter;
+  private InterpreterContext context;
+
+  public void setUpTez(boolean includeJobStats) {
+    Properties properties = new Properties();
+    properties.put("zeppelin.pig.execType", "tez_local");
+    properties.put("zeppelin.pig.includeJobStats", includeJobStats + "");
+    pigInterpreter = new PigInterpreter(properties);
+    pigInterpreter.open();
+    context = new InterpreterContext(null, "paragraph_id", null, null, null, null, null,
null, null, null,
+        null, null);
+
+  }
+  @After
+  public void tearDown() {
+    pigInterpreter.close();
+  }
+
+  @Test
+  public void testBasics() throws IOException {
+    setUpTez(false);
+
+    String content = "1\tandy\n"
+        + "2\tpeter\n";
+    File tmpFile = File.createTempFile("zeppelin", "test");
+    FileWriter writer = new FileWriter(tmpFile);
+    IOUtils.write(content, writer);
+    writer.close();
+
+    // simple pig script using dump
+    String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+        + "dump a;";
+    InterpreterResult result = pigInterpreter.interpret(pigscript, context);
+    assertEquals(Type.TEXT, result.message().get(0).getType());
+    assertEquals(Code.SUCCESS, result.code());
+    assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)"));
+
+    // describe
+    pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+        + "describe a;";
+    result = pigInterpreter.interpret(pigscript, context);
+    assertEquals(Type.TEXT, result.message().get(0).getType());
+    assertEquals(Code.SUCCESS, result.code());
+    assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}"));
+
+    // syntax error (compilation error)
+    pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+        + "describe a;";
+    result = pigInterpreter.interpret(pigscript, context);
+    assertEquals(Type.TEXT, result.message().get(0).getType());
+    assertEquals(Code.ERROR, result.code());
+    assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol
at or near 'a'"));
+
+    // syntax error
+    pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+        + "foreach a generate $0;";
+    result = pigInterpreter.interpret(pigscript, context);
+    assertEquals(Type.TEXT, result.message().get(0).getType());
+    assertEquals(Code.ERROR, result.code());
+    assertTrue(result.message().get(0).getData().contains("expecting one of"));
+  }
+
+  @Test
+  public void testIncludeJobStats() throws IOException {
+    setUpTez(true);
+
+    String content = "1\tandy\n"
+        + "2\tpeter\n";
+    File tmpFile = File.createTempFile("zeppelin", "test");
+    FileWriter writer = new FileWriter(tmpFile);
+    IOUtils.write(content, writer);
+    writer.close();
+
+    // simple pig script using dump
+    String pigscript = "a = load '" + tmpFile.getAbsolutePath() + "';"
+        + "dump a;";
+    InterpreterResult result = pigInterpreter.interpret(pigscript, context);
+    assertEquals(Type.TEXT, result.message().get(0).getType());
+    assertEquals(Code.SUCCESS, result.code());
+    assertTrue(result.message().get(0).getData().contains("Vertex Stats"));
+    assertTrue(result.message().get(0).getData().contains("(1,andy)\n(2,peter)"));
+
+    // describe
+    pigscript = "a = load '" + tmpFile.getAbsolutePath() + "' as (id: int, name: bytearray);"
+        + "describe a;";
+    result = pigInterpreter.interpret(pigscript, context);
+    assertEquals(Type.TEXT, result.message().get(0).getType());
+    assertEquals(Code.SUCCESS, result.code());
+    // no job is launched, so no jobStats
+    assertTrue(!result.message().get(0).getData().contains("Vertex Stats"));
+    assertTrue(result.message().get(0).getData().contains("a: {id: int,name: bytearray}"));
+
+    // syntax error (compilation error)
+    pigscript = "a = loa '" + tmpFile.getAbsolutePath() + "';"
+        + "describe a;";
+    result = pigInterpreter.interpret(pigscript, context);
+    assertEquals(Type.TEXT, result.message().get(0).getType());
+    assertEquals(Code.ERROR, result.code());
+    // no job is launched, so no jobStats
+    assertTrue(!result.message().get(0).getData().contains("Vertex Stats"));
+    assertTrue(result.message().get(0).getData().contains("Syntax error, unexpected symbol
at or near 'a'"));
+
+    // execution error
+    pigscript = "a = load 'invalid_path';"
+        + "dump a;";
+    result = pigInterpreter.interpret(pigscript, context);
+    assertEquals(Type.TEXT, result.message().get(0).getType());
+    assertEquals(Code.ERROR, result.code());
+    assertTrue(!result.message().get(0).getData().contains("Vertex Stats"));
+    assertTrue(result.message().get(0).getData().contains("Input path does not exist"));
+  }
+}

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e7c0537/pig/src/test/resources/core-site.xml
----------------------------------------------------------------------
diff --git a/pig/src/test/resources/core-site.xml b/pig/src/test/resources/core-site.xml
new file mode 100644
index 0000000..f1fe563
--- /dev/null
+++ b/pig/src/test/resources/core-site.xml
@@ -0,0 +1,3 @@
+<configuration>
+
+</configuration>
\ No newline at end of file


Mime
View raw message