zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jongy...@apache.org
Subject [1/2] zeppelin git commit: Rewrite PythonInterpreter.
Date Sat, 18 Mar 2017 10:50:45 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/master 1b5c3a3f3 -> 287ffd50e


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/287ffd50/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
----------------------------------------------------------------------
diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
index bf50c23..b5cd680 100644
--- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
+++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java
@@ -1,19 +1,19 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package org.apache.zeppelin.python;
 
@@ -21,52 +21,39 @@ import static org.apache.zeppelin.python.PythonInterpreter.DEFAULT_ZEPPELIN_PYTH
 import static org.apache.zeppelin.python.PythonInterpreter.MAX_RESULT;
 import static org.apache.zeppelin.python.PythonInterpreter.ZEPPELIN_PYTHON;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
+import java.io.File;
 import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketAddress;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.HashMap;
 import java.util.LinkedList;
+import java.util.Map;
 import java.util.Properties;
-import java.util.concurrent.TimeUnit;
 
-import org.apache.zeppelin.interpreter.ClassloaderInterpreter;
+import org.apache.commons.exec.environment.EnvironmentUtils;
+import org.apache.zeppelin.display.AngularObjectRegistry;
+import org.apache.zeppelin.display.GUI;
 import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterContextRunner;
 import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterOutput;
+import org.apache.zeppelin.interpreter.InterpreterOutputListener;
 import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResultMessageOutput;
+import org.apache.zeppelin.resource.LocalResourcePool;
+import org.apache.zeppelin.user.AuthenticationInfo;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * Python interpreter unit test
- *
- * Important: ALL tests here DO NOT REQUIRE Python to be installed
- * If Python dependency is required, please look at PythonInterpreterWithPythonInstalledTest
- */
-public class PythonInterpreterTest {
-  private static final Logger LOG = LoggerFactory.getLogger(PythonProcess.class);
 
-  PythonInterpreter zeppelinPythonInterpreter = null;
+public class PythonInterpreterTest implements InterpreterOutputListener {
   PythonInterpreter pythonInterpreter = null;
-  PythonProcess mockPythonProcess;
   String cmdHistory;
+  private InterpreterContext context;
+  InterpreterOutput out;
 
   public static Properties getPythonTestProperties() {
     Properties p = new Properties();
@@ -79,18 +66,8 @@ public class PythonInterpreterTest {
   public void beforeTest() throws IOException {
     cmdHistory = "";
 
-    /*Mock python process*/
-    mockPythonProcess = mock(PythonProcess.class);
-    when(mockPythonProcess.getPid()).thenReturn(1L);
-    when(mockPythonProcess.sendAndGetResult(anyString())).thenAnswer(new Answer<String>()
{
-      @Override public String answer(InvocationOnMock invocationOnMock) throws Throwable
{
-        return answerFromPythonMock(invocationOnMock);
-      }
-    });
-
     // python interpreter
-    pythonInterpreter = spy(new PythonInterpreter(getPythonTestProperties()));
-    zeppelinPythonInterpreter = new PythonInterpreter(getPythonTestProperties());
+    pythonInterpreter = new PythonInterpreter(getPythonTestProperties());
 
     // create interpreter group
     InterpreterGroup group = new InterpreterGroup();
@@ -98,186 +75,49 @@ public class PythonInterpreterTest {
     group.get("note").add(pythonInterpreter);
     pythonInterpreter.setInterpreterGroup(group);
 
-    when(pythonInterpreter.getPythonProcess()).thenReturn(mockPythonProcess);
-    when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("ImportError");
-  }
-
-  @After
-  public void afterTest() throws IOException {
-    pythonInterpreter.close();
-    zeppelinPythonInterpreter.close();
-  }
-
-  @Test
-  public void testOpenInterpreter() {
-    pythonInterpreter.open();
-    assertEquals(pythonInterpreter.getPythonProcess().getPid(), 1);
-  }
-
-  /**
-   * If Py4J is not installed, bootstrap_input.py
-   * is not sent to Python process and py4j JavaGateway is not running
-   */
-  @Test
-  public void testPy4jIsNotInstalled() {
-    pythonInterpreter.open();
-    assertNull(pythonInterpreter.getPy4jPort());
-    assertTrue(cmdHistory.contains("def help()"));
-    assertTrue(cmdHistory.contains("class PyZeppelinContext(object):"));
-    assertTrue(cmdHistory.contains("z = PyZeppelinContext"));
-    assertTrue(cmdHistory.contains("def show"));
-    assertFalse(cmdHistory.contains("GatewayClient"));
-  }
-
-  /**
-   * If Py4J installed, bootstrap_input.py
-   * is sent to interpreter and JavaGateway is running
-   */
-  @Test
-  public void testPy4jInstalled() throws IOException, InterruptedException {
-    when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("");
+    out = new InterpreterOutput(this);
 
+    context = new InterpreterContext("note", "id", null, "title", "text",
+        new AuthenticationInfo(),
+        new HashMap<String, Object>(),
+        new GUI(),
+        new AngularObjectRegistry(group.getId(), null),
+        new LocalResourcePool("id"),
+        new LinkedList<InterpreterContextRunner>(),
+        out);
     pythonInterpreter.open();
-    Integer py4jPort = pythonInterpreter.getPy4jPort();
-    assertNotNull(py4jPort);
-
-    assertTrue(cmdHistory.contains("def help()"));
-    assertTrue(cmdHistory.contains("class PyZeppelinContext(object):"));
-    assertTrue(cmdHistory.contains("z = Py4jZeppelinContext"));
-    assertTrue(cmdHistory.contains("def show"));
-    assertTrue(cmdHistory.contains("GatewayClient(port=" + py4jPort + ")"));
-    assertTrue(cmdHistory.contains("org.apache.zeppelin.display.Input"));
-
-    assertTrue(serverIsListeningOn(py4jPort));
-    pythonInterpreter.close();
-    TimeUnit.MILLISECONDS.sleep(100);
-    assertFalse(serverIsListeningOn(py4jPort));
   }
 
-  @Test
-  public void testClose() throws IOException, InterruptedException {
-    //given: py4j is installed
-    when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn("");
-
-    pythonInterpreter.open();
-    Integer py4jPort = pythonInterpreter.getPy4jPort();
-    assertNotNull(py4jPort);
-
-    //when
+  @After
+  public void afterTest() throws IOException {
     pythonInterpreter.close();
-    TimeUnit.MILLISECONDS.sleep(100);
-
-    //then
-    assertFalse(serverIsListeningOn(py4jPort));
-    verify(mockPythonProcess, times(1)).close();
   }
 
   @Test
-  public void testInterpret() {
-    pythonInterpreter.open();
-    cmdHistory = "";
-    InterpreterResult result = pythonInterpreter.interpret("print a", null);
+  public void testInterpret() throws InterruptedException, IOException {
+    InterpreterResult result = pythonInterpreter.interpret("print (\"hi\")", context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
-    assertEquals("%text print a", result.message().get(0).toString());
   }
 
   @Test
-  public void testInterpretInvalidSyntax() {
-    zeppelinPythonInterpreter.open();
-    InterpreterResult result = zeppelinPythonInterpreter.interpret("for x in range(0,3):
 print (\"hi\")\n\nz._displayhook()", null);
+  public void testInterpretInvalidSyntax() throws IOException {
+    InterpreterResult result = pythonInterpreter.interpret("for x in range(0,3):  print (\"hi\")\n",
context);
     assertEquals(InterpreterResult.Code.SUCCESS, result.code());
-    assertTrue(result.message().get(0).toString().contains("hi\nhi\nhi"));
-
-    result = zeppelinPythonInterpreter.interpret("for x in range(0,3):  print (\"hi\")\nz._displayhook()",
null);
-    assertEquals(InterpreterResult.Code.ERROR, result.code());
-    assertTrue(result.message().get(0).toString().contains("SyntaxError: invalid syntax"));
-  }
-
-  /**
-   * Checks if given port is open on 'localhost'
-   * @param port
-   */
-  private boolean serverIsListeningOn(Integer port) {
-    Socket s = new Socket();
-    boolean serverIsListening = false;
+    assertTrue(new String(out.getOutputAt(0).toByteArray()).contains("hi\nhi\nhi"));
+ }
 
-    int retryCount = 0;
-    boolean connected = false;
-    while (connected = tryToConnect(s, port) && retryCount < 10) {
-      serverIsListening = connected;
-      tryToClose(s);
-      retryCount++;
-      s = new Socket();
-    }
-    return serverIsListening;
-  }
-
-  private boolean tryToConnect(Socket s, Integer port) {
-    boolean connected = false;
-    SocketAddress sa = new InetSocketAddress("localhost", port);
-    try {
-      s.connect(sa, 10000);
-      connected = true;
-    } catch (IOException e) {
-      //LOG.warn("Can't open connection to " + sa, e);
-    }
-    return connected;
-  }
+  @Override
+  public void onUpdateAll(InterpreterOutput out) {
 
-  private void tryToClose(Socket s) {
-    try {
-      s.close();
-    } catch (IOException e) {
-      LOG.error("Can't close connection to " + s.getInetAddress(), e);
-    }
   }
 
-  private String answerFromPythonMock(InvocationOnMock invocationOnMock) {
-    Object[] inputs = invocationOnMock.getArguments();
-    String cmdToExecute = (String) inputs[0];
+  @Override
+  public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {
 
-    if (cmdToExecute != null) {
-      cmdHistory += cmdToExecute;
-      String[] lines = cmdToExecute.split("\\n");
-      String output = "";
-
-      for (int i = 0; i < lines.length; i++) {
-        output += lines[i];
-      }
-      return output;
-    } else {
-      return "";
-    }
   }
 
-  @Test
-  public void checkMultiRowErrorFails() {
-
-    PythonInterpreter pythonInterpreter = new PythonInterpreter(
-      PythonInterpreterTest.getPythonTestProperties()
-    );
-    // create interpreter group
-    InterpreterGroup group = new InterpreterGroup();
-    group.put("note", new LinkedList<Interpreter>());
-    group.get("note").add(pythonInterpreter);
-    pythonInterpreter.setInterpreterGroup(group);
-
-    pythonInterpreter.open();
-
-    String codeRaiseException = "raise Exception(\"test exception\")";
-    InterpreterResult ret = pythonInterpreter.interpret(codeRaiseException, null);
+  @Override
+  public void onUpdate(int index, InterpreterResultMessageOutput out) {
 
-    assertNotNull("Interpreter result for raise exception is Null", ret);
-
-    System.err.println("ret = '" + ret + "'");
-    assertEquals(InterpreterResult.Code.ERROR, ret.code());
-    assertTrue(ret.message().get(0).getData().length() > 0);
-
-    assertNotNull("Interpreter result for text is Null", ret);
-    String codePrintText = "print (\"Exception(\\\"test exception\\\")\")";
-    ret = pythonInterpreter.interpret(codePrintText, null);
-    assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
-    assertTrue(ret.message().get(0).getData().length() > 0);
   }
-
 }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/287ffd50/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterWithPythonInstalledTest.java
----------------------------------------------------------------------
diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterWithPythonInstalledTest.java
b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterWithPythonInstalledTest.java
deleted file mode 100644
index 7b889ad..0000000
--- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterWithPythonInstalledTest.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*  http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.zeppelin.python;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import org.apache.zeppelin.interpreter.Interpreter;
-import org.apache.zeppelin.interpreter.InterpreterGroup;
-import org.apache.zeppelin.interpreter.InterpreterResult;
-import org.junit.Test;
-
-import java.util.Arrays;
-
-/**
- * Python interpreter unit test that user real Python
- *
- * Important: ALL tests here REQUIRE Python to be installed
- * They are excluded from default build, to run them manually do:
- *
- * <code>
- * mvn "-Dtest=org.apache.zeppelin.python.PythonInterpreterWithPythonInstalledTest" test
-pl python
- * </code>
- *
- * or
- * <code>
- * mvn -Dpython.test.exclude='' test -pl python -am
- * </code>
- */
-public class PythonInterpreterWithPythonInstalledTest {
-
-  @Test
-  public void badPythonSyntaxFails() {
-    //given
-    PythonInterpreter realPython = new PythonInterpreter(
-        PythonInterpreterTest.getPythonTestProperties());
-    // create interpreter group
-    InterpreterGroup group = new InterpreterGroup();
-    group.put("note", Arrays.asList((Interpreter) realPython));
-    realPython.setInterpreterGroup(group);
-
-    realPython.open();
-
-    //when
-    InterpreterResult ret = realPython.interpret("select wrong syntax", null);
-
-    //then
-    assertNotNull("Interpreter returned 'null'", ret);
-    //System.out.println("\nInterpreter response: \n" + ret.message());
-    assertEquals(InterpreterResult.Code.ERROR, ret.code());
-    assertTrue(ret.message().get(0).getData().length() > 0);
-
-    realPython.close();
-  }
-
-  @Test
-  public void goodPythonSyntaxRuns() {
-    //given
-    PythonInterpreter realPython = new PythonInterpreter(
-        PythonInterpreterTest.getPythonTestProperties());
-    InterpreterGroup group = new InterpreterGroup();
-    group.put("note", Arrays.asList((Interpreter) realPython));
-    realPython.setInterpreterGroup(group);
-    realPython.open();
-
-    //when
-    InterpreterResult ret = realPython.interpret("help()", null);
-
-    //then
-    assertNotNull("Interpreter returned 'null'", ret);
-    //System.out.println("\nInterpreter response: \n" + ret.message());
-    assertEquals(InterpreterResult.Code.SUCCESS, ret.code());
-    assertTrue(ret.message().get(0).getData().length() > 0);
-
-    realPython.close();
-  }
-
-  @Test
-  public void testZeppelin1555() {
-    //given
-    PythonInterpreter realPython = new PythonInterpreter(
-            PythonInterpreterTest.getPythonTestProperties());
-    InterpreterGroup group = new InterpreterGroup();
-    group.put("note", Arrays.asList((Interpreter) realPython));
-    realPython.setInterpreterGroup(group);
-    realPython.open();
-
-    //when
-    InterpreterResult ret1 = realPython.interpret("print(\"...\")", null);
-
-    //then
-    //System.out.println("\nInterpreter response: \n" + ret.message());
-    assertEquals(InterpreterResult.Code.SUCCESS, ret1.code());
-    assertEquals("...\n", ret1.message().get(0).getData());
-
-
-    InterpreterResult ret2 = realPython.interpret("for i in range(5):", null);
-    //then
-    //System.out.println("\nInterpreterResultterpreter response: \n" + ret2.message());
-    assertEquals(InterpreterResult.Code.ERROR, ret2.code());
-    assertEquals("   File \"<stdin>\", line 2\n" +
-            "    \n" +
-            "    ^\n" +
-            "IndentationError: expected an indented block\n", ret2.message().get(0).getData());
-
-    realPython.close();
-  }
-
-}


Mime
View raw message