From commits-return-4882-archive-asf-public=cust-asf.ponee.io@zeppelin.apache.org Wed Aug 29 12:07:10 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id B18231807B5 for ; Wed, 29 Aug 2018 12:07:07 +0200 (CEST) Received: (qmail 12091 invoked by uid 500); 29 Aug 2018 10:07:00 -0000 Mailing-List: contact commits-help@zeppelin.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@zeppelin.apache.org Delivered-To: mailing list commits@zeppelin.apache.org Received: (qmail 9972 invoked by uid 99); 29 Aug 2018 10:06:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 29 Aug 2018 10:06:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 5D184E11BE; Wed, 29 Aug 2018 10:06:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jongyoul@apache.org To: commits@zeppelin.apache.org Date: Wed, 29 Aug 2018 10:07:36 -0000 Message-Id: In-Reply-To: <1d92b8c95bd74d6f86837ffa5986b9fa@git.apache.org> References: <1d92b8c95bd74d6f86837ffa5986b9fa@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [40/50] [abbrv] zeppelin git commit: Revert "[ZEPPELIN-3740] Adopt `google-java-format` and `fmt-maven-plugin`" http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java index 43639f7..5eb34e4 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonCondaInterpreter.java @@ -16,6 +16,18 @@ */ package org.apache.zeppelin.python; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.InterpreterResult.Type; +import org.apache.zeppelin.scheduler.Scheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -28,19 +40,11 @@ import java.util.Map; import java.util.Properties; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.lang.StringUtils; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterOutput; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.InterpreterResult.Type; -import org.apache.zeppelin.scheduler.Scheduler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -/** Conda support TODO(zjffdu) Add removing conda env */ +/** + * Conda support + * TODO(zjffdu) Add removing conda env + */ public class PythonCondaInterpreter extends Interpreter { private static Logger logger = LoggerFactory.getLogger(PythonCondaInterpreter.class); public static final String ZEPPELIN_PYTHON = "zeppelin.python"; @@ -66,10 +70,14 @@ public class PythonCondaInterpreter extends Interpreter { } @Override - public void open() throws InterpreterException {} + public void open() throws InterpreterException { + + } @Override - public void close() {} + public void close() { + + } @Override public InterpreterResult interpret(String st, InterpreterContext context) @@ -159,6 +167,7 @@ public class PythonCondaInterpreter extends Interpreter { getInterpreterInTheSameSessionByClassName(PythonInterpreter.class, false); pythonInterpreter.close(); pythonInterpreter.open(); + } public static String runCondaCommandForTextOutput(String title, List commands) @@ -179,7 +188,8 @@ public class PythonCondaInterpreter extends Interpreter { return wrapCondaTableOutputStyle(title, envPerName); } - protected Map getCondaEnvs() throws IOException, InterruptedException { + protected Map getCondaEnvs() + throws IOException, InterruptedException { String result = runCommand("conda", "env", "list"); Map envList = parseCondaCommonStdout(result); return envList; @@ -189,7 +199,8 @@ public class PythonCondaInterpreter extends Interpreter { return wrapCondaTableOutputStyle("Environment List", getCondaEnvs()); } - private String runCondaEnv(List restArgs) throws IOException, InterruptedException { + private String runCondaEnv(List restArgs) + throws IOException, InterruptedException { restArgs.add(0, "conda"); restArgs.add(1, "env"); @@ -248,7 +259,8 @@ public class PythonCondaInterpreter extends Interpreter { return runCondaCommandForTextOutput("Conda Information", commands); } - private String runCondaCreate(List restArgs) throws IOException, InterruptedException { + private String runCondaCreate(List restArgs) + throws IOException, InterruptedException { restArgs.add(0, "conda"); restArgs.add(1, "create"); restArgs.add(2, "--yes"); @@ -256,7 +268,8 @@ public class PythonCondaInterpreter extends Interpreter { return runCondaCommandForTextOutput("Environment Creation", restArgs); } - private String runCondaInstall(List restArgs) throws IOException, InterruptedException { + private String runCondaInstall(List restArgs) + throws IOException, InterruptedException { restArgs.add(0, "conda"); restArgs.add(1, "install"); @@ -269,7 +282,8 @@ public class PythonCondaInterpreter extends Interpreter { return runCondaCommandForTextOutput("Package Installation", restArgs); } - private String runCondaUninstall(List restArgs) throws IOException, InterruptedException { + private String runCondaUninstall(List restArgs) + throws IOException, InterruptedException { restArgs.add(0, "conda"); restArgs.add(1, "uninstall"); @@ -285,9 +299,12 @@ public class PythonCondaInterpreter extends Interpreter { public static String wrapCondaBasicOutputStyle(String title, String content) { StringBuilder sb = new StringBuilder(); if (null != title && !title.isEmpty()) { - sb.append("

").append(title).append("

\n").append("
\n"); + sb.append("

").append(title).append("

\n") + .append("
\n"); } - sb.append("
\n").append(content).append("
"); + sb.append("
\n") + .append(content) + .append("
"); return sb.toString(); } @@ -303,13 +320,11 @@ public class PythonCondaInterpreter extends Interpreter { for (String name : kv.keySet()) { String path = kv.get(name); - sb.append( - String.format( - "
" - + "
%s
" - + "
%s
" - + "
\n", - name, path)); + sb.append(String.format("
" + + "
%s
" + + "
%s
" + + "
\n", + name, path)); } sb.append("\n"); @@ -337,7 +352,9 @@ public class PythonCondaInterpreter extends Interpreter { } @Override - public void cancel(InterpreterContext context) {} + public void cancel(InterpreterContext context) { + + } @Override public FormType getFormType() { @@ -350,8 +367,8 @@ public class PythonCondaInterpreter extends Interpreter { } /** - * Use python interpreter's scheduler. To make sure %python.conda paragraph and %python paragraph - * runs sequentially + * Use python interpreter's scheduler. + * To make sure %python.conda paragraph and %python paragraph runs sequentially */ @Override public Scheduler getScheduler() { @@ -364,7 +381,8 @@ public class PythonCondaInterpreter extends Interpreter { } } - public static String runCommand(List commands) throws IOException, InterruptedException { + public static String runCommand(List commands) + throws IOException, InterruptedException { logger.info("Starting shell commands: " + StringUtils.join(commands, " ")); Process process = Runtime.getRuntime().exec(commands.toArray(new String[0])); StreamGobbler errorGobbler = new StreamGobbler(process.getErrorStream()); @@ -411,7 +429,8 @@ public class PythonCondaInterpreter extends Interpreter { } } - public static String runCommand(String... command) throws IOException, InterruptedException { + public static String runCommand(String... command) + throws IOException, InterruptedException { List list = new ArrayList<>(command.length); for (String arg : command) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java index 1774187..52caf35 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonDockerInterpreter.java @@ -16,6 +16,15 @@ */ package org.apache.zeppelin.python; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.scheduler.Scheduler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -25,16 +34,10 @@ import java.nio.file.Paths; import java.util.Properties; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContext; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterOutput; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.scheduler.Scheduler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -/** Helps run python interpreter on a docker container */ +/** + * Helps run python interpreter on a docker container + */ public class PythonDockerInterpreter extends Interpreter { Logger logger = LoggerFactory.getLogger(PythonDockerInterpreter.class); Pattern activatePattern = Pattern.compile("activate\\s*(.*)"); @@ -58,7 +61,9 @@ public class PythonDockerInterpreter extends Interpreter { } @Override - public void close() {} + public void close() { + + } @Override public InterpreterResult interpret(String st, InterpreterContext context) @@ -78,26 +83,23 @@ public class PythonDockerInterpreter extends Interpreter { pull(out, image); // mount pythonscript dir - String mountPythonScript = "-v " + pythonWorkDir.getAbsolutePath() + ":/_python_workdir "; + String mountPythonScript = "-v " + pythonWorkDir.getAbsolutePath() + + ":/_python_workdir "; // mount zeppelin dir - String mountPy4j = "-v " + zeppelinHome.getAbsolutePath() + ":/_zeppelin "; + String mountPy4j = "-v " + zeppelinHome.getAbsolutePath() + + ":/_zeppelin "; // set PYTHONPATH String pythonPath = ".:/_python_workdir/py4j-src-0.10.7.zip:/_python_workdir"; - setPythonCommand( - "docker run -i --rm " - + mountPythonScript - + mountPy4j - + "-e PYTHONPATH=\"" - + pythonPath - + "\" " - + image - + " " - + pythonInterpreter.getPythonExec() - + " " - + "/_python_workdir/zeppelin_python.py"); + setPythonCommand("docker run -i --rm " + + mountPythonScript + + mountPy4j + + "-e PYTHONPATH=\"" + pythonPath + "\" " + + image + " " + + pythonInterpreter.getPythonExec() + " " + + "/_python_workdir/zeppelin_python.py"); restartPythonProcess(); out.clear(); return new InterpreterResult(InterpreterResult.Code.SUCCESS, "\"" + image + "\" activated"); @@ -110,6 +112,7 @@ public class PythonDockerInterpreter extends Interpreter { } } + public void setPythonCommand(String cmd) throws InterpreterException { pythonInterpreter.setPythonExec(cmd); } @@ -124,7 +127,9 @@ public class PythonDockerInterpreter extends Interpreter { } @Override - public void cancel(InterpreterContext context) {} + public void cancel(InterpreterContext context) { + + } @Override public FormType getFormType() { @@ -137,8 +142,8 @@ public class PythonDockerInterpreter extends Interpreter { } /** - * Use python interpreter's scheduler. To make sure %python.docker paragraph and %python paragraph - * runs sequentially + * Use python interpreter's scheduler. + * To make sure %python.docker paragraph and %python paragraph runs sequentially */ @Override public Scheduler getScheduler() { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java index ebab2db..fb4ba9c 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java @@ -19,14 +19,6 @@ package org.apache.zeppelin.python; import com.google.common.io.Files; import com.google.gson.Gson; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.DefaultExecutor; import org.apache.commons.exec.ExecuteException; @@ -53,10 +45,19 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import py4j.GatewayServer; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; + /** * Interpreter for Python, it is the first implementation of interpreter for Python, so with less - * features compared to IPythonInterpreter, but requires less prerequisites than IPythonInterpreter, - * only python installation is required. + * features compared to IPythonInterpreter, but requires less prerequisites than + * IPythonInterpreter, only python installation is required. */ public class PythonInterpreter extends Interpreter implements ExecuteResultHandler { private static final Logger LOGGER = LoggerFactory.getLogger(PythonInterpreter.class); @@ -74,7 +75,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl private long pythonPid = -1; private IPythonInterpreter iPythonInterpreter; private BaseZeppelinContext zeppelinContext; - private String condaPythonExec; // set by PythonCondaInterpreter + private String condaPythonExec; // set by PythonCondaInterpreter private boolean usePy4jAuth = false; public PythonInterpreter(Properties property) { @@ -85,8 +86,9 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl public void open() throws InterpreterException { // try IPythonInterpreter first iPythonInterpreter = getIPythonInterpreter(); - if (getProperty("zeppelin.python.useIPython", "true").equals("true") - && StringUtils.isEmpty(iPythonInterpreter.checkIPythonPrerequisite(getPythonExec()))) { + if (getProperty("zeppelin.python.useIPython", "true").equals("true") && + StringUtils.isEmpty( + iPythonInterpreter.checkIPythonPrerequisite(getPythonExec()))) { try { iPythonInterpreter.open(); LOGGER.info("IPython is available, Use IPythonInterpreter to replace PythonInterpreter"); @@ -128,8 +130,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl // container can also connect to this gateway server. String serverAddress = PythonUtils.getLocalIP(properties); String secret = PythonUtils.createSecret(256); - this.gatewayServer = - PythonUtils.createGatewayServer(this, serverAddress, port, secret, usePy4jAuth); + this.gatewayServer = PythonUtils.createGatewayServer(this, serverAddress, port, secret, + usePy4jAuth); gatewayServer.start(); // launch python process to connect to the gateway server in JVM side @@ -152,15 +154,14 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl if (usePy4jAuth) { env.put("PY4J_GATEWAY_SECRET", secret); } - LOGGER.info( - "Launching Python Process Command: " - + cmd.getExecutable() - + " " - + StringUtils.join(cmd.getArguments(), " ")); + LOGGER.info("Launching Python Process Command: " + cmd.getExecutable() + + " " + StringUtils.join(cmd.getArguments(), " ")); executor.execute(cmd, env, this); pythonScriptRunning.set(true); } + + private void createPythonScript() throws IOException { // set java.io.tmpdir to /tmp on MacOS, because docker can not share the /var folder which will // cause PythonDockerInterpreter fails. @@ -183,12 +184,14 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl return this.iPythonInterpreter != null; } - private void copyResourceToPythonWorkDir(String srcResourceName, String dstFileName) - throws IOException { + private void copyResourceToPythonWorkDir(String srcResourceName, + String dstFileName) throws IOException { FileOutputStream out = null; try { out = new FileOutputStream(pythonWorkDir.getAbsoluteFile() + "/" + dstFileName); - IOUtils.copy(getClass().getClassLoader().getResourceAsStream(srcResourceName), out); + IOUtils.copy( + getClass().getClassLoader().getResourceAsStream(srcResourceName), + out); } finally { if (out != null) { out.close(); @@ -258,7 +261,9 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl this.condaPythonExec = pythonExec; } - /** Request send to Python Daemon */ + /** + * Request send to Python Daemon + */ public class PythonInterpretRequest { public String statements; public boolean isForCompletion; @@ -330,7 +335,9 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl } // used by subclass such as PySparkInterpreter to set JobGroup before executing spark code - protected void preCallPython(InterpreterContext context) {} + protected void preCallPython(InterpreterContext context) { + + } // blocking call. Send python code to python process and get response protected void callPython(PythonInterpretRequest request) { @@ -359,8 +366,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl } if (!pythonScriptRunning.get()) { - return new InterpreterResult( - Code.ERROR, "python process not running " + outputStream.toString()); + return new InterpreterResult(Code.ERROR, "python process not running " + + outputStream.toString()); } outputStream.setInterpreterOutput(context.out); @@ -388,8 +395,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl if (!pythonScriptInitialized.get()) { // timeout. didn't get initialized message - errorMessage.add( - new InterpreterResultMessage(InterpreterResult.Type.TEXT, "Failed to initialize Python")); + errorMessage.add(new InterpreterResultMessage( + InterpreterResult.Type.TEXT, "Failed to initialize Python")); return new InterpreterResult(Code.ERROR, errorMessage); } @@ -450,9 +457,11 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl return 0; } + @Override - public List completion( - String buf, int cursor, InterpreterContext interpreterContext) throws InterpreterException { + public List completion(String buf, int cursor, + InterpreterContext interpreterContext) + throws InterpreterException { if (iPythonInterpreter != null) { return iPythonInterpreter.completion(buf, cursor, interpreterContext); } @@ -473,7 +482,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl String[] completionList = null; synchronized (statementFinishedNotifier) { long startTime = System.currentTimeMillis(); - while (statementOutput == null && pythonScriptRunning.get()) { + while (statementOutput == null + && pythonScriptRunning.get()) { try { if (System.currentTimeMillis() - startTime > MAX_TIMEOUT_SEC * 1000) { LOGGER.error("Python completion didn't have response for {}sec.", MAX_TIMEOUT_SEC); @@ -492,7 +502,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl Gson gson = new Gson(); completionList = gson.fromJson(statementOutput, String[].class); } - // end code for completion + //end code for completion if (completionList == null) { return new LinkedList<>(); } @@ -528,6 +538,7 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl if (indexOfReverseSeqPostion < completionStartPosition && indexOfReverseSeqPostion > 0) { completionStartPosition = indexOfReverseSeqPostion; } + } if (completionStartPosition == completionEndPosition) { @@ -535,8 +546,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl } else { completionStartPosition = completionEndPosition - completionStartPosition; } - resultCompletionText = - completionScriptText.substring(completionStartPosition, completionEndPosition); + resultCompletionText = completionScriptText.substring( + completionStartPosition, completionEndPosition); return resultCompletionText; } @@ -564,8 +575,8 @@ public class PythonInterpreter extends Interpreter implements ExecuteResultHandl IOUtils.toString(getClass().getClassLoader().getResourceAsStream(resourceName)); try { // Add hook explicitly, otherwise python will fail to execute the statement - InterpreterResult result = - interpret(bootstrapCode + "\n" + "__zeppelin__._displayhook()", InterpreterContext.get()); + InterpreterResult result = interpret(bootstrapCode + "\n" + "__zeppelin__._displayhook()", + InterpreterContext.get()); if (result.code() != Code.SUCCESS) { throw new IOException("Fail to run bootstrap script: " + resourceName); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java index fb812bd..4fccc3c 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreterPandasSql.java @@ -17,8 +17,6 @@ package org.apache.zeppelin.python; -import java.io.IOException; -import java.util.Properties; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -26,10 +24,13 @@ import org.apache.zeppelin.interpreter.InterpreterResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Properties; + /** * SQL over Pandas DataFrame interpreter for %python group - * - *

Match experience of %sparpk.sql over Spark DataFrame + *

+ * Match experience of %sparpk.sql over Spark DataFrame */ public class PythonInterpreterPandasSql extends Interpreter { private static final Logger LOG = LoggerFactory.getLogger(PythonInterpreterPandasSql.class); @@ -72,7 +73,9 @@ public class PythonInterpreterPandasSql extends Interpreter { } @Override - public void cancel(InterpreterContext context) {} + public void cancel(InterpreterContext context) { + + } @Override public FormType getFormType() { @@ -83,4 +86,5 @@ public class PythonInterpreterPandasSql extends Interpreter { public int getProgress(InterpreterContext context) { return 0; } + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/main/java/org/apache/zeppelin/python/PythonUtils.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonUtils.java b/python/src/main/java/org/apache/zeppelin/python/PythonUtils.java index 3cf2493..996518b 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonUtils.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonUtils.java @@ -17,6 +17,11 @@ package org.apache.zeppelin.python; +import org.apache.commons.codec.binary.Base64; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import py4j.GatewayServer; + import java.io.IOException; import java.net.Inet4Address; import java.net.InetAddress; @@ -24,36 +29,28 @@ import java.net.UnknownHostException; import java.security.SecureRandom; import java.util.List; import java.util.Properties; -import org.apache.commons.codec.binary.Base64; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import py4j.GatewayServer; public class PythonUtils { private static final Logger LOGGER = LoggerFactory.getLogger(PythonUtils.class); - public static GatewayServer createGatewayServer( - Object entryPoint, String serverAddress, int port, String secretKey, boolean useAuth) - throws IOException { - LOGGER.info( - "Launching GatewayServer at " + serverAddress + ":" + port + ", useAuth: " + useAuth); + public static GatewayServer createGatewayServer(Object entryPoint, + String serverAddress, + int port, + String secretKey, + boolean useAuth) throws IOException { + LOGGER.info("Launching GatewayServer at " + serverAddress + ":" + port + + ", useAuth: " + useAuth); if (useAuth) { try { - Class clz = - Class.forName( - "py4j.GatewayServer$GatewayServerBuilder", - true, - Thread.currentThread().getContextClassLoader()); + Class clz = Class.forName("py4j.GatewayServer$GatewayServerBuilder", true, + Thread.currentThread().getContextClassLoader()); Object builder = clz.getConstructor(Object.class).newInstance(entryPoint); builder.getClass().getMethod("authToken", String.class).invoke(builder, secretKey); builder.getClass().getMethod("javaPort", int.class).invoke(builder, port); - builder - .getClass() - .getMethod("javaAddress", InetAddress.class) - .invoke(builder, InetAddress.getByName(serverAddress)); - builder - .getClass() + builder.getClass().getMethod("javaAddress", InetAddress.class).invoke(builder, + InetAddress.getByName(serverAddress)); + builder.getClass() .getMethod("callbackClient", int.class, InetAddress.class, String.class) .invoke(builder, port, InetAddress.getByName(serverAddress), secretKey); return (GatewayServer) builder.getClass().getMethod("build").invoke(builder); @@ -61,8 +58,7 @@ public class PythonUtils { throw new IOException(e); } } else { - return new GatewayServer( - entryPoint, + return new GatewayServer(entryPoint, port, GatewayServer.DEFAULT_PYTHON_PORT, InetAddress.getByName(serverAddress), @@ -76,7 +72,8 @@ public class PythonUtils { public static String getLocalIP(Properties properties) { // zeppelin.python.gatewayserver_address is only for unit test on travis. // Because the FQDN would fail unit test on travis ci. - String gatewayserver_address = properties.getProperty("zeppelin.python.gatewayserver_address"); + String gatewayserver_address = + properties.getProperty("zeppelin.python.gatewayserver_address"); if (gatewayserver_address != null) { return gatewayserver_address; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java ---------------------------------------------------------------------- diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java b/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java index 57b41bf..526784e 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonZeppelinContext.java @@ -17,12 +17,15 @@ package org.apache.zeppelin.python; -import java.util.List; -import java.util.Map; import org.apache.zeppelin.interpreter.BaseZeppelinContext; import org.apache.zeppelin.interpreter.InterpreterHookRegistry; -/** ZeppelinContext for Python */ +import java.util.List; +import java.util.Map; + +/** + * ZeppelinContext for Python + */ public class PythonZeppelinContext extends BaseZeppelinContext { public PythonZeppelinContext(InterpreterHookRegistry hooks, int maxResult) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java index 81189e7..a51c053 100644 --- a/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/BasePythonInterpreterTest.java @@ -17,12 +17,6 @@ package org.apache.zeppelin.python; -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; - -import java.io.IOException; -import java.util.List; import org.apache.zeppelin.display.ui.CheckBox; import org.apache.zeppelin.display.ui.Password; import org.apache.zeppelin.display.ui.Select; @@ -40,6 +34,13 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.util.List; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + public abstract class BasePythonInterpreterTest { protected InterpreterGroup intpGroup; @@ -51,11 +52,13 @@ public abstract class BasePythonInterpreterTest { @After public abstract void tearDown() throws InterpreterException; + @Test public void testPythonBasics() throws InterpreterException, InterruptedException, IOException { InterpreterContext context = getInterpreterContext(); - InterpreterResult result = interpreter.interpret("import sys\nprint(sys.version[0])", context); + InterpreterResult result = + interpreter.interpret("import sys\nprint(sys.version[0])", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); Thread.sleep(100); List interpreterResultMessages = @@ -142,8 +145,8 @@ public abstract class BasePythonInterpreterTest { if (interpreter instanceof IPythonInterpreter) { interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); - assertTrue( - interpreterResultMessages.get(0).getData().contains("name 'unknown' is not defined")); + assertTrue(interpreterResultMessages.get(0).getData().contains( + "name 'unknown' is not defined")); } else if (interpreter instanceof PythonInterpreter) { assertTrue(result.message().get(0).getData().contains("name 'unknown' is not defined")); } @@ -163,13 +166,11 @@ public abstract class BasePythonInterpreterTest { // ZEPPELIN-1133 context = getInterpreterContext(); - result = - interpreter.interpret( - "from __future__ import print_function\n" - + "def greet(name):\n" - + " print('Hello', name)\n" - + "greet('Jack')", - context); + result = interpreter.interpret( + "from __future__ import print_function\n" + + "def greet(name):\n" + + " print('Hello', name)\n" + + "greet('Jack')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); @@ -194,8 +195,8 @@ public abstract class BasePythonInterpreterTest { assertEquals(0, interpreterResultMessages.size()); context = getInterpreterContext(); - result = - interpreter.interpret("# print('Hello')\n# print('How are u?')\n# time.sleep(1)", context); + result = interpreter.interpret( + "# print('Hello')\n# print('How are u?')\n# time.sleep(1)", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); @@ -258,7 +259,8 @@ public abstract class BasePythonInterpreterTest { // Password context = getInterpreterContext(); - result = interpreter.interpret("z.password(name='pwd_1')", context); + result = + interpreter.interpret("z.password(name='pwd_1')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertTrue(context.getGui().getForms().get("pwd_1") instanceof Password); @@ -267,11 +269,8 @@ public abstract class BasePythonInterpreterTest { // Select context = getInterpreterContext(); - result = - interpreter.interpret( - "z.select(name='select_1'," - + " options=[('value_1', 'name_1'), ('value_2', 'name_2')])", - context); + result = interpreter.interpret("z.select(name='select_1'," + + " options=[('value_1', 'name_1'), ('value_2', 'name_2')])", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals(1, context.getGui().getForms().size()); assertTrue(context.getGui().getForms().get("select_1") instanceof Select); @@ -283,11 +282,8 @@ public abstract class BasePythonInterpreterTest { // CheckBox context = getInterpreterContext(); - result = - interpreter.interpret( - "z.checkbox(name='checkbox_1'," - + "options=[('value_1', 'name_1'), ('value_2', 'name_2')])", - context); + result = interpreter.interpret("z.checkbox(name='checkbox_1'," + + "options=[('value_1', 'name_1'), ('value_2', 'name_2')])", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals(1, context.getGui().getForms().size()); assertTrue(context.getGui().getForms().get("checkbox_1") instanceof CheckBox); @@ -299,11 +295,8 @@ public abstract class BasePythonInterpreterTest { // Pandas DataFrame context = getInterpreterContext(); - result = - interpreter.interpret( - "import pandas as pd\n" - + "df = pd.DataFrame({'id':[1,2,3], 'name':['a','b','c']})\nz.show(df)", - context); + result = interpreter.interpret("import pandas as pd\n" + + "df = pd.DataFrame({'id':[1,2,3], 'name':['a','b','c']})\nz.show(df)", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); @@ -311,28 +304,21 @@ public abstract class BasePythonInterpreterTest { assertEquals("id\tname\n1\ta\n2\tb\n3\tc\n", interpreterResultMessages.get(0).getData()); context = getInterpreterContext(); - result = - interpreter.interpret( - "import pandas as pd\n" - + "df = pd.DataFrame({'id':[1,2,3,4], 'name':['a','b','c', 'd']})\nz.show(df)", - context); + result = interpreter.interpret("import pandas as pd\n" + + "df = pd.DataFrame({'id':[1,2,3,4], 'name':['a','b','c', 'd']})\nz.show(df)", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(2, interpreterResultMessages.size()); assertEquals(InterpreterResult.Type.TABLE, interpreterResultMessages.get(0).getType()); assertEquals("id\tname\n1\ta\n2\tb\n3\tc\n", interpreterResultMessages.get(0).getData()); assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(1).getType()); - assertEquals( - "Results are limited by 3.\n", + assertEquals("Results are limited by 3.\n", interpreterResultMessages.get(1).getData()); // z.show(matplotlib) context = getInterpreterContext(); - result = - interpreter.interpret( - "import matplotlib.pyplot as plt\n" - + "data=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)\nz.show(plt)", - context); + result = interpreter.interpret("import matplotlib.pyplot as plt\n" + + "data=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)\nz.show(plt)", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); assertEquals(1, interpreterResultMessages.size()); @@ -340,11 +326,8 @@ public abstract class BasePythonInterpreterTest { // clear output context = getInterpreterContext(); - result = - interpreter.interpret( - "import time\nprint(\"Hello\")\n" - + "time.sleep(0.5)\nz.getInterpreterContext().out().clear()\nprint(\"world\")\n", - context); + result = interpreter.interpret("import time\nprint(\"Hello\")\n" + + "time.sleep(0.5)\nz.getInterpreterContext().out().clear()\nprint(\"world\")\n", context); assertEquals("%text world\n", context.out.getCurrentOutput().toString()); } @@ -354,20 +337,15 @@ public abstract class BasePythonInterpreterTest { String restoreCode = "z = __zeppelin__\n"; String validCode = "z.input(\"test\")\n"; - assertEquals( - InterpreterResult.Code.SUCCESS, + assertEquals(InterpreterResult.Code.SUCCESS, interpreter.interpret(validCode, getInterpreterContext()).code()); - assertEquals( - InterpreterResult.Code.SUCCESS, + assertEquals(InterpreterResult.Code.SUCCESS, interpreter.interpret(redefinitionCode, getInterpreterContext()).code()); - assertEquals( - InterpreterResult.Code.ERROR, + assertEquals(InterpreterResult.Code.ERROR, interpreter.interpret(validCode, getInterpreterContext()).code()); - assertEquals( - InterpreterResult.Code.SUCCESS, + assertEquals(InterpreterResult.Code.SUCCESS, interpreter.interpret(restoreCode, getInterpreterContext()).code()); - assertEquals( - InterpreterResult.Code.SUCCESS, + assertEquals(InterpreterResult.Code.SUCCESS, interpreter.interpret(validCode, getInterpreterContext()).code()); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java index 5fba6ba..4b6bfdb 100644 --- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java @@ -17,13 +17,6 @@ package org.apache.zeppelin.python; -import static junit.framework.TestCase.assertTrue; -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -33,6 +26,15 @@ import org.apache.zeppelin.interpreter.InterpreterResultMessage; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.junit.Test; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertEquals; + + public class IPythonInterpreterTest extends BasePythonInterpreterTest { protected Properties initIntpProperties() { @@ -111,11 +113,8 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest { public void testIPythonPlotting() throws InterpreterException, InterruptedException, IOException { // matplotlib InterpreterContext context = getInterpreterContext(); - InterpreterResult result = - interpreter.interpret( - "%matplotlib inline\n" - + "import matplotlib.pyplot as plt\ndata=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)", - context); + InterpreterResult result = interpreter.interpret("%matplotlib inline\n" + + "import matplotlib.pyplot as plt\ndata=[1,1,2,3,4]\nplt.figure()\nplt.plot(data)", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); List interpreterResultMessages = @@ -145,13 +144,10 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest { // bokeh // bokeh initialization context = getInterpreterContext(); - result = - interpreter.interpret( - "from bokeh.io import output_notebook, show\n" - + "from bokeh.plotting import figure\n" - + "import bkzep\n" - + "output_notebook(notebook_type='zeppelin')", - context); + result = interpreter.interpret("from bokeh.io import output_notebook, show\n" + + "from bokeh.plotting import figure\n" + + "import bkzep\n" + + "output_notebook(notebook_type='zeppelin')", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); @@ -163,15 +159,12 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest { // bokeh plotting context = getInterpreterContext(); - result = - interpreter.interpret( - "from bokeh.plotting import figure, output_file, show\n" - + "x = [1, 2, 3, 4, 5]\n" - + "y = [6, 7, 2, 4, 5]\n" - + "p = figure(title=\"simple line example\", x_axis_label='x', y_axis_label='y')\n" - + "p.line(x, y, legend=\"Temp.\", line_width=2)\n" - + "show(p)", - context); + result = interpreter.interpret("from bokeh.plotting import figure, output_file, show\n" + + "x = [1, 2, 3, 4, 5]\n" + + "y = [6, 7, 2, 4, 5]\n" + + "p = figure(title=\"simple line example\", x_axis_label='x', y_axis_label='y')\n" + + "p.line(x, y, legend=\"Temp.\", line_width=2)\n" + + "show(p)", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); @@ -183,13 +176,10 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest { // ggplot context = getInterpreterContext(); - result = - interpreter.interpret( - "from ggplot import *\n" - + "ggplot(diamonds, aes(x='price', fill='cut')) +\\\n" - + " geom_density(alpha=0.25) +\\\n" - + " facet_wrap(\"clarity\")", - context); + result = interpreter.interpret("from ggplot import *\n" + + "ggplot(diamonds, aes(x='price', fill='cut')) +\\\n" + + " geom_density(alpha=0.25) +\\\n" + + " facet_wrap(\"clarity\")", context); Thread.sleep(100); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.toInterpreterResultMessage(); @@ -244,4 +234,5 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest { result = interpreter.interpret("print('1'*3000)", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); } + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java index 1a8ab03..a2cdac8 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonCondaInterpreterTest.java @@ -17,22 +17,7 @@ package org.apache.zeppelin.python; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; -import java.io.IOException; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.regex.Matcher; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; @@ -41,6 +26,23 @@ import org.apache.zeppelin.interpreter.InterpreterResult; import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Matcher; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + public class PythonCondaInterpreterTest { private PythonCondaInterpreter conda; private PythonInterpreter python; @@ -103,14 +105,14 @@ public class PythonCondaInterpreterTest { } @Test - public void testParseCondaCommonStdout() throws IOException, InterruptedException { + public void testParseCondaCommonStdout() + throws IOException, InterruptedException { - StringBuilder sb = - new StringBuilder() - .append("# comment1\n") - .append("# comment2\n") - .append("env1 /location1\n") - .append("env2 /location2\n"); + StringBuilder sb = new StringBuilder() + .append("# comment1\n") + .append("# comment2\n") + .append("env1 /location1\n") + .append("env2 /location2\n"); Map locationPerEnv = PythonCondaInterpreter.parseCondaCommonStdout(sb.toString()); @@ -121,15 +123,20 @@ public class PythonCondaInterpreterTest { @Test public void testGetRestArgsFromMatcher() { - Matcher m = PythonCondaInterpreter.PATTERN_COMMAND_ENV.matcher("env remove --name test --yes"); + Matcher m = + PythonCondaInterpreter.PATTERN_COMMAND_ENV.matcher("env remove --name test --yes"); m.matches(); List restArgs = PythonCondaInterpreter.getRestArgsFromMatcher(m); - List expected = Arrays.asList(new String[] {"remove", "--name", "test", "--yes"}); + List expected = Arrays.asList(new String[]{"remove", "--name", "test", "--yes"}); assertEquals(expected, restArgs); } private InterpreterContext getInterpreterContext() { - return InterpreterContext.builder().setInterpreterOut(new InterpreterOutput(null)).build(); + return InterpreterContext.builder() + .setInterpreterOut(new InterpreterOutput(null)) + .build(); } + + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java index 39041f2..04bb414 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonDockerInterpreterTest.java @@ -16,17 +16,6 @@ */ package org.apache.zeppelin.python; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import java.io.File; -import java.util.Arrays; -import java.util.Properties; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; @@ -35,6 +24,18 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; +import java.io.File; +import java.util.Arrays; +import java.util.Properties; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + public class PythonDockerInterpreterTest { private PythonDockerInterpreter docker; private PythonInterpreter python; @@ -76,6 +77,8 @@ public class PythonDockerInterpreterTest { } private InterpreterContext getInterpreterContext() { - return InterpreterContext.builder().setInterpreterOut(new InterpreterOutput(null)).build(); + return InterpreterContext.builder() + .setInterpreterOut(new InterpreterOutput(null)) + .build(); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java index 1fb2418..8326612 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterMatplotlibTest.java @@ -17,14 +17,6 @@ package org.apache.zeppelin.python; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.Properties; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; @@ -38,6 +30,15 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; + public class PythonInterpreterMatplotlibTest implements InterpreterOutputListener { private InterpreterGroup intpGroup; private PythonInterpreter python; @@ -63,11 +64,10 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene out = new InterpreterOutput(this); - context = - InterpreterContext.builder() - .setInterpreterOut(out) - .setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null)) - .build(); + context = InterpreterContext.builder() + .setInterpreterOut(out) + .setAngularObjectRegistry(new AngularObjectRegistry(intpGroup.getId(), null)) + .build(); InterpreterContext.set(context); python.open(); @@ -98,16 +98,12 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene ret = python.interpret("plt.plot([1, 2, 3])", context); ret = python.interpret("plt.show()", context); - assertEquals( - new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code()); - assertEquals( - new String(out.getOutputAt(0).toByteArray()), - InterpreterResult.Type.TEXT, - out.getOutputAt(0).getType()); - assertEquals( - new String(out.getOutputAt(1).toByteArray()), - InterpreterResult.Type.HTML, - out.getOutputAt(1).getType()); + assertEquals(new String(out.getOutputAt(0).toByteArray()), + InterpreterResult.Code.SUCCESS, ret.code()); + assertEquals(new String(out.getOutputAt(0).toByteArray()), + InterpreterResult.Type.TEXT, out.getOutputAt(0).getType()); + assertEquals(new String(out.getOutputAt(1).toByteArray()), + InterpreterResult.Type.HTML, out.getOutputAt(1).getType()); assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("data:image/png;base64")); assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("

")); } @@ -129,8 +125,8 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene // type to HTML. ret = python.interpret("plt.show()", context); - assertEquals( - new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code()); + assertEquals(new String(out.getOutputAt(0).toByteArray()), + InterpreterResult.Code.SUCCESS, ret.code()); assertEquals(0, ret.message().size()); // Now test that new plot is drawn. It should be identical to the @@ -177,12 +173,19 @@ public class PythonInterpreterMatplotlibTest implements InterpreterOutputListene assertNotSame(msg1, msg2); } + @Override - public void onUpdateAll(InterpreterOutput out) {} + public void onUpdateAll(InterpreterOutput out) { + + } @Override - public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {} + public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) { + + } @Override - public void onUpdate(int index, InterpreterResultMessageOutput out) {} + public void onUpdate(int index, InterpreterResultMessageOutput out) { + + } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java index 9d4c445..8f6cab2 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterPandasSqlTest.java @@ -17,13 +17,6 @@ package org.apache.zeppelin.python; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Properties; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; @@ -36,17 +29,25 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.util.Arrays; +import java.util.Properties; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + /** * In order for this test to work, test env must have installed: - * *
    - * - - *
  1. Python - - *
  2. NumPy - - *
  3. Pandas - - *
  4. PandaSql - *
      - *

      To run manually on such environment, use: + * -

    1. Python
      2. + * -
    2. NumPy
      3. + * -
    3. Pandas
      4. + * -
    4. PandaSql
      5. + *
        + *

        + * To run manually on such environment, use: + * * mvn -Dpython.test.exclude='' test -pl python -am * */ @@ -69,7 +70,9 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener intpGroup = new InterpreterGroup(); out = new InterpreterOutput(this); - context = InterpreterContext.builder().setInterpreterOut(out).build(); + context = InterpreterContext.builder() + .setInterpreterOut(out) + .build(); InterpreterContext.set(context); python = new PythonInterpreter(p); @@ -81,6 +84,7 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener intpGroup.put("note", Arrays.asList(python, sql)); + // to make sure python is running. InterpreterResult ret = python.interpret("print(\"python initialized\")\n", context); assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code()); @@ -114,40 +118,36 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener public void sqlOverTestDataPrintsTable() throws IOException, InterpreterException { InterpreterResult ret; // given - // String expectedTable = "name\tage\n\nmoon\t33\n\npark\t34"; + //String expectedTable = "name\tage\n\nmoon\t33\n\npark\t34"; ret = python.interpret("import pandas as pd", context); ret = python.interpret("import numpy as np", context); // DataFrame df2 \w test data - ret = - python.interpret( - "df2 = pd.DataFrame({ 'age' : np.array([33, 51, 51, 34]), " - + "'name' : pd.Categorical(['moon','jobs','gates','park'])})", - context); + ret = python.interpret("df2 = pd.DataFrame({ 'age' : np.array([33, 51, 51, 34]), " + + "'name' : pd.Categorical(['moon','jobs','gates','park'])})", context); assertEquals(ret.message().toString(), InterpreterResult.Code.SUCCESS, ret.code()); - // when + //when ret = sql.interpret("select name, age from df2 where age < 40", context); - // then - assertEquals( - new String(out.getOutputAt(1).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code()); - assertEquals( - new String(out.getOutputAt(1).toByteArray()), Type.TABLE, out.getOutputAt(1).getType()); + //then + assertEquals(new String(out.getOutputAt(1).toByteArray()), + InterpreterResult.Code.SUCCESS, ret.code()); + assertEquals(new String(out.getOutputAt(1).toByteArray()), Type.TABLE, + out.getOutputAt(1).getType()); assertTrue(new String(out.getOutputAt(1).toByteArray()).indexOf("moon\t33") > 0); assertTrue(new String(out.getOutputAt(1).toByteArray()).indexOf("park\t34") > 0); - assertEquals( - InterpreterResult.Code.SUCCESS, - sql.interpret("select case when name==\"aa\" then name else name end from df2", context) - .code()); + assertEquals(InterpreterResult.Code.SUCCESS, + sql.interpret("select case when name==\"aa\" then name else name end from df2", + context).code()); } @Test public void badSqlSyntaxFails() throws IOException, InterpreterException { - // when + //when InterpreterResult ret = sql.interpret("select wrong syntax", context); - // then + //then assertNotNull("Interpreter returned 'null'", ret); assertEquals(ret.toString(), InterpreterResult.Code.ERROR, ret.code()); } @@ -168,21 +168,27 @@ public class PythonInterpreterPandasSqlTest implements InterpreterOutputListener ret = python.interpret("z.show(df1, show_index=True)", context); // then - assertEquals( - new String(out.getOutputAt(0).toByteArray()), InterpreterResult.Code.SUCCESS, ret.code()); - assertEquals( - new String(out.getOutputAt(1).toByteArray()), Type.TABLE, out.getOutputAt(1).getType()); + assertEquals(new String(out.getOutputAt(0).toByteArray()), + InterpreterResult.Code.SUCCESS, ret.code()); + assertEquals(new String(out.getOutputAt(1).toByteArray()), + Type.TABLE, out.getOutputAt(1).getType()); assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("index_name")); assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("nan")); assertTrue(new String(out.getOutputAt(1).toByteArray()).contains("6.7")); } @Override - public void onUpdateAll(InterpreterOutput out) {} + public void onUpdateAll(InterpreterOutput out) { + + } @Override - public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) {} + public void onAppend(int index, InterpreterResultMessageOutput out, byte[] line) { + + } @Override - public void onUpdate(int index, InterpreterResultMessageOutput out) {} + public void onUpdate(int index, InterpreterResultMessageOutput out) { + + } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java ---------------------------------------------------------------------- diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java index 10c6a8e..8748c00 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java @@ -17,16 +17,6 @@ package org.apache.zeppelin.python; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.LinkedList; -import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterException; @@ -35,6 +25,18 @@ import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.junit.Test; +import java.io.IOException; +import java.util.LinkedList; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + + public class PythonInterpreterTest extends BasePythonInterpreterTest { @Override @@ -65,7 +67,7 @@ public class PythonInterpreterTest extends BasePythonInterpreterTest { public void testCodeCompletion() throws InterpreterException, IOException, InterruptedException { super.testCodeCompletion(); - // TODO(zjffdu) PythonInterpreter doesn't support this kind of code completion for now. + //TODO(zjffdu) PythonInterpreter doesn't support this kind of code completion for now. // completion // InterpreterContext context = getInterpreterContext(); // List completions = interpreter.completion("ab", 2, context); @@ -93,8 +95,7 @@ public class PythonInterpreterTest extends BasePythonInterpreterTest { @Test public void testCancelIntp() throws InterruptedException, InterpreterException { - assertEquals( - InterpreterResult.Code.SUCCESS, + assertEquals(InterpreterResult.Code.SUCCESS, interpreter.interpret("a = 1\n", getInterpreterContext()).code()); Thread t = new Thread(new infinityPythonJob()); t.start(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0d746fa2/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java ---------------------------------------------------------------------- diff --git a/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java b/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java index cdeeacb..17da1c9 100644 --- a/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java +++ b/sap/src/main/java/org/apache/zeppelin/sap/UniverseInterpreter.java @@ -17,10 +17,6 @@ package org.apache.zeppelin.sap; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import org.apache.commons.lang.StringUtils; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; @@ -31,7 +27,15 @@ import org.apache.zeppelin.sap.universe.*; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; -/** SAP Universe interpreter for Zeppelin. */ + +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * SAP Universe interpreter for Zeppelin. + */ public class UniverseInterpreter extends Interpreter { public UniverseInterpreter(Properties properties) { @@ -47,8 +51,8 @@ public class UniverseInterpreter extends Interpreter { private static final char NEWLINE = '\n'; private static final char TAB = '\t'; private static final String TABLE_MAGIC_TAG = "%table "; - private static final String EMPTY_DATA_MESSAGE = - "%html\n" + "

        No Data Available

        "; + private static final String EMPTY_DATA_MESSAGE = "%html\n" + + "

        No Data Available

        "; private static final String CONCURRENT_EXECUTION_KEY = "universe.concurrent.use"; private static final String CONCURRENT_EXECUTION_COUNT = "universe.concurrent.maxConnection"; @@ -59,10 +63,10 @@ public class UniverseInterpreter extends Interpreter { String password = getProperty("universe.password"); String apiUrl = getProperty("universe.api.url"); String authType = getProperty("universe.authType"); - final int queryTimeout = - Integer.parseInt( - StringUtils.defaultIfEmpty(getProperty("universe.queryTimeout"), "7200000")); - this.client = new UniverseClient(user, password, apiUrl, authType, queryTimeout); + final int queryTimeout = Integer.parseInt( + StringUtils.defaultIfEmpty(getProperty("universe.queryTimeout"), "7200000")); + this.client = + new UniverseClient(user, password, apiUrl, authType, queryTimeout); this.universeUtil = new UniverseUtil(); } @@ -78,10 +82,8 @@ public class UniverseInterpreter extends Interpreter { @Override public InterpreterResult interpret(String originalSt, InterpreterContext context) throws InterpreterException { - final String st = - Boolean.parseBoolean(getProperty("universe.interpolation", "false")) - ? interpolate(originalSt, context.getResourcePool()) - : originalSt; + final String st = Boolean.parseBoolean(getProperty("universe.interpolation", "false")) ? + interpolate(originalSt, context.getResourcePool()) : originalSt; try { InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); String paragraphId = context.getParagraphId(); @@ -123,7 +125,7 @@ public class UniverseInterpreter extends Interpreter { try { client.closeSession(context.getParagraphId()); } catch (Exception e) { - logger.error("Error close SAP session", e); + logger.error("Error close SAP session", e ); } } } @@ -133,7 +135,7 @@ public class UniverseInterpreter extends Interpreter { try { client.closeSession(context.getParagraphId()); } catch (Exception e) { - logger.error("Error close SAP session", e); + logger.error("Error close SAP session", e ); } } @@ -148,15 +150,16 @@ public class UniverseInterpreter extends Interpreter { } @Override - public List completion( - String buf, int cursor, InterpreterContext interpreterContext) throws InterpreterException { + public List completion(String buf, int cursor, + InterpreterContext interpreterContext) + throws InterpreterException { List candidates = new ArrayList<>(); try { universeCompleter = createOrUpdateUniverseCompleter(interpreterContext, buf, cursor); universeCompleter.complete(buf, cursor, candidates); } catch (UniverseException e) { - logger.error("Error update completer", e); + logger.error("Error update completer", e ); } return candidates; @@ -165,9 +168,9 @@ public class UniverseInterpreter extends Interpreter { @Override public Scheduler getScheduler() { String schedulerName = UniverseInterpreter.class.getName() + this.hashCode(); - return isConcurrentExecution() - ? SchedulerFactory.singleton() - .createOrGetParallelScheduler(schedulerName, getMaxConcurrentConnection()) + return isConcurrentExecution() ? + SchedulerFactory.singleton().createOrGetParallelScheduler(schedulerName, + getMaxConcurrentConnection()) : SchedulerFactory.singleton().createOrGetFIFOScheduler(schedulerName); } @@ -209,8 +212,8 @@ public class UniverseInterpreter extends Interpreter { return str.replace(TAB, WHITESPACE).replace(NEWLINE, WHITESPACE); } - private UniverseCompleter createOrUpdateUniverseCompleter( - InterpreterContext interpreterContext, final String buf, final int cursor) + private UniverseCompleter createOrUpdateUniverseCompleter(InterpreterContext interpreterContext, + final String buf, final int cursor) throws UniverseException { final UniverseCompleter completer; if (universeCompleter == null) { @@ -221,13 +224,12 @@ public class UniverseInterpreter extends Interpreter { try { final String token = client.getToken(interpreterContext.getParagraphId()); ExecutorService executorService = Executors.newFixedThreadPool(1); - executorService.execute( - new Runnable() { - @Override - public void run() { - completer.createOrUpdate(client, token, buf, cursor); - } - }); + executorService.execute(new Runnable() { + @Override + public void run() { + completer.createOrUpdate(client, token, buf, cursor); + } + }); executorService.shutdown(); @@ -238,7 +240,7 @@ public class UniverseInterpreter extends Interpreter { try { client.closeSession(interpreterContext.getParagraphId()); } catch (Exception e) { - logger.error("Error close SAP session", e); + logger.error("Error close SAP session", e ); } } return completer;