flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ches...@apache.org
Subject flink git commit: [FLINK-7283][python] fix PythonPlanBinderTest issues with python paths
Date Mon, 31 Jul 2017 14:14:04 GMT
Repository: flink
Updated Branches:
  refs/heads/release-1.3 9495138de -> 2455df962


[FLINK-7283][python] fix PythonPlanBinderTest issues with python paths

* the path is not set correctly (only inside config, not the configuration that
  is passed on to the PythonPlanBinder
* linux distributions have become quite inventive regarding python binary names:
  some offer python as Python 2, some as Python 3. Similarly, python3 and/or
  python2 may not be available. If we really want to test both, we need to take
  this into account.

This closes #4409.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/2455df96
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/2455df96
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/2455df96

Branch: refs/heads/release-1.3
Commit: 2455df962c76d20f1c07a57d6ed0118d1d1a067c
Parents: 9495138
Author: Nico Kruber <nico@data-artisans.com>
Authored: Thu Jul 27 16:35:37 2017 +0200
Committer: zentol <chesnay@apache.org>
Committed: Mon Jul 31 14:03:04 2017 +0200

----------------------------------------------------------------------
 .../flink/python/api/PythonPlanBinderTest.java  | 85 ++++++++++++++------
 1 file changed, 59 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/2455df96/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java
----------------------------------------------------------------------
diff --git a/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java
b/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java
index 20f3503..9287856 100644
--- a/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java
+++ b/flink-libraries/flink-python/src/test/java/org/apache/flink/python/api/PythonPlanBinderTest.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
agreements. See the NOTICE
  * file distributed with this work for additional information regarding copyright ownership.
The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the "License"); you may not use this file
except in compliance with the
@@ -19,7 +19,9 @@ import org.apache.flink.core.fs.Path;
 import org.apache.flink.python.api.streaming.data.PythonStreamer;
 import org.apache.flink.test.util.JavaProgramTestBase;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -51,50 +53,81 @@ public class PythonPlanBinderTest extends JavaProgramTestBase {
 		return files;
 	}
 
-	private static boolean isPython2Supported() throws IOException {
+	/**
+	 * Finds the python binary for the given version.
+	 *
+	 * @param possibleBinaries
+	 * 		binaries to test for
+	 * @param expectedVersionPrefix
+	 * 		expected output prefix for <tt>&lt;binary&gt; -V</tt>, e.g. <tt>"Python
2."</tt>
+	 *
+	 * @return python binary or <tt>null</tt> if not supported
+	 *
+	 * @throws IOException
+	 * 		if the process to test for the binaries failed to exit properly
+	 */
+	private static String getPythonPath(String[] possibleBinaries, String expectedVersionPrefix)
+		throws IOException {
 		Process process = null;
+		for (String python : possibleBinaries) {
+			try {
+				process = new ProcessBuilder(python, "-V").redirectErrorStream(true).start();
+				BufferedReader stdInput = new BufferedReader(new
+					InputStreamReader(process.getInputStream()));
 
-		try {
-			process = Runtime.getRuntime().exec("python");
-			return true;
-		} catch (IOException ex) {
-			return false;
-		} finally {
-			if (process != null) {
-				PythonStreamer.destroyProcess(process);
+				if (stdInput.readLine().startsWith(expectedVersionPrefix)) {
+					return python;
+				}
+			} catch (IOException ignored) {
+			} finally {
+				if (process != null) {
+					PythonStreamer.destroyProcess(process);
+				}
 			}
 		}
+		return null;
 	}
 
-	private static boolean isPython3Supported() throws IOException {
-		Process process = null;
+	/**
+	 * Finds the binary that executes python2 programs.
+	 *
+	 * @return python2 binary or <tt>null</tt> if not supported
+	 *
+	 * @throws IOException
+	 * 		if the process to test for the binaries failed to exit properly
+	 */
+	private static String getPython2Path() throws IOException {
+		return getPythonPath(new String[] {"python2", "python"}, "Python 2.");
+	}
 
-		try {
-			process = Runtime.getRuntime().exec("python3");
-			return true;
-		} catch (IOException ex) {
-			return false;
-		} finally {
-			if (process != null) {
-				PythonStreamer.destroyProcess(process);
-			}
-		}
+	/**
+	 * Finds the binary that executes python3 programs.
+	 *
+	 * @return python3 binary or <tt>null</tt> if not supported
+	 *
+	 * @throws IOException
+	 * 		if the process to test for the binaries failed to exit properly
+	 */
+	private static String getPython3Path() throws IOException {
+		return getPythonPath(new String[] {"python3", "python"}, "Python 3.");
 	}
 
 	@Override
 	protected void testProgram() throws Exception {
 		String utils = findUtilsFile();
-		if (isPython2Supported()) {
+		String python2 = getPython2Path();
+		if (python2 != null) {
 			for (String file : findTestFiles()) {
 				Configuration configuration = new Configuration();
-				config.setString(PythonOptions.PYTHON_BINARY_PATH, "python");
+				configuration.setString(PythonOptions.PYTHON_BINARY_PATH, python2);
 				new PythonPlanBinder(configuration).runPlan(new String[]{file, utils});
 			}
 		}
-		if (isPython3Supported()) {
+		String python3 = getPython3Path();
+		if (python3 != null) {
 			for (String file : findTestFiles()) {
 				Configuration configuration = new Configuration();
-				config.setString(PythonOptions.PYTHON_BINARY_PATH, "python3");
+				configuration.setString(PythonOptions.PYTHON_BINARY_PATH, python3);
 				new PythonPlanBinder(configuration).runPlan(new String[]{file, utils});
 			}
 		}


Mime
View raw message