ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sboi...@apache.org
Subject [05/16] ignite git commit: ignite-7135 IgniteCluster.startNodes() returns successful ClusterStartNodeResult even though the remote process fails.
Date Fri, 12 Jan 2018 07:54:13 GMT
ignite-7135 IgniteCluster.startNodes() returns successful ClusterStartNodeResult even though
the remote process fails.

Fix windows startup scripts.
Add startNodes() Windows support through task scheduling.
Check successful node startup by reading the output log.

Signed-off-by: Andrey Gura <agura@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a7f0422d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a7f0422d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a7f0422d

Branch: refs/heads/ignite-zk-join
Commit: a7f0422d77c9a713ab3912cc137b045f2ec732e6
Parents: a4b2446
Author: Alexandr Kuramshin <ein.nsk.ru@gmail.com>
Authored: Thu Jan 11 17:05:17 2018 +0300
Committer: Andrey Gura <agura@apache.org>
Committed: Thu Jan 11 17:05:17 2018 +0300

----------------------------------------------------------------------
 bin/ignite.bat                                  |   2 +-
 bin/include/parseargs.bat                       |   2 +-
 .../startup/cmdline/CommandLineTransformer.java |   6 +-
 .../util/nodestart/StartNodeCallableImpl.java   | 260 ++++++++++++++++---
 4 files changed, 237 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/a7f0422d/bin/ignite.bat
----------------------------------------------------------------------
diff --git a/bin/ignite.bat b/bin/ignite.bat
index db686cc..f697dc0 100644
--- a/bin/ignite.bat
+++ b/bin/ignite.bat
@@ -133,7 +133,7 @@ set RESTART_SUCCESS_OPT=-DIGNITE_SUCCESS_FILE=%RESTART_SUCCESS_FILE%
 :: This is executed if -nojmx is not specified
 ::
 if not "%NO_JMX%" == "1" (
-    for /F "tokens=*" %%A in ('""!JAVA_HOME!\bin\java" -cp %CP% org.apache.ignite.internal.util.portscanner.GridJmxPortFinder"')
do (
+    for /F "usebackq tokens=*" %%A in (`"!JAVA_HOME!\bin\java.exe -cp %CP% org.apache.ignite.internal.util.portscanner.GridJmxPortFinder"`)
do (
         set JMX_PORT=%%A
     )
 )

http://git-wip-us.apache.org/repos/asf/ignite/blob/a7f0422d/bin/include/parseargs.bat
----------------------------------------------------------------------
diff --git a/bin/include/parseargs.bat b/bin/include/parseargs.bat
index f2151a7..0a584d1 100644
--- a/bin/include/parseargs.bat
+++ b/bin/include/parseargs.bat
@@ -39,7 +39,7 @@
 :: )
 :: in other scripts to parse common command lines parameters.
 
-set convertArgsCmd="!JAVA_HOME!\bin\java.exe" -cp %CP% org.apache.ignite.startup.cmdline.CommandLineTransformer
%*
+set convertArgsCmd=!JAVA_HOME!\bin\java.exe -Dfile.encoding=IBM866 -cp %CP% org.apache.ignite.startup.cmdline.CommandLineTransformer
%*
 for /f "usebackq tokens=*" %%i in (`!convertArgsCmd!`) do set reformattedArgs=%%i
 
 for %%i in (%reformattedArgs%) do (

http://git-wip-us.apache.org/repos/asf/ignite/blob/a7f0422d/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineTransformer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineTransformer.java
b/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineTransformer.java
index be758fa..b98e8b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineTransformer.java
+++ b/modules/core/src/main/java/org/apache/ignite/startup/cmdline/CommandLineTransformer.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.startup.cmdline;
 
 import java.io.PrintStream;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -84,8 +85,9 @@ public class CommandLineTransformer {
         PrintStream ps = null;
 
         try {
-            // Intentionality configure output stream with UTF-8 encoding to support  non-ASCII
named parameter values.
-            ps = new PrintStream(System.out, true, "UTF-8");
+            String encoding = System.getProperty("file.encoding", Charset.defaultCharset().name());
+
+            ps = new PrintStream(System.out, true, encoding);
 
             ps.println(transform(args));
         }

http://git-wip-us.apache.org/repos/asf/ignite/blob/a7f0422d/modules/ssh/src/main/java/org/apache/ignite/internal/util/nodestart/StartNodeCallableImpl.java
----------------------------------------------------------------------
diff --git a/modules/ssh/src/main/java/org/apache/ignite/internal/util/nodestart/StartNodeCallableImpl.java
b/modules/ssh/src/main/java/org/apache/ignite/internal/util/nodestart/StartNodeCallableImpl.java
index f192d59..c9bc7ee 100644
--- a/modules/ssh/src/main/java/org/apache/ignite/internal/util/nodestart/StartNodeCallableImpl.java
+++ b/modules/ssh/src/main/java/org/apache/ignite/internal/util/nodestart/StartNodeCallableImpl.java
@@ -25,17 +25,26 @@ import com.jcraft.jsch.Session;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.InterruptedIOException;
 import java.io.PrintStream;
+import java.nio.charset.Charset;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.UUID;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterStartNodeResult;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterStartNodeResultImpl;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.LoggerResource;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SSH_HOST;
@@ -51,12 +60,30 @@ public class StartNodeCallableImpl implements StartNodeCallable {
     /** Default Ignite home path for Linux (taken from environment variable). */
     private static final String DFLT_IGNITE_HOME_LINUX = "$IGNITE_HOME";
 
+    /** Windows console encoding */
+    private static final String WINDOWS_ENCODING = "IBM866";
+
+    /** Default start script path for Windows. */
+    private static final String DFLT_SCRIPT_WIN = "bin\\ignite.bat -v -np";
+
     /** Default start script path for Linux. */
     private static final String DFLT_SCRIPT_LINUX = "bin/ignite.sh -v";
 
     /** Date format for log file name. */
     private static final SimpleDateFormat FILE_NAME_DATE_FORMAT = new SimpleDateFormat("MM-dd-yyyy--HH-mm-ss");
 
+    /** Used to register successful node start in log */
+    private static final String SUCCESSFUL_START_MSG = "Successfully bound to TCP port";
+
+    /**  */
+    private static final long EXECUTE_WAIT_TIME = 1000;
+
+    /**  */
+    private static final long NODE_START_CHECK_PERIOD = 2000;
+
+    /**  */
+    private static final long NODE_START_CHECK_LIMIT = 5;
+
     /** Specification. */
     private final IgniteRemoteStartSpecification spec;
 
@@ -65,13 +92,18 @@ public class StartNodeCallableImpl implements StartNodeCallable {
 
     /** Logger. */
     @LoggerResource
-    private IgniteLogger log;
+    private transient IgniteLogger log;
+
+    /** Ignite. */
+    @IgniteInstanceResource
+    private transient Ignite ignite;
 
     /**
      * Required by Externalizable.
      */
     public StartNodeCallableImpl() {
         spec = null;
+
         timeout = 0;
 
         assert false;
@@ -87,6 +119,7 @@ public class StartNodeCallableImpl implements StartNodeCallable {
         assert spec != null;
 
         this.spec = spec;
+
         this.timeout = timeout;
     }
 
@@ -111,6 +144,8 @@ public class StartNodeCallableImpl implements StartNodeCallable {
 
             boolean win = isWindows(ses);
 
+            info("Windows mode: " + win, spec.logger(), log);
+
             char separator = win ? '\\' : '/';
 
             spec.fixPaths(separator);
@@ -123,30 +158,95 @@ public class StartNodeCallableImpl implements StartNodeCallable {
             String script = spec.script();
 
             if (script == null)
-                script = DFLT_SCRIPT_LINUX;
+                script = win ? DFLT_SCRIPT_WIN : DFLT_SCRIPT_LINUX;
 
             String cfg = spec.configuration();
 
             if (cfg == null)
                 cfg = "";
 
-            String startNodeCmd;
-            String scriptOutputFileName = FILE_NAME_DATE_FORMAT.format(new Date()) + '-'
-                + UUID.randomUUID().toString().substring(0, 8) + ".log";
+            String id = FILE_NAME_DATE_FORMAT.format(new Date()) + '-' + UUID.randomUUID().toString().substring(0,
8);
+
+            String scriptOutputFileName = id + ".log";
+
+            int spaceIdx = script.indexOf(' ');
 
-            if (win)
-                throw new UnsupportedOperationException("Apache Ignite cannot be auto-started
on Windows from IgniteCluster.startNodes(…) API.");
+            String scriptPath = spaceIdx > -1 ? script.substring(0, spaceIdx) : script;
+
+            String scriptArgs = spaceIdx > -1 ? script.substring(spaceIdx + 1) : "";
+
+            String rmtLogArgs = buildRemoteLogArguments(spec.username(), spec.host());
+
+            String scriptOutputDir;
+
+            String dfltTmpDir = igniteHome + separator + "work" + separator + "log";
+
+            if (win) {
+                String tmpDir = env(ses, "%TMPDIR%", dfltTmpDir, WINDOWS_ENCODING);
+
+                if ("%TMPDIR%".equals(tmpDir))
+                    tmpDir = dfltTmpDir;
+
+                scriptOutputDir = tmpDir + "\\ignite-startNodes";
+            }
             else { // Assume Unix.
-                int spaceIdx = script.indexOf(' ');
+                String logDir = env(ses, "$TMPDIR", dfltTmpDir);
+
+                scriptOutputDir = logDir + "/ignite-startNodes";
+            }
+
+            shell(ses, "mkdir " + scriptOutputDir);
 
-                String scriptPath = spaceIdx > -1 ? script.substring(0, spaceIdx) : script;
-                String scriptArgs = spaceIdx > -1 ? script.substring(spaceIdx + 1) : "";
-                String rmtLogArgs = buildRemoteLogArguments(spec.username(), spec.host());
-                String tmpDir = env(ses, "$TMPDIR", "/tmp/");
-                String scriptOutputDir = tmpDir + "ignite-startNodes";
+            String scriptOutputPath = scriptOutputDir + separator + scriptOutputFileName;
 
-                shell(ses, "mkdir " + scriptOutputDir);
+            String findSuccess;
 
+            if (win) {
+                String scriptFileName = scriptOutputDir + '\\' + id + ".bat";
+
+                String createScript = new SB()
+                    .a("echo \"").a(igniteHome).a('\\').a(scriptPath).a("\" ")
+                    .a(scriptArgs)
+                    .a(!cfg.isEmpty() ? " \"" : "").a(cfg).a(!cfg.isEmpty() ? "\"" : "")
+                    .a(rmtLogArgs)
+                    .a(" ^> ").a(scriptOutputPath).a(" ^2^>^&^1")
+                    .a(" > ").a(scriptFileName)
+                    .toString();
+
+                info("Create script with command: " + createScript, spec.logger(), log);
+
+                shell(ses, createScript);
+
+                try {
+                    String createTask = new SB()
+                        .a("schtasks /create /f /sc onstart")
+                        .a(" /ru ").a(spec.username())
+                        .a(" /rp ").a(spec.password())
+                        .a(" /tn ").a(id)
+                        .a(" /np /tr \"").a(scriptFileName).a('\"')
+                        .toString();
+
+                    info("Create task with command: " + createTask, spec.logger(), log);
+
+                    shell(ses, createTask);
+
+                    String runTask = "schtasks /run /i /tn " + id;
+
+                    info("Run task with command: " + runTask, spec.logger(), log);
+
+                    shell(ses, runTask);
+                }
+                finally {
+                    String deleteTask = "schtasks /delete /f /tn " + id;
+
+                    info("Delete task with command: " + deleteTask, spec.logger(), log);
+
+                    shell(ses, deleteTask);
+                }
+
+                findSuccess = "find \"" + SUCCESSFUL_START_MSG + "\" " + scriptOutputPath;
+            }
+            else { // Assume Unix.
                 // Mac os don't support ~ in double quotes. Trying get home path from remote
system.
                 if (igniteHome.startsWith("~")) {
                     String homeDir = env(ses, "$HOME", "~");
@@ -154,22 +254,36 @@ public class StartNodeCallableImpl implements StartNodeCallable {
                     igniteHome = igniteHome.replaceFirst("~", homeDir);
                 }
 
-                startNodeCmd = new SB().
+                String startNodeCmd = new SB()
                     // Console output is consumed, started nodes must use Ignite file appenders
for log.
-                        a("nohup ").
-                    a("\"").a(igniteHome).a('/').a(scriptPath).a("\"").
-                    a(" ").a(scriptArgs).
-                    a(!cfg.isEmpty() ? " \"" : "").a(cfg).a(!cfg.isEmpty() ? "\"" : "").
-                    a(rmtLogArgs).
-                    a(" > ").a(scriptOutputDir).a("/").a(scriptOutputFileName).a(" 2>&
1 &").
-                    toString();
+                    .a("nohup ")
+                    .a("\"").a(igniteHome).a('/').a(scriptPath).a("\"")
+                    .a(" ").a(scriptArgs)
+                    .a(!cfg.isEmpty() ? " \"" : "").a(cfg).a(!cfg.isEmpty() ? "\"" : "")
+                    .a(rmtLogArgs)
+                    .a(" > ").a(scriptOutputDir).a('/').a(scriptOutputFileName).a(" 2>&
1 &")
+                    .toString();
+
+                info("Starting remote node with SSH command: " + startNodeCmd, spec.logger(),
log);
+
+                shell(ses, startNodeCmd);
+
+                findSuccess = "grep \"" + SUCCESSFUL_START_MSG + "\" " + scriptOutputPath;
             }
 
-            info("Starting remote node with SSH command: " + startNodeCmd, spec.logger(),
log);
+            for (int i = 0; i < NODE_START_CHECK_LIMIT; ++i) {
+                Thread.sleep(NODE_START_CHECK_PERIOD);
+
+                String res = exec(ses, findSuccess, win ? WINDOWS_ENCODING : null);
 
-            shell(ses, startNodeCmd);
+                info("Find result: " + res, spec.logger(), log);
+
+                if (res != null && res.contains(SUCCESSFUL_START_MSG))
+                    return new ClusterStartNodeResultImpl(spec.host(), true, null);
+            }
 
-            return new ClusterStartNodeResultImpl(spec.host(), true, null);
+            return new ClusterStartNodeResultImpl(spec.host(), false, "Remote node could
not start. " +
+                "See log for details: " + scriptOutputPath);
         }
         catch (IgniteInterruptedCheckedException e) {
             return new ClusterStartNodeResultImpl(spec.host(), false, e.getMessage());
@@ -203,7 +317,7 @@ public class StartNodeCallableImpl implements StartNodeCallable {
             try (PrintStream out = new PrintStream(ch.getOutputStream(), true)) {
                 out.println(cmd);
 
-                U.sleep(1000);
+                U.sleep(EXECUTE_WAIT_TIME);
             }
         }
         finally {
@@ -238,8 +352,29 @@ public class StartNodeCallableImpl implements StartNodeCallable {
      * @throws JSchException In case of SSH error.
      */
     private String env(Session ses, String name, String dflt) throws JSchException {
+        return env(ses, name, dflt, null);
+    }
+
+    /**
+     * Gets the value of the specified environment variable.
+     *
+     * @param ses SSH session.
+     * @param name environment variable name.
+     * @param dflt default value.
+     * @param encoding Process output encoding, {@code null} for default charset encoding.
+     * @return environment variable value.
+     * @throws JSchException In case of SSH error.
+     */
+    private String env(Session ses, String name, String dflt, String encoding) throws JSchException
{
         try {
-            return exec(ses, "echo " + name);
+            String res = exec(ses, "echo " + name, encoding);
+
+            if (res == null)
+                return dflt;
+
+            res = res.trim();
+
+            return res.isEmpty() ? dflt : res;
         }
         catch (IOException ignored) {
             return dflt;
@@ -256,6 +391,20 @@ public class StartNodeCallableImpl implements StartNodeCallable {
      * @throws IOException If failed.
      */
     private String exec(Session ses, String cmd) throws JSchException, IOException {
+        return exec(ses, cmd, null);
+    }
+
+    /**
+     * Gets the value of the specified environment variable.
+     *
+     * @param ses SSH session.
+     * @param cmd environment variable name.
+     * @param encoding Process output encoding, {@code null} for default charset encoding.
+     * @return environment variable value.
+     * @throws JSchException In case of SSH error.
+     * @throws IOException If failed.
+     */
+    private String exec(Session ses, final String cmd, String encoding) throws JSchException,
IOException {
         ChannelExec ch = null;
 
         try {
@@ -265,9 +414,62 @@ public class StartNodeCallableImpl implements StartNodeCallable {
 
             ch.connect();
 
-            try (BufferedReader reader = new BufferedReader(new InputStreamReader(ch.getInputStream())))
{
-                return reader.readLine();
+            if (encoding == null)
+                encoding = Charset.defaultCharset().name();
+
+            IgniteEx grid = (IgniteEx)ignite;
+
+            GridTimeoutProcessor proc = grid.context().timeout();
+
+            GridTimeoutObject to = null;
+
+            SB out = null;
+
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(ch.getInputStream(),
encoding))) {
+                String line;
+
+                boolean first = true;
+
+                while ((line = reader.readLine()) != null) {
+                    if (first)
+                        out = new SB();
+                    else
+                        out.a('\n');
+
+                    out.a(line);
+
+                    if (first) {
+                        to = new GridTimeoutObjectAdapter(EXECUTE_WAIT_TIME) {
+                            /**  */
+                            private final Thread thread = Thread.currentThread();
+
+                            @Override public void onTimeout() {
+                                thread.interrupt();
+                            }
+
+                            @Override public String toString() {
+                                return S.toString("GridTimeoutObject", "cmd", cmd, "thread",
thread);
+                            }
+                        };
+
+                        assert proc.addTimeoutObject(to) : "Timeout object was not added:
" + to;
+
+                        first = false;
+                    }
+                }
             }
+            catch (InterruptedIOException ignore) {
+                // No-op.
+            }
+            finally {
+                if (to != null) {
+                    boolean r = proc.removeTimeoutObject(to);
+
+                    assert r || to.endTime() <= U.currentTimeMillis() : "Timeout object
was not removed: " + to;
+                }
+            }
+
+            return out == null ? null : out.toString();
         }
         finally {
             if (ch != null && ch.isConnected())


Mime
View raw message