Return-Path: X-Original-To: apmail-ignite-commits-archive@minotaur.apache.org Delivered-To: apmail-ignite-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 29A631780C for ; Fri, 10 Jul 2015 10:47:18 +0000 (UTC) Received: (qmail 52389 invoked by uid 500); 10 Jul 2015 10:47:18 -0000 Delivered-To: apmail-ignite-commits-archive@ignite.apache.org Received: (qmail 52356 invoked by uid 500); 10 Jul 2015 10:47:18 -0000 Mailing-List: contact commits-help@ignite.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.incubator.apache.org Delivered-To: mailing list commits@ignite.incubator.apache.org Received: (qmail 52345 invoked by uid 99); 10 Jul 2015 10:47:18 -0000 Received: from Unknown (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Jul 2015 10:47:18 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id 9CB9A18199F for ; Fri, 10 Jul 2015 10:47:17 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 0.791 X-Spam-Level: X-Spam-Status: No, score=0.791 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, T_RP_MATCHES_RCVD=-0.01, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-us-west.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id wUcyw_GbD2Ph for ; Fri, 10 Jul 2015 10:47:04 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-west.apache.org (ASF Mail Server at mx1-us-west.apache.org) with SMTP id 5DC62261E6 for ; Fri, 10 Jul 2015 10:46:53 +0000 (UTC) Received: (qmail 46328 invoked by uid 99); 10 Jul 2015 10:46:53 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Jul 2015 10:46:53 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C18A8E6866; Fri, 10 Jul 2015 10:46:52 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sboikov@apache.org To: commits@ignite.incubator.apache.org Date: Fri, 10 Jul 2015 10:47:30 -0000 Message-Id: <91bfc97f61674ca8b8a9651c9c56984f@git.apache.org> In-Reply-To: <25eef573190e411489510a566d2abc7f@git.apache.org> References: <25eef573190e411489510a566d2abc7f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [39/50] [abbrv] incubator-ignite git commit: Fixed review notes. Fixed review notes. Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/7e0aef7f Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/7e0aef7f Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/7e0aef7f Branch: refs/heads/ignite-1056 Commit: 7e0aef7f061e393406a1bdafb62530f5d5bb8d43 Parents: f627976 Author: nikolay_tikhonov Authored: Thu Jul 9 17:53:46 2015 +0300 Committer: nikolay_tikhonov Committed: Thu Jul 9 17:53:46 2015 +0300 ---------------------------------------------------------------------- .../apache/ignite/yarn/ApplicationMaster.java | 5 +- .../apache/ignite/yarn/ClusterProperties.java | 22 +++----- .../org/apache/ignite/yarn/IgniteContainer.java | 4 +- .../org/apache/ignite/yarn/IgniteProvider.java | 58 ++++++-------------- .../apache/ignite/yarn/IgniteYarnClient.java | 5 +- 5 files changed, 33 insertions(+), 61 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7e0aef7f/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java index 9169178..0d1fc53 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java @@ -74,7 +74,8 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { private Map containers = new ConcurrentHashMap<>(); /** - * @param + * @param ignitePath Hdfs path to ignite. + * @param props Cluster properties. */ public ApplicationMaster(String ignitePath, ClusterProperties props) throws Exception { this.conf = new YarnConfiguration(); @@ -146,7 +147,7 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler { /** * @param cont Container. - * @return {@code True} if + * @return {@code True} if container satisfies requirements. */ private boolean checkContainer(Container cont) { // Check limit on running nodes. http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7e0aef7f/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java index a2d88c0..e7bdc84 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ClusterProperties.java @@ -69,13 +69,10 @@ public class ClusterProperties { private double nodeCnt = DEFAULT_IGNITE_NODE_COUNT; /** */ - public static final String IGNITE_VERSION = "IGNITE_VERSION"; - - /** */ - public static final String DEFAULT_IGNITE_VERSION = "latest"; + public static final String IGNITE_URL = "IGNITE_URL"; /** Ignite version. */ - private String igniteVer = DEFAULT_IGNITE_VERSION; + private String igniteUrl = null; /** */ public static final String IGNITE_WORKING_DIR = "IGNITE_WORKING_DIR"; @@ -152,7 +149,6 @@ public class ClusterProperties { return clusterName; } - /** * @return CPU count limit. */ @@ -209,8 +205,8 @@ public class ClusterProperties { /** * @return Ignite version. */ - public String igniteVer() { - return igniteVer; + public String igniteUrl() { + return igniteUrl; } /** @@ -256,14 +252,14 @@ public class ClusterProperties { } /** - * @return Licence path. + * @return Ignite hdfs path. */ public String ignitePath() { return ignitePath; } /** - * @return Licence path. + * @return Jvm opts. */ public String jvmOpts() { return jvmOpts; @@ -298,7 +294,7 @@ public class ClusterProperties { prop.memPerNode = getDoubleProperty(IGNITE_MEMORY_PER_NODE, props, DEFAULT_MEM_PER_NODE); prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, props, DEFAULT_IGNITE_NODE_COUNT); - prop.igniteVer = getStringProperty(IGNITE_VERSION, props, DEFAULT_IGNITE_VERSION); + prop.igniteUrl = getStringProperty(IGNITE_URL, props, null); prop.ignitePath = getStringProperty(IGNITE_PATH, props, null); prop.licencePath = getStringProperty(LICENCE_PATH, props, null); prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, props, null); @@ -338,7 +334,7 @@ public class ClusterProperties { prop.memPerNode = getDoubleProperty(IGNITE_MEMORY_PER_NODE, null, DEFAULT_MEM_PER_NODE); prop.nodeCnt = getDoubleProperty(IGNITE_NODE_COUNT, null, DEFAULT_IGNITE_NODE_COUNT); - prop.igniteVer = getStringProperty(IGNITE_VERSION, null, DEFAULT_IGNITE_VERSION); + prop.igniteUrl = getStringProperty(IGNITE_URL, null, null); prop.ignitePath = getStringProperty(IGNITE_PATH, null, null); prop.licencePath = getStringProperty(LICENCE_PATH, null, null); prop.jvmOpts = getStringProperty(IGNITE_JVM_OPTS, null, null); @@ -376,7 +372,7 @@ public class ClusterProperties { envs.put(IGNITE_MEMORY_PER_NODE, toEnvVal(memPerNode)); envs.put(IGNITE_NODE_COUNT, toEnvVal(nodeCnt)); - envs.put(IGNITE_VERSION, toEnvVal(igniteVer)); + envs.put(IGNITE_URL, toEnvVal(igniteUrl)); envs.put(IGNITE_PATH, toEnvVal(ignitePath)); envs.put(LICENCE_PATH, toEnvVal(licencePath)); envs.put(IGNITE_JVM_OPTS, toEnvVal(jvmOpts)); http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7e0aef7f/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteContainer.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteContainer.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteContainer.java index 313e9e1..c3cec52 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteContainer.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteContainer.java @@ -77,9 +77,7 @@ public class IgniteContainer { return mem; } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public String toString() { return "IgniteTask [host=" + nodeId.getHost() + ", cpuCores=" + cpuCores + ", mem=" + mem + ']'; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7e0aef7f/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java index 253752d..a5d655d 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteProvider.java @@ -33,9 +33,6 @@ public class IgniteProvider { public static final String DOWNLOAD_LINK = "http://tiny.cc/updater/download_community.php"; /** */ - public static final String DIRECT_DOWNLOAD_LINK = "http://www.gridgain.com/media/gridgain-community-fabric-"; - - /** */ private ClusterProperties props; /** */ @@ -175,44 +172,19 @@ public class IgniteProvider { * @return Ignite. */ public Path getIgnite(String version) throws Exception { - File folder = checkDownloadFolder(); - - // Check to hdfs contains required ignite version. - List hdfsFiles = findIgnites(fs, props.igniteReleasesDir()); - - if (hdfsFiles != null && !hdfsFiles.isEmpty()) { - for (String fileName : hdfsFiles) { - if (fileName.equals("gridgain-community-fabric-" + version + ".zip")) - return new Path(formatPath(props.igniteReleasesDir(), version)); - } - } - - // Check local fs. - List localFiles = findIgnites(folder); - - if (localFiles != null) { - for (String fileName : localFiles) { - if (fileName.equals("gridgain-community-fabric-" + version + ".zip")) { - Path dst = new Path(formatPath(props.igniteReleasesDir(), version)); - - fs.copyFromLocalFile(new Path(formatPath(props.igniteLocalWorkDir(), latestVersion)), dst); - - return dst; - } - } - } + checkDownloadFolder(); // Download ignite. - downloadIgnite(version); + String fileName = downloadIgnite(version); - Path dst = new Path(formatPath(props.igniteReleasesDir(), version)); + Path dst = new Path(props.igniteReleasesDir() + File.separator + fileName); - fs.copyFromLocalFile(new Path(formatPath(props.igniteLocalWorkDir(), latestVersion)), dst); + if (!fs.exists(dst)) + fs.copyFromLocalFile(new Path(props.igniteLocalWorkDir() + File.separator + fileName), dst); return dst; } - /** * @param folder folder * @param version version @@ -265,23 +237,26 @@ public class IgniteProvider { } /** - * @param version The current latest version. - * @return Ignite archive. + * @param igniteUrl Url to ignite. + * @return Ignite file name. */ - private String downloadIgnite(String version) { + private String downloadIgnite(String igniteUrl) { try { - URL url = new URL(DIRECT_DOWNLOAD_LINK + version + ".zip"); + URL url = new URL(igniteUrl); HttpURLConnection conn = (HttpURLConnection)url.openConnection(); int code = conn.getResponseCode(); if (code == 200) { - checkDownloadFolder(); - String fileName = fileName(url.toString()); - FileOutputStream outFile = new FileOutputStream(props.igniteLocalWorkDir() + File.separator + fileName); + String filePath = props.igniteLocalWorkDir() + File.separator + fileName; + + if (new File(filePath).exists()) + return fileName; + + FileOutputStream outFile = new FileOutputStream(filePath); outFile.getChannel().transferFrom(Channels.newChannel(conn.getInputStream()), 0, Long.MAX_VALUE); @@ -306,6 +281,9 @@ public class IgniteProvider { if (!file.exists()) file.mkdirs(); + if (!file.exists()) + throw new RuntimeException("Couldn't create local directory! Path: " + file.toURI()); + return file; } http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7e0aef7f/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java ---------------------------------------------------------------------- diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java index 299f028..78e262d 100644 --- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java +++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java @@ -154,11 +154,10 @@ public class IgniteYarnClient { private static Path getIgnite(ClusterProperties props, FileSystem fileSystem) throws Exception { IgniteProvider provider = new IgniteProvider(props, fileSystem); - if (props.igniteVer() == null - || props.igniteVer().equalsIgnoreCase(ClusterProperties.DEFAULT_IGNITE_VERSION)) + if (props.igniteUrl() == null) return provider.getIgnite(); else - return provider.getIgnite(props.igniteVer()); + return provider.getIgnite(props.igniteUrl()); } /**