Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 4AF262009EE for ; Wed, 18 May 2016 12:58:10 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 49961160A29; Wed, 18 May 2016 10:58:10 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 16DA7160A33 for ; Wed, 18 May 2016 12:58:08 +0200 (CEST) Received: (qmail 50375 invoked by uid 500); 18 May 2016 10:58:08 -0000 Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.apache.org Delivered-To: mailing list commits@ignite.apache.org Received: (qmail 49784 invoked by uid 99); 18 May 2016 10:58:08 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 May 2016 10:58:08 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id F2F32E05E1; Wed, 18 May 2016 10:58:07 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vozerov@apache.org To: commits@ignite.apache.org Date: Wed, 18 May 2016 10:58:25 -0000 Message-Id: <8212f341c6e64254a554dd315948c0cd@git.apache.org> In-Reply-To: <96323b7653014efd85e6d708e9f78bd8@git.apache.org> References: <96323b7653014efd85e6d708e9f78bd8@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [19/38] ignite git commit: IGNITE-3093: Hadoop: Fixed map reduce planner for a case when passed file has "igfs://" scheme, but doesn't exist in the file system. archived-at: Wed, 18 May 2016 10:58:10 -0000 IGNITE-3093: Hadoop: Fixed map reduce planner for a case when passed file has "igfs://" scheme, but doesn't exist in the file system. Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2461d0d1 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2461d0d1 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2461d0d1 Branch: refs/heads/ignite-3165 Commit: 2461d0d1b18803c497a5494709b499012ac1e30f Parents: 8f7e650 Author: vozerov-gridgain Authored: Fri May 6 16:12:36 2016 +0300 Committer: vozerov-gridgain Committed: Fri May 6 16:18:15 2016 +0300 ---------------------------------------------------------------------- .../mapreduce/IgniteHadoopMapReducePlanner.java | 72 +++++++++++--------- .../HadoopDefaultMapReducePlannerSelfTest.java | 2 +- 2 files changed, 39 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/2461d0d1/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java index 1562a89..287b5ec 100644 --- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java +++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/mapreduce/IgniteHadoopMapReducePlanner.java @@ -179,54 +179,58 @@ public class IgniteHadoopMapReducePlanner implements HadoopMapReducePlanner { igfs = (IgfsEx)((IgniteEx)ignite).igfsx(endpoint.igfs()); if (igfs != null && !igfs.isProxy(split0.file())) { - Collection blocks; + IgfsPath path = new IgfsPath(split0.file()); - try { - blocks = igfs.affinity(new IgfsPath(split0.file()), split0.start(), split0.length()); - } - catch (IgniteException e) { - throw new IgniteCheckedException(e); - } + if (igfs.exists(path)) { + Collection blocks; + + try { + blocks = igfs.affinity(path, split0.start(), split0.length()); + } + catch (IgniteException e) { + throw new IgniteCheckedException(e); + } - assert blocks != null; + assert blocks != null; - if (blocks.size() == 1) - // Fast-path, split consists of one IGFS block (as in most cases). - return bestNode(blocks.iterator().next().nodeIds(), topIds, nodeLoads, false); - else { - // Slow-path, file consists of multiple IGFS blocks. First, find the most co-located nodes. - Map nodeMap = new HashMap<>(); + if (blocks.size() == 1) + // Fast-path, split consists of one IGFS block (as in most cases). + return bestNode(blocks.iterator().next().nodeIds(), topIds, nodeLoads, false); + else { + // Slow-path, file consists of multiple IGFS blocks. First, find the most co-located nodes. + Map nodeMap = new HashMap<>(); - List bestNodeIds = null; - long bestLen = -1L; + List bestNodeIds = null; + long bestLen = -1L; - for (IgfsBlockLocation block : blocks) { - for (UUID blockNodeId : block.nodeIds()) { - if (topIds.contains(blockNodeId)) { - Long oldLen = nodeMap.get(blockNodeId); - long newLen = oldLen == null ? block.length() : oldLen + block.length(); + for (IgfsBlockLocation block : blocks) { + for (UUID blockNodeId : block.nodeIds()) { + if (topIds.contains(blockNodeId)) { + Long oldLen = nodeMap.get(blockNodeId); + long newLen = oldLen == null ? block.length() : oldLen + block.length(); - nodeMap.put(blockNodeId, newLen); + nodeMap.put(blockNodeId, newLen); - if (bestNodeIds == null || bestLen < newLen) { - bestNodeIds = new ArrayList<>(1); + if (bestNodeIds == null || bestLen < newLen) { + bestNodeIds = new ArrayList<>(1); - bestNodeIds.add(blockNodeId); + bestNodeIds.add(blockNodeId); - bestLen = newLen; - } - else if (bestLen == newLen) { - assert !F.isEmpty(bestNodeIds); + bestLen = newLen; + } + else if (bestLen == newLen) { + assert !F.isEmpty(bestNodeIds); - bestNodeIds.add(blockNodeId); + bestNodeIds.add(blockNodeId); + } } } } - } - if (bestNodeIds != null) { - return bestNodeIds.size() == 1 ? bestNodeIds.get(0) : - bestNode(bestNodeIds, topIds, nodeLoads, true); + if (bestNodeIds != null) { + return bestNodeIds.size() == 1 ? bestNodeIds.get(0) : + bestNode(bestNodeIds, topIds, nodeLoads, true); + } } } } http://git-wip-us.apache.org/repos/asf/ignite/blob/2461d0d1/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java index 2bdf28c..b38f3a2 100644 --- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java +++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java @@ -785,7 +785,7 @@ public class HadoopDefaultMapReducePlannerSelfTest extends HadoopAbstractSelfTes /** {@inheritDoc} */ @Override public boolean exists(IgfsPath path) { - return false; + return true; } /** {@inheritDoc} */