Return-Path: X-Original-To: apmail-ignite-user-archive@minotaur.apache.org Delivered-To: apmail-ignite-user-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D6EB2184BF for ; Mon, 21 Dec 2015 15:07:09 +0000 (UTC) Received: (qmail 18301 invoked by uid 500); 21 Dec 2015 15:07:09 -0000 Delivered-To: apmail-ignite-user-archive@ignite.apache.org Received: (qmail 18244 invoked by uid 500); 21 Dec 2015 15:07:09 -0000 Mailing-List: contact user-help@ignite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: user@ignite.apache.org Delivered-To: mailing list user@ignite.apache.org Received: (qmail 18234 invoked by uid 99); 21 Dec 2015 15:07:09 -0000 Received: from Unknown (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 21 Dec 2015 15:07:09 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id E179BC068A for ; Mon, 21 Dec 2015 15:07:08 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 3 X-Spam-Level: *** X-Spam-Status: No, score=3 tagged_above=-999 required=6.31 tests=[DKIM_SIGNED=0.1, DKIM_VALID=-0.1, HTML_MESSAGE=3, SPF_PASS=-0.001, URIBL_BLOCKED=0.001] autolearn=disabled Authentication-Results: spamd1-us-west.apache.org (amavisd-new); dkim=pass (2048-bit key) header.d=gridgain-com.20150623.gappssmtp.com Received: from mx1-us-east.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id ZDtBm8-b5T1v for ; Mon, 21 Dec 2015 15:06:57 +0000 (UTC) Received: from mail-oi0-f47.google.com (mail-oi0-f47.google.com [209.85.218.47]) by mx1-us-east.apache.org (ASF Mail Server at mx1-us-east.apache.org) with ESMTPS id 8D569439B3 for ; Mon, 21 Dec 2015 15:06:57 +0000 (UTC) Received: by mail-oi0-f47.google.com with SMTP id y66so93158060oig.0 for ; Mon, 21 Dec 2015 07:06:57 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gridgain-com.20150623.gappssmtp.com; s=20150623; h=mime-version:in-reply-to:references:date:message-id:subject:from:to :content-type; bh=10XUMOC8GpLjC/h0tt8ZwYHR+Qj1+KUzzzVmfnCT7LY=; b=NUrz9RRT++1VucoC2g5XRSb6Viwo/s8h8tl4DirkMboCf3i9wMeEm5w8bG5mbY9t9r L9j0vQu8PfrwG8yPWhvYjQ2/2pQM58bxHF/V66Xr25Z5bV2C5JIpMpHrop0mvILFOgPM XMHOOf2Z+wcp0w12OItVE7PbmQ+tfokbSn7OSVeKiC8YoBeox1b9iIvfzivKpKUWEkd4 CW+AEcvwGMqRWFFGin5K38ZWIhG2kLKjMsrYr3Dr9OpdJo9sPKENA+Kksxb3C9g5HaeN MJ9vHx5jx9h7C99Syne7ITcpUUbpBeeNMm653wA8CLhtES5GL4HXl83/LbzbLhHuGyB9 EOCg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20130820; h=x-gm-message-state:mime-version:in-reply-to:references:date :message-id:subject:from:to:content-type; bh=10XUMOC8GpLjC/h0tt8ZwYHR+Qj1+KUzzzVmfnCT7LY=; b=MxmE8oH6wxKIZfCVYxxjQ89nST+3pTMMV2UHRvkoNLXj64R22EBGydUqQqBLjqiZ2P CYSbKft87hQQ9DAfVVLJKNYUASLDJHZGyiKGV4jqM8bDFYK9O8iZPZaboCquLACiKI5+ iL+KqnBgzlV9mbcxPTqgji1yf0RZeGLKd+pOp8nIPDjfjFAPrRQu8nX/p7HTCgomAsBq v1L4BBMr5+Dzhc9ImM+YiZk8SSLOaon6CTZ8gqEpKTWerpbCdDa7MawjkNG87sZjim4z bjcddnkO1qnJ7/ixLkUxu52684Uuy3quBcwkOG7QLAqKMmK3TuHBe2UVuwjC3Ncifnpu ZReg== X-Gm-Message-State: ALoCoQkkfT1+tKP6dBxxSWq7/qdpJNXqJwczBN1wwdGf6u7zOPhTJqJBiGw+E+B1NBE27Plj9wjKF7VYKS/fsCfCooKj7ZRHfDktzOPfllzkJ9Goa4mwxeA= MIME-Version: 1.0 X-Received: by 10.202.187.6 with SMTP id l6mr7582606oif.17.1450710411225; Mon, 21 Dec 2015 07:06:51 -0800 (PST) Received: by 10.182.39.105 with HTTP; Mon, 21 Dec 2015 07:06:51 -0800 (PST) In-Reply-To: References: <1450456274807-2257.post@n6.nabble.com> <1450702423708-2262.post@n6.nabble.com> Date: Mon, 21 Dec 2015 18:06:51 +0300 Message-ID: Subject: Re: Ignite and Spark Streaming Integration Using Java From: Andrey Gura To: user@ignite.apache.org Content-Type: multipart/alternative; boundary=001a113cc348bbc642052769d6f5 --001a113cc348bbc642052769d6f5 Content-Type: text/plain; charset=UTF-8 Jennifer, There is no need in ExampleNodes for this test. On Mon, Dec 21, 2015 at 6:03 PM, Jennifer Coston < Jennifer.Coston@raytheon.com> wrote: > Andrey, > > Thank you for the feedback. I have updated my code with the changes that > you provided, but I am seeing the following errors when I try to run it. > When I run the test, I first start two ExampleNodes and then run the JUnit > test. Is that still the proper procedure? > > class org.apache.ignite.IgniteException: Failed to start manager: > GridManagerAdapter [enabled=true, > name=org.apache.ignite.internal.managers.discovery.GridDiscoveryManager] > at > org.apache.ignite.internal.util.IgniteUtils.convertException(IgniteUtils.java:881) > at org.apache.ignite.Ignition.start(Ignition.java:306) > at > wordCount_test.TestSparkIgniteWordCount.setUp(TestSparkIgniteWordCount.java:40) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at > org.junit.internal.runners.BeforeAndAfterRunner.invokeMethod(BeforeAndAfterRunner.java:74) > at > org.junit.internal.runners.BeforeAndAfterRunner.runBefores(BeforeAndAfterRunner.java:50) > at > org.junit.internal.runners.BeforeAndAfterRunner.runProtected(BeforeAndAfterRunner.java:33) > at > org.junit.internal.runners.TestMethodRunner.runMethod(TestMethodRunner.java:75) > at > org.junit.internal.runners.TestMethodRunner.run(TestMethodRunner.java:45) > at > org.junit.internal.runners.TestClassMethodsRunner.invokeTestMethod(TestClassMethodsRunner.java:71) > at > org.junit.internal.runners.TestClassMethodsRunner.run(TestClassMethodsRunner.java:35) > at > org.junit.internal.runners.TestClassRunner$1.runUnprotected(TestClassRunner.java:42) > at > org.junit.internal.runners.BeforeAndAfterRunner.runProtected(BeforeAndAfterRunner.java:34) > at org.junit.internal.runners.TestClassRunner.run(TestClassRunner.java:52) > at > org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:50) > at > org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38) > at > org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:459) > at > org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:675) > at > org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:382) > at > org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:192) > Caused by: class org.apache.ignite.IgniteCheckedException: Failed to start > manager: GridManagerAdapter [enabled=true, > name=org.apache.ignite.internal.managers.discovery.GridDiscoveryManager] > at > org.apache.ignite.internal.IgniteKernal.startManager(IgniteKernal.java:1488) > at org.apache.ignite.internal.IgniteKernal.start(IgniteKernal.java:908) > at > org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:1617) > at > org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1484) > at org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:965) > at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:494) > at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:470) > at org.apache.ignite.Ignition.start(Ignition.java:303) > ... 21 more > Caused by: class org.apache.ignite.IgniteCheckedException: Remote node has > peer class loading enabled flag different from local [locId8=3b7130ea, > locPeerClassLoading=false, rmtId8=d92e93a6, rmtPeerClassLoading=true, > rmtAddrs=[2601:151:c200:6d4:0:0:0:ef63/0:0:0:0:0:0:0:1, > DULL-AI356580.us.ray.com/127.0.0.1, > DULL-AI356580.us.ray.com/147.25.241.114, /2002:9319:f172:0:0:0:9319:f172, > /2601:151:c200:6d4:0:0:0:ef63]] > at > org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.checkAttributes(GridDiscoveryManager.java:1028) > at > org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.start(GridDiscoveryManager.java:686) > at > org.apache.ignite.internal.IgniteKernal.startManager(IgniteKernal.java:1485) > ... 28 more > > > Here is my updated JUnit Test: > > package wordCount_test; > > import static org.junit.Assert.*; > > import java.io.File; > import java.io.InputStream; > import java.io.InputStreamReader; > import java.io.LineNumberReader; > > import org.apache.ignite.lang.IgniteOutClosure; > import org.apache.ignite.spark.JavaIgniteContext; > import org.apache.ignite.cache.affinity.AffinityUuid; > import org.apache.ignite.configuration.*; > import org.apache.ignite.internal.util.typedef.F; > import org.apache.ignite.Ignite; > import org.apache.ignite.IgniteCache; > import org.apache.ignite.IgniteDataStreamer; > import org.apache.ignite.Ignition; > import org.apache.spark.api.java.JavaPairRDD; > import org.apache.spark.api.java.JavaRDD; > import org.apache.spark.api.java.JavaSparkContext; > import org.junit.After; > import org.junit.Before; > import org.junit.Test; > > import testWordCount.SparkIgniteWordCount; > import testWordCount.CacheConfig; > import testWordCount.ExamplesUtils; > > public class TestSparkIgniteWordCount { > > JavaSparkContext jsc; > JavaIgniteContext jic; > File txtFile; > > > //In order to run this test, you must first start two ExampleNodeStartup > Servers > @Before > public void setUp() throws Exception { > Ignite ignite = Ignition.start(); > jsc = new JavaSparkContext("local[2]", "testSparkWordCount"); > jic = new JavaIgniteContext<>(jsc, new > IgniteOutClosure(){ > @Override public IgniteConfiguration apply() { > return new IgniteConfiguration(); > } > }); > > txtFile = new File("AIW_WordCount"); > if(txtFile.exists()){ > txtFile.delete(); > } > > } > > @After > public void tearDown() throws Exception { > if(jsc != null){ > jsc.stop(); > jsc = null; > } > if(jic != null){ > jic = null; > } > Ignition.stopAll(true); > } > > @Test > public void testSparkInit() { > assertNotNull(jsc.sc()); > } > > @Test > public void testIgniteInit(){ > assertNotNull(jic); > } > > @Test > public void testSparkWordCount() { > SparkIgniteWordCount streamWords = new SparkIgniteWordCount(); > > try { > JavaRDD textFile = jsc > .textFile("alice-in-wonderland.txt"); > JavaPairRDD wordCounts = streamWords > .countWords(textFile); > > wordCounts.saveAsTextFile("AIW_WordCount"); > } catch (Exception e) { > e.printStackTrace(); > } > > assertTrue(true); > } > > } > > > Thank you, > Jennifer > > -- Andrey Gura GridGain Systems, Inc. www.gridgain.com --001a113cc348bbc642052769d6f5 Content-Type: text/html; charset=UTF-8 Content-Transfer-Encoding: quoted-printable
Jennifer,

There is no need in ExampleNo= des for this test.

On Mon, Dec 21, 2015 at 6:03 PM, Jennifer Coston <Je= nnifer.Coston@raytheon.com> wrote:

Andrey,

Thank you for the feedback. I have upd= ated my code with the changes that you provided, but I am seeing the follow= ing errors when I try to run it. When I run the test, I first start two Exa= mpleNodes and then run the JUnit test. Is that still the proper procedure?<= /font>

class org.apache.ignit= e.IgniteException: Failed to start manager: GridManagerAdapter [enabled=3Dt= rue, name=3Dorg.apache.ignite.internal.managers.discovery.GridDiscoveryMana= ger]
at org.apache.ignite.= internal.util.IgniteUtils.convertException(IgniteUtils.java:881)
at org.apache.ignite.= Ignition.start(Ignition.java:306)
at wordCount_test.Tes= tSparkIgniteWordCount.setUp(TestSparkIgniteWordCount.java:40)
at sun.reflect.Native= MethodAccessorImpl.invoke0(Native Method)
at sun.reflect.Native= MethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.Delega= tingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)<= br> at java.lang.reflect.= Method.invoke(Method.java:606)
at org.junit.internal= .runners.BeforeAndAfterRunner.invokeMethod(BeforeAndAfterRunner.java:74)
at org.junit.internal= .runners.BeforeAndAfterRunner.runBefores(BeforeAndAfterRunner.java:50)
at org.junit.internal= .runners.BeforeAndAfterRunner.runProtected(BeforeAndAfterRunner.java:33)
at org.junit.internal= .runners.TestMethodRunner.runMethod(TestMethodRunner.java:75)
at org.junit.internal= .runners.TestMethodRunner.run(TestMethodRunner.java:45)
at org.junit.internal= .runners.TestClassMethodsRunner.invokeTestMethod(TestClassMethodsRunner.jav= a:71)
at org.junit.internal= .runners.TestClassMethodsRunner.run(TestClassMethodsRunner.java:35)<= br> at org.junit.internal= .runners.TestClassRunner$1.runUnprotected(TestClassRunner.java:42) at org.junit.internal= .runners.BeforeAndAfterRunner.runProtected(BeforeAndAfterRunner.java:34)
at org.junit.internal= .runners.TestClassRunner.run(TestClassRunner.java:52)
at org.eclipse.jdt.in= ternal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:50)
at org.eclipse.jdt.in= ternal.junit.runner.TestExecution.run(TestExecution.java:38)
at org.eclipse.jdt.in= ternal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:459)
at org.eclipse.jdt.in= ternal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:675)
at org.eclipse.jdt.in= ternal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:382)<= br> at org.eclipse.jdt.in= ternal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:192)=
Caused by: class org.a= pache.ignite.IgniteCheckedException: Failed to start manager: GridManagerAd= apter [enabled=3Dtrue, name=3Dorg.apache.ignite.internal.managers.discovery= .GridDiscoveryManager]
at org.apache.ignite.= internal.IgniteKernal.startManager(IgniteKernal.java:1488)
at org.apache.ignite.= internal.IgniteKernal.start(IgniteKernal.java:908)
at org.apache.ignite.= internal.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:1617)=
at org.apache.ignite.= internal.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1484)<= br> at org.apache.ignite.= internal.IgnitionEx.start0(IgnitionEx.java:965)
at org.apache.ignite.= internal.IgnitionEx.start(IgnitionEx.java:494)
at org.apache.ignite.= internal.IgnitionEx.start(IgnitionEx.java:470)
at org.apache.ignite.= Ignition.start(Ignition.java:303)
... 21 more Caused by: class org.a= pache.ignite.IgniteCheckedException: Remote node has peer class loading ena= bled flag different from local [locId8=3D3b7130ea, locPeerClassLoading=3Dfa= lse, rmtId8=3Dd92e93a6, rmtPeerClassLoading=3Dtrue, rmtAddrs=3D[2601:151:c2= 00:6d4:0:0:0:ef63/0:0:0:0:0:0:0:1, DULL-AI356580.us.ray.com/127.0.0.1, D= ULL-AI356580.us.ray.com/147.25.241.114, /2002:9319:f172:0:0:0:9319:f172= , /2601:151:c200:6d4:0:0:0:ef63]]
at org.apache.ignite.= internal.managers.discovery.GridDiscoveryManager.checkAttributes(GridDiscov= eryManager.java:1028)
at org.apache.ignite.= internal.managers.discovery.GridDiscoveryManager.start(GridDiscoveryManager= .java:686)
at org.apache.ignite.= internal.IgniteKernal.startManager(IgniteKernal.java:1485)
... 28 more

Here is my updated JUnit Test:<= span class=3D"">

package wordCount_test;

import static org.junit.Assert.*;

import java.io.File;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;

import org.apache.ignite.lang.IgniteOu= tClosure;
import org.apache.ignite.spark.JavaIgn= iteContext;
import org.apache.ignite.cache.affinit= y.AffinityUuid;
import org.apache.ignite.configuration= .*;
import org.apache.ignite.internal.util= .typedef.F;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;<= /font>
import org.apache.ignite.IgniteDataStr= eamer;
import org.apache.ignite.Ignition;
import org.apache.spark.api.java.JavaP= airRDD;
import org.apache.spark.api.java.JavaR= DD;
import org.apache.spark.api.java.JavaS= parkContext;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import testWordCount.SparkIgnit= eWordCount;
import testWordCount.CacheConfig;
import testWordCount.ExamplesUtils;

public class TestSparkIgniteWordCount = {

JavaSparkContext jsc;
JavaIgniteContext<String, Integer&g= t; jic;
File txtFile;


//In order to run this test, yo= u must first start two ExampleNodeStartup Servers @Before
public void setUp() throws Exception {=
Ignite ignite =3D Ignition.start();
=C2=A0 =C2=A0 jsc =3D new JavaSparkCon= text("local[2]", "testSparkWordCount");
=C2=A0 =C2=A0 jic =3D new JavaIgniteCo= ntext<>(jsc, new IgniteOutClosure<IgniteConfiguration>(){
=C2=A0 =C2=A0 @Override public Ignite= Configuration apply() {
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2= =A0 return new IgniteConfiguration();
=C2=A0 =C2=A0 =C2=A0 =C2=A0 } =C2=A0 =C2=A0 });
=C2=A0 =C2=A0
=C2=A0 =C2=A0 txtFile =3D new File(&qu= ot;AIW_WordCount");
=C2=A0 =C2=A0 if(txtFile.exists()){
=C2=A0 =C2=A0 =C2=A0 =C2=A0 txtFile.de= lete();
=C2=A0 =C2=A0 }

}

@After
public void tearDown() throws Exceptio= n {
=C2=A0 =C2=A0 if(jsc !=3D null)= {
=C2=A0 =C2=A0 =C2=A0 =C2=A0 jsc.stop()= ;
=C2=A0 =C2=A0 =C2=A0 =C2=A0 jsc =3D nu= ll;
=C2=A0 =C2=A0 }
=C2=A0 =C2=A0 if(jic !=3D null){
=C2=A0 =C2=A0 jic =3D null; =C2=A0 =C2=A0 }
=C2=A0 =C2=A0 Ignition.stopAll(true);<= /font>
}

@Test
public void testSparkInit() { =C2=A0 =C2=A0 assertNotNull(jsc.sc());
}

@Test
public void testIgniteInit(){ assertNotNull(jic);
}
=C2=A0
@Test
public void testSparkWordCount() {
SparkIgniteWordCount streamWords =3D = new SparkIgniteWordCount();

try {
JavaRDD<String> textFile =3D j= sc
 .textFile("alice-in-wo= nderland.txt");
JavaPairRDD<String, Integer> w= ordCounts =3D streamWords
.countWords(textFile);

wordCounts.saveAsTextFile("AIW_= WordCount");
 } catch (Exception e) {=
e.printStackTrace();
}

assertTrue(true);
}

}


    Thank you,
    Jennifer




--
Andrey Gura
GridGain = Systems, Inc.
--001a113cc348bbc642052769d6f5--