spark-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Hyukjin Kwon (Jira)" <j...@apache.org>
Subject [jira] [Updated] (SPARK-31247) Flaky test: KafkaContinuousSourceSuite.assign from latest offsets (failOnDataLoss: false)
Date Thu, 26 Mar 2020 07:00:05 GMT

     [ https://issues.apache.org/jira/browse/SPARK-31247?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

Hyukjin Kwon updated SPARK-31247:
---------------------------------
    Description: 
https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/120336/testReport/

{code}
Error Message
org.scalatest.exceptions.TestFailedException:  Error adding data: Timeout after waiting for 10000 ms. org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:78)  org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:30)  org.apache.spark.sql.kafka010.KafkaTestUtils.$anonfun$sendMessages$3(KafkaTestUtils.scala:425)  scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)  scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)  scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)  scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)  scala.collection.TraversableLike.map(TraversableLike.scala:238)  scala.collection.TraversableLike.map$(TraversableLike.scala:231)  scala.collection.AbstractTraversable.map(Traversable.scala:108)   == Progress ==    AssertOnQuery(<condition>, )    AddKafkaData(topics = Set(topic-13), data = WrappedArray(1, 2, 3), message = )    CheckAnswer: [2],[3],[4]    StopStream    StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1f1a9495,Map(),null)    CheckAnswer: [2],[3],[4]    StopStream    AddKafkaData(topics = Set(topic-13), data = WrappedArray(4, 5, 6), message = )    StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@2b3bec2c,Map(),null)    CheckAnswer: [2],[3],[4],[5],[6],[7] => AddKafkaData(topics = Set(topic-13), data = WrappedArray(7, 8), message = )    CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9]    AssertOnQuery(<condition>, Add partitions)    AddKafkaData(topics = Set(topic-13), data = WrappedArray(9, 10, 11, 12, 13, 14, 15, 16), message = )    CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17]  == Stream == Output Mode: Append Stream state: {KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]]: {"topic-13":{"2":2,"4":2,"1":1,"3":1,"0":1}}} Thread state: alive Thread stack trace: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:242) scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:258) scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:187) org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:336) org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:746) org.apache.spark.SparkContext.runJob(SparkContext.scala:2104) org.apache.spark.SparkContext.runJob(SparkContext.scala:2125) org.apache.spark.SparkContext.runJob(SparkContext.scala:2144) org.apache.spark.SparkContext.runJob(SparkContext.scala:2169) org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1006) org.apache.spark.rdd.RDD$$Lambda$2999/724038556.apply(Unknown Source) org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) org.apache.spark.rdd.RDD.withScope(RDD.scala:390) org.apache.spark.rdd.RDD.collect(RDD.scala:1005) org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:57) org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) org.apache.spark.sql.execution.SparkPlan$$Lambda$2791/4135277.apply(Unknown Source) org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) org.apache.spark.sql.execution.SparkPlan$$Lambda$2823/504830038.apply(Unknown Source) org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$4(ContinuousExecution.scala:256) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2765/297007729.apply(Unknown Source) org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100) org.apache.spark.sql.execution.SQLExecution$$$Lambda$2773/697863343.apply(Unknown Source) org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160) org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87) org.apache.spark.sql.execution.SQLExecution$$$Lambda$2766/1162891508.apply(Unknown Source) org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763) org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$3(ContinuousExecution.scala:256) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2761/1229001373.apply(Unknown Source) org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:342) org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:340) org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:255) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:109) org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:333) org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:244)   == Sink == 0:  1: [3] [4] [2] 2:  3:  4:  5: [5] [7] [6] 6:    == Plan == == Parsed Logical Plan == WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6 +- SerializeFromObject [input[0, int, false] AS value#4334]    +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int       +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2          +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]             +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}  == Analyzed Logical Plan ==  WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6 +- SerializeFromObject [input[0, int, false] AS value#4334]    +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int       +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2          +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]             +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}  == Optimized Logical Plan == WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6 +- SerializeFromObject [input[0, int, false] AS value#4334]    +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int       +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2          +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]             +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}  == Physical Plan == WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6 +- *(1) SerializeFromObject [input[0, int, false] AS value#4334]    +- *(1) MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, obj#4333: int       +- *(1) DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2          +- *(1) Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]             +- *(1) Project [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314]                +- ContinuousScan[key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan                     
Stacktrace
sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException: 
Error adding data: Timeout after waiting for 10000 ms.
org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:78)
	org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:30)
	org.apache.spark.sql.kafka010.KafkaTestUtils.$anonfun$sendMessages$3(KafkaTestUtils.scala:425)
	scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
	scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
	scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
	scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
	scala.collection.TraversableLike.map(TraversableLike.scala:238)
	scala.collection.TraversableLike.map$(TraversableLike.scala:231)
	scala.collection.AbstractTraversable.map(Traversable.scala:108)


== Progress ==
   AssertOnQuery(<condition>, )
   AddKafkaData(topics = Set(topic-13), data = WrappedArray(1, 2, 3), message = )
   CheckAnswer: [2],[3],[4]
   StopStream
   StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1f1a9495,Map(),null)
   CheckAnswer: [2],[3],[4]
   StopStream
   AddKafkaData(topics = Set(topic-13), data = WrappedArray(4, 5, 6), message = )
   StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@2b3bec2c,Map(),null)
   CheckAnswer: [2],[3],[4],[5],[6],[7]
=> AddKafkaData(topics = Set(topic-13), data = WrappedArray(7, 8), message = )
   CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9]
   AssertOnQuery(<condition>, Add partitions)
   AddKafkaData(topics = Set(topic-13), data = WrappedArray(9, 10, 11, 12, 13, 14, 15, 16), message = )
   CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17]

== Stream ==
Output Mode: Append
Stream state: {KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]]: {"topic-13":{"2":2,"4":2,"1":1,"3":1,"0":1}}}
Thread state: alive
Thread stack trace: sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:242)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:258)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:187)
org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:336)
org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:746)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2104)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2125)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2144)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2169)
org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1006)
org.apache.spark.rdd.RDD$$Lambda$2999/724038556.apply(Unknown Source)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
org.apache.spark.rdd.RDD.withScope(RDD.scala:390)
org.apache.spark.rdd.RDD.collect(RDD.scala:1005)
org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:57)
org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)
org.apache.spark.sql.execution.SparkPlan$$Lambda$2791/4135277.apply(Unknown Source)
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)
org.apache.spark.sql.execution.SparkPlan$$Lambda$2823/504830038.apply(Unknown Source)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)
org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$4(ContinuousExecution.scala:256)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2765/297007729.apply(Unknown Source)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
org.apache.spark.sql.execution.SQLExecution$$$Lambda$2773/697863343.apply(Unknown Source)
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
org.apache.spark.sql.execution.SQLExecution$$$Lambda$2766/1162891508.apply(Unknown Source)
org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$3(ContinuousExecution.scala:256)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2761/1229001373.apply(Unknown Source)
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:342)
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:340)
org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:255)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:109)
org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:333)
org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:244)


== Sink ==
0: 
1: [3] [4] [2]
2: 
3: 
4: 
5: [5] [7] [6]
6: 


== Plan ==
== Parsed Logical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334]
   +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int
      +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2
         +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]
            +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}

== Analyzed Logical Plan ==

WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334]
   +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int
      +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2
         +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]
            +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}

== Optimized Logical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334]
   +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int
      +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2
         +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]
            +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}

== Physical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- *(1) SerializeFromObject [input[0, int, false] AS value#4334]
   +- *(1) MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, obj#4333: int
      +- *(1) DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2
         +- *(1) Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]
            +- *(1) Project [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314]
               +- ContinuousScan[key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan

         
         
	at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
	at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
	at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
	at org.scalatest.Assertions.fail(Assertions.scala:1091)
	at org.scalatest.Assertions.fail$(Assertions.scala:1087)
	at org.scalatest.FunSuite.fail(FunSuite.scala:1560)
	at org.apache.spark.sql.streaming.StreamTest.failTest$1(StreamTest.scala:448)
	at org.apache.spark.sql.streaming.StreamTest.executeAction$1(StreamTest.scala:720)
	at org.apache.spark.sql.streaming.StreamTest.$anonfun$testStream$56(StreamTest.scala:774)
	at org.apache.spark.sql.streaming.StreamTest.$anonfun$testStream$56$adapted(StreamTest.scala:761)
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
	at org.apache.spark.sql.streaming.StreamTest.liftedTree1$1(StreamTest.scala:761)
	at org.apache.spark.sql.streaming.StreamTest.testStream(StreamTest.scala:760)
	at org.apache.spark.sql.streaming.StreamTest.testStream$(StreamTest.scala:330)
	at org.apache.spark.sql.kafka010.KafkaSourceTest.testStream(KafkaMicroBatchSourceSuite.scala:53)
	at org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.testFromLatestOffsets(KafkaMicroBatchSourceSuite.scala:1728)
	at org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.$anonfun$new$130(KafkaMicroBatchSourceSuite.scala:1312)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
	at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
	at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
	at org.scalatest.Transformer.apply(Transformer.scala:22)
	at org.scalatest.Transformer.apply(Transformer.scala:20)
	at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
	at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:151)
	at org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
	at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
	at org.scalatest.SuperEngine.runTestImpl(Engine.scala:286)
	at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
	at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
	at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:58)
	at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221)
	at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214)
	at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:58)
	at org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229)
	at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:393)
	at scala.collection.immutable.List.foreach(List.scala:392)
	at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:381)
	at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:376)
	at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:458)
	at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229)
	at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228)
	at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
	at org.scalatest.Suite.run(Suite.scala:1124)
	at org.scalatest.Suite.run$(Suite.scala:1106)
	at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
	at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233)
	at org.scalatest.SuperEngine.runImpl(Engine.scala:518)
	at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233)
	at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232)
	at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:58)
	at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
	at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
	at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
	at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:58)
	at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:317)
	at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:510)
	at sbt.ForkMain$Run$2.call(ForkMain.java:296)
	at sbt.ForkMain$Run$2.call(ForkMain.java:286)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)
{code}

  was:
Error Message
org.scalatest.exceptions.TestFailedException:  Error adding data: Timeout after waiting for 10000 ms. org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:78)  org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:30)  org.apache.spark.sql.kafka010.KafkaTestUtils.$anonfun$sendMessages$3(KafkaTestUtils.scala:425)  scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)  scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)  scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)  scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)  scala.collection.TraversableLike.map(TraversableLike.scala:238)  scala.collection.TraversableLike.map$(TraversableLike.scala:231)  scala.collection.AbstractTraversable.map(Traversable.scala:108)   == Progress ==    AssertOnQuery(<condition>, )    AddKafkaData(topics = Set(topic-13), data = WrappedArray(1, 2, 3), message = )    CheckAnswer: [2],[3],[4]    StopStream    StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1f1a9495,Map(),null)    CheckAnswer: [2],[3],[4]    StopStream    AddKafkaData(topics = Set(topic-13), data = WrappedArray(4, 5, 6), message = )    StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@2b3bec2c,Map(),null)    CheckAnswer: [2],[3],[4],[5],[6],[7] => AddKafkaData(topics = Set(topic-13), data = WrappedArray(7, 8), message = )    CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9]    AssertOnQuery(<condition>, Add partitions)    AddKafkaData(topics = Set(topic-13), data = WrappedArray(9, 10, 11, 12, 13, 14, 15, 16), message = )    CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17]  == Stream == Output Mode: Append Stream state: {KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]]: {"topic-13":{"2":2,"4":2,"1":1,"3":1,"0":1}}} Thread state: alive Thread stack trace: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:242) scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:258) scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:187) org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:336) org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:746) org.apache.spark.SparkContext.runJob(SparkContext.scala:2104) org.apache.spark.SparkContext.runJob(SparkContext.scala:2125) org.apache.spark.SparkContext.runJob(SparkContext.scala:2144) org.apache.spark.SparkContext.runJob(SparkContext.scala:2169) org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1006) org.apache.spark.rdd.RDD$$Lambda$2999/724038556.apply(Unknown Source) org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) org.apache.spark.rdd.RDD.withScope(RDD.scala:390) org.apache.spark.rdd.RDD.collect(RDD.scala:1005) org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:57) org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) org.apache.spark.sql.execution.SparkPlan$$Lambda$2791/4135277.apply(Unknown Source) org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) org.apache.spark.sql.execution.SparkPlan$$Lambda$2823/504830038.apply(Unknown Source) org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$4(ContinuousExecution.scala:256) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2765/297007729.apply(Unknown Source) org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100) org.apache.spark.sql.execution.SQLExecution$$$Lambda$2773/697863343.apply(Unknown Source) org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160) org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87) org.apache.spark.sql.execution.SQLExecution$$$Lambda$2766/1162891508.apply(Unknown Source) org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763) org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$3(ContinuousExecution.scala:256) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2761/1229001373.apply(Unknown Source) org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:342) org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:340) org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:255) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:109) org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:333) org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:244)   == Sink == 0:  1: [3] [4] [2] 2:  3:  4:  5: [5] [7] [6] 6:    == Plan == == Parsed Logical Plan == WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6 +- SerializeFromObject [input[0, int, false] AS value#4334]    +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int       +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2          +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]             +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}  == Analyzed Logical Plan ==  WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6 +- SerializeFromObject [input[0, int, false] AS value#4334]    +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int       +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2          +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]             +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}  == Optimized Logical Plan == WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6 +- SerializeFromObject [input[0, int, false] AS value#4334]    +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int       +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2          +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]             +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}  == Physical Plan == WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6 +- *(1) SerializeFromObject [input[0, int, false] AS value#4334]    +- *(1) MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, obj#4333: int       +- *(1) DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2          +- *(1) Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]             +- *(1) Project [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314]                +- ContinuousScan[key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan                     
Stacktrace
sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException: 
Error adding data: Timeout after waiting for 10000 ms.
org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:78)
	org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:30)
	org.apache.spark.sql.kafka010.KafkaTestUtils.$anonfun$sendMessages$3(KafkaTestUtils.scala:425)
	scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
	scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
	scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
	scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
	scala.collection.TraversableLike.map(TraversableLike.scala:238)
	scala.collection.TraversableLike.map$(TraversableLike.scala:231)
	scala.collection.AbstractTraversable.map(Traversable.scala:108)


== Progress ==
   AssertOnQuery(<condition>, )
   AddKafkaData(topics = Set(topic-13), data = WrappedArray(1, 2, 3), message = )
   CheckAnswer: [2],[3],[4]
   StopStream
   StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1f1a9495,Map(),null)
   CheckAnswer: [2],[3],[4]
   StopStream
   AddKafkaData(topics = Set(topic-13), data = WrappedArray(4, 5, 6), message = )
   StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@2b3bec2c,Map(),null)
   CheckAnswer: [2],[3],[4],[5],[6],[7]
=> AddKafkaData(topics = Set(topic-13), data = WrappedArray(7, 8), message = )
   CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9]
   AssertOnQuery(<condition>, Add partitions)
   AddKafkaData(topics = Set(topic-13), data = WrappedArray(9, 10, 11, 12, 13, 14, 15, 16), message = )
   CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17]

== Stream ==
Output Mode: Append
Stream state: {KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]]: {"topic-13":{"2":2,"4":2,"1":1,"3":1,"0":1}}}
Thread state: alive
Thread stack trace: sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:242)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:258)
scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:187)
org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:336)
org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:746)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2104)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2125)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2144)
org.apache.spark.SparkContext.runJob(SparkContext.scala:2169)
org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1006)
org.apache.spark.rdd.RDD$$Lambda$2999/724038556.apply(Unknown Source)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
org.apache.spark.rdd.RDD.withScope(RDD.scala:390)
org.apache.spark.rdd.RDD.collect(RDD.scala:1005)
org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:57)
org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)
org.apache.spark.sql.execution.SparkPlan$$Lambda$2791/4135277.apply(Unknown Source)
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)
org.apache.spark.sql.execution.SparkPlan$$Lambda$2823/504830038.apply(Unknown Source)
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)
org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$4(ContinuousExecution.scala:256)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2765/297007729.apply(Unknown Source)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
org.apache.spark.sql.execution.SQLExecution$$$Lambda$2773/697863343.apply(Unknown Source)
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
org.apache.spark.sql.execution.SQLExecution$$$Lambda$2766/1162891508.apply(Unknown Source)
org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$3(ContinuousExecution.scala:256)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2761/1229001373.apply(Unknown Source)
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:342)
org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:340)
org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:255)
org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:109)
org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:333)
org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:244)


== Sink ==
0: 
1: [3] [4] [2]
2: 
3: 
4: 
5: [5] [7] [6]
6: 


== Plan ==
== Parsed Logical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334]
   +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int
      +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2
         +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]
            +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}

== Analyzed Logical Plan ==

WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334]
   +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int
      +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2
         +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]
            +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}

== Optimized Logical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- SerializeFromObject [input[0, int, false] AS value#4334]
   +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int
      +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2
         +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]
            +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}

== Physical Plan ==
WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
+- *(1) SerializeFromObject [input[0, int, false] AS value#4334]
   +- *(1) MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, obj#4333: int
      +- *(1) DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2
         +- *(1) Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]
            +- *(1) Project [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314]
               +- ContinuousScan[key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan

         
         
	at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
	at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
	at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
	at org.scalatest.Assertions.fail(Assertions.scala:1091)
	at org.scalatest.Assertions.fail$(Assertions.scala:1087)
	at org.scalatest.FunSuite.fail(FunSuite.scala:1560)
	at org.apache.spark.sql.streaming.StreamTest.failTest$1(StreamTest.scala:448)
	at org.apache.spark.sql.streaming.StreamTest.executeAction$1(StreamTest.scala:720)
	at org.apache.spark.sql.streaming.StreamTest.$anonfun$testStream$56(StreamTest.scala:774)
	at org.apache.spark.sql.streaming.StreamTest.$anonfun$testStream$56$adapted(StreamTest.scala:761)
	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
	at org.apache.spark.sql.streaming.StreamTest.liftedTree1$1(StreamTest.scala:761)
	at org.apache.spark.sql.streaming.StreamTest.testStream(StreamTest.scala:760)
	at org.apache.spark.sql.streaming.StreamTest.testStream$(StreamTest.scala:330)
	at org.apache.spark.sql.kafka010.KafkaSourceTest.testStream(KafkaMicroBatchSourceSuite.scala:53)
	at org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.testFromLatestOffsets(KafkaMicroBatchSourceSuite.scala:1728)
	at org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.$anonfun$new$130(KafkaMicroBatchSourceSuite.scala:1312)
	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
	at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
	at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
	at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
	at org.scalatest.Transformer.apply(Transformer.scala:22)
	at org.scalatest.Transformer.apply(Transformer.scala:20)
	at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
	at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:151)
	at org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
	at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
	at org.scalatest.SuperEngine.runTestImpl(Engine.scala:286)
	at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
	at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
	at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:58)
	at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221)
	at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214)
	at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:58)
	at org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229)
	at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:393)
	at scala.collection.immutable.List.foreach(List.scala:392)
	at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:381)
	at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:376)
	at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:458)
	at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229)
	at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228)
	at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
	at org.scalatest.Suite.run(Suite.scala:1124)
	at org.scalatest.Suite.run$(Suite.scala:1106)
	at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
	at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233)
	at org.scalatest.SuperEngine.runImpl(Engine.scala:518)
	at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233)
	at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232)
	at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:58)
	at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
	at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
	at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
	at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:58)
	at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:317)
	at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:510)
	at sbt.ForkMain$Run$2.call(ForkMain.java:296)
	at sbt.ForkMain$Run$2.call(ForkMain.java:286)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)


> Flaky test: KafkaContinuousSourceSuite.assign from latest offsets (failOnDataLoss: false)
> -----------------------------------------------------------------------------------------
>
>                 Key: SPARK-31247
>                 URL: https://issues.apache.org/jira/browse/SPARK-31247
>             Project: Spark
>          Issue Type: Bug
>          Components: Structured Streaming, Tests
>    Affects Versions: 3.0.0, 3.1.0
>            Reporter: Gabor Somogyi
>            Priority: Major
>
> https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/120336/testReport/
> {code}
> Error Message
> org.scalatest.exceptions.TestFailedException:  Error adding data: Timeout after waiting for 10000 ms. org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:78)  org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:30)  org.apache.spark.sql.kafka010.KafkaTestUtils.$anonfun$sendMessages$3(KafkaTestUtils.scala:425)  scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)  scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)  scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)  scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)  scala.collection.TraversableLike.map(TraversableLike.scala:238)  scala.collection.TraversableLike.map$(TraversableLike.scala:231)  scala.collection.AbstractTraversable.map(Traversable.scala:108)   == Progress ==    AssertOnQuery(<condition>, )    AddKafkaData(topics = Set(topic-13), data = WrappedArray(1, 2, 3), message = )    CheckAnswer: [2],[3],[4]    StopStream    StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1f1a9495,Map(),null)    CheckAnswer: [2],[3],[4]    StopStream    AddKafkaData(topics = Set(topic-13), data = WrappedArray(4, 5, 6), message = )    StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@2b3bec2c,Map(),null)    CheckAnswer: [2],[3],[4],[5],[6],[7] => AddKafkaData(topics = Set(topic-13), data = WrappedArray(7, 8), message = )    CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9]    AssertOnQuery(<condition>, Add partitions)    AddKafkaData(topics = Set(topic-13), data = WrappedArray(9, 10, 11, 12, 13, 14, 15, 16), message = )    CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17]  == Stream == Output Mode: Append Stream state: {KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]]: {"topic-13":{"2":2,"4":2,"1":1,"3":1,"0":1}}} Thread state: alive Thread stack trace: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:242) scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:258) scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:187) org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:336) org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:746) org.apache.spark.SparkContext.runJob(SparkContext.scala:2104) org.apache.spark.SparkContext.runJob(SparkContext.scala:2125) org.apache.spark.SparkContext.runJob(SparkContext.scala:2144) org.apache.spark.SparkContext.runJob(SparkContext.scala:2169) org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1006) org.apache.spark.rdd.RDD$$Lambda$2999/724038556.apply(Unknown Source) org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112) org.apache.spark.rdd.RDD.withScope(RDD.scala:390) org.apache.spark.rdd.RDD.collect(RDD.scala:1005) org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:57) org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175) org.apache.spark.sql.execution.SparkPlan$$Lambda$2791/4135277.apply(Unknown Source) org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213) org.apache.spark.sql.execution.SparkPlan$$Lambda$2823/504830038.apply(Unknown Source) org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210) org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$4(ContinuousExecution.scala:256) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2765/297007729.apply(Unknown Source) org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100) org.apache.spark.sql.execution.SQLExecution$$$Lambda$2773/697863343.apply(Unknown Source) org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160) org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87) org.apache.spark.sql.execution.SQLExecution$$$Lambda$2766/1162891508.apply(Unknown Source) org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763) org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$3(ContinuousExecution.scala:256) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2761/1229001373.apply(Unknown Source) org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:342) org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:340) org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:255) org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:109) org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:333) org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:244)   == Sink == 0:  1: [3] [4] [2] 2:  3:  4:  5: [5] [7] [6] 6:    == Plan == == Parsed Logical Plan == WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6 +- SerializeFromObject [input[0, int, false] AS value#4334]    +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int       +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2          +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]             +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}  == Analyzed Logical Plan ==  WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6 +- SerializeFromObject [input[0, int, false] AS value#4334]    +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int       +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2          +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]             +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}  == Optimized Logical Plan == WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6 +- SerializeFromObject [input[0, int, false] AS value#4334]    +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int       +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2          +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]             +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}  == Physical Plan == WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6 +- *(1) SerializeFromObject [input[0, int, false] AS value#4334]    +- *(1) MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, obj#4333: int       +- *(1) DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2          +- *(1) Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]             +- *(1) Project [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314]                +- ContinuousScan[key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan                     
> Stacktrace
> sbt.ForkMain$ForkError: org.scalatest.exceptions.TestFailedException: 
> Error adding data: Timeout after waiting for 10000 ms.
> org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:78)
> 	org.apache.kafka.clients.producer.internals.FutureRecordMetadata.get(FutureRecordMetadata.java:30)
> 	org.apache.spark.sql.kafka010.KafkaTestUtils.$anonfun$sendMessages$3(KafkaTestUtils.scala:425)
> 	scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
> 	scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
> 	scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
> 	scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:38)
> 	scala.collection.TraversableLike.map(TraversableLike.scala:238)
> 	scala.collection.TraversableLike.map$(TraversableLike.scala:231)
> 	scala.collection.AbstractTraversable.map(Traversable.scala:108)
> == Progress ==
>    AssertOnQuery(<condition>, )
>    AddKafkaData(topics = Set(topic-13), data = WrappedArray(1, 2, 3), message = )
>    CheckAnswer: [2],[3],[4]
>    StopStream
>    StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@1f1a9495,Map(),null)
>    CheckAnswer: [2],[3],[4]
>    StopStream
>    AddKafkaData(topics = Set(topic-13), data = WrappedArray(4, 5, 6), message = )
>    StartStream(ContinuousTrigger(1000),org.apache.spark.util.SystemClock@2b3bec2c,Map(),null)
>    CheckAnswer: [2],[3],[4],[5],[6],[7]
> => AddKafkaData(topics = Set(topic-13), data = WrappedArray(7, 8), message = )
>    CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9]
>    AssertOnQuery(<condition>, Add partitions)
>    AddKafkaData(topics = Set(topic-13), data = WrappedArray(9, 10, 11, 12, 13, 14, 15, 16), message = )
>    CheckAnswer: [2],[3],[4],[5],[6],[7],[8],[9],[10],[11],[12],[13],[14],[15],[16],[17]
> == Stream ==
> Output Mode: Append
> Stream state: {KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]]: {"topic-13":{"2":2,"4":2,"1":1,"3":1,"0":1}}}
> Thread state: alive
> Thread stack trace: sun.misc.Unsafe.park(Native Method)
> java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
> scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:242)
> scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:258)
> scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:187)
> org.apache.spark.util.ThreadUtils$.awaitReady(ThreadUtils.scala:336)
> org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:746)
> org.apache.spark.SparkContext.runJob(SparkContext.scala:2104)
> org.apache.spark.SparkContext.runJob(SparkContext.scala:2125)
> org.apache.spark.SparkContext.runJob(SparkContext.scala:2144)
> org.apache.spark.SparkContext.runJob(SparkContext.scala:2169)
> org.apache.spark.rdd.RDD.$anonfun$collect$1(RDD.scala:1006)
> org.apache.spark.rdd.RDD$$Lambda$2999/724038556.apply(Unknown Source)
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
> org.apache.spark.rdd.RDD.withScope(RDD.scala:390)
> org.apache.spark.rdd.RDD.collect(RDD.scala:1005)
> org.apache.spark.sql.execution.streaming.continuous.WriteToContinuousDataSourceExec.doExecute(WriteToContinuousDataSourceExec.scala:57)
> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:175)
> org.apache.spark.sql.execution.SparkPlan$$Lambda$2791/4135277.apply(Unknown Source)
> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:213)
> org.apache.spark.sql.execution.SparkPlan$$Lambda$2823/504830038.apply(Unknown Source)
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:210)
> org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:171)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$4(ContinuousExecution.scala:256)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2765/297007729.apply(Unknown Source)
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:100)
> org.apache.spark.sql.execution.SQLExecution$$$Lambda$2773/697863343.apply(Unknown Source)
> org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:87)
> org.apache.spark.sql.execution.SQLExecution$$$Lambda$2766/1162891508.apply(Unknown Source)
> org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:763)
> org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.$anonfun$runContinuous$3(ContinuousExecution.scala:256)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution$$Lambda$2761/1229001373.apply(Unknown Source)
> org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken(ProgressReporter.scala:342)
> org.apache.spark.sql.execution.streaming.ProgressReporter.reportTimeTaken$(ProgressReporter.scala:340)
> org.apache.spark.sql.execution.streaming.StreamExecution.reportTimeTaken(StreamExecution.scala:68)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runContinuous(ContinuousExecution.scala:255)
> org.apache.spark.sql.execution.streaming.continuous.ContinuousExecution.runActivatedStream(ContinuousExecution.scala:109)
> org.apache.spark.sql.execution.streaming.StreamExecution.org$apache$spark$sql$execution$streaming$StreamExecution$$runStream(StreamExecution.scala:333)
> org.apache.spark.sql.execution.streaming.StreamExecution$$anon$1.run(StreamExecution.scala:244)
> == Sink ==
> 0: 
> 1: [3] [4] [2]
> 2: 
> 3: 
> 4: 
> 5: [5] [7] [6]
> 6: 
> == Plan ==
> == Parsed Logical Plan ==
> WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
> +- SerializeFromObject [input[0, int, false] AS value#4334]
>    +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int
>       +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2
>          +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]
>             +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}
> == Analyzed Logical Plan ==
> WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
> +- SerializeFromObject [input[0, int, false] AS value#4334]
>    +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int
>       +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2
>          +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]
>             +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}
> == Optimized Logical Plan ==
> WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
> +- SerializeFromObject [input[0, int, false] AS value#4334]
>    +- MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, class scala.Tuple2, [StructField(_1,StringType,true), StructField(_2,StringType,true)], obj#4333: int
>       +- DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2
>          +- Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]
>             +- StreamingDataSourceV2Relation [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314], org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@2db9dc2d, KafkaSource[Assign[topic-13-4, topic-13-3, topic-13-2, topic-13-1, topic-13-0]], {"topic-13":{"2":1,"4":1,"1":0,"3":1,"0":1}}
> == Physical Plan ==
> WriteToContinuousDataSource org.apache.spark.sql.execution.streaming.sources.MemoryStreamingWrite@2d9d12e6
> +- *(1) SerializeFromObject [input[0, int, false] AS value#4334]
>    +- *(1) MapElements org.apache.spark.sql.kafka010.KafkaSourceSuiteBase$$Lambda$3788/671932169@73bb0a67, obj#4333: int
>       +- *(1) DeserializeToObject newInstance(class scala.Tuple2), obj#4332: scala.Tuple2
>          +- *(1) Project [cast(key#4308 as string) AS key#4322, cast(value#4309 as string) AS value#4323]
>             +- *(1) Project [key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314]
>                +- ContinuousScan[key#4308, value#4309, topic#4310, partition#4311, offset#4312L, timestamp#4313, timestampType#4314] class org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan
>          
>          
> 	at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
> 	at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:529)
> 	at org.scalatest.FunSuite.newAssertionFailedException(FunSuite.scala:1560)
> 	at org.scalatest.Assertions.fail(Assertions.scala:1091)
> 	at org.scalatest.Assertions.fail$(Assertions.scala:1087)
> 	at org.scalatest.FunSuite.fail(FunSuite.scala:1560)
> 	at org.apache.spark.sql.streaming.StreamTest.failTest$1(StreamTest.scala:448)
> 	at org.apache.spark.sql.streaming.StreamTest.executeAction$1(StreamTest.scala:720)
> 	at org.apache.spark.sql.streaming.StreamTest.$anonfun$testStream$56(StreamTest.scala:774)
> 	at org.apache.spark.sql.streaming.StreamTest.$anonfun$testStream$56$adapted(StreamTest.scala:761)
> 	at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
> 	at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
> 	at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
> 	at org.apache.spark.sql.streaming.StreamTest.liftedTree1$1(StreamTest.scala:761)
> 	at org.apache.spark.sql.streaming.StreamTest.testStream(StreamTest.scala:760)
> 	at org.apache.spark.sql.streaming.StreamTest.testStream$(StreamTest.scala:330)
> 	at org.apache.spark.sql.kafka010.KafkaSourceTest.testStream(KafkaMicroBatchSourceSuite.scala:53)
> 	at org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.testFromLatestOffsets(KafkaMicroBatchSourceSuite.scala:1728)
> 	at org.apache.spark.sql.kafka010.KafkaSourceSuiteBase.$anonfun$new$130(KafkaMicroBatchSourceSuite.scala:1312)
> 	at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)
> 	at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
> 	at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
> 	at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
> 	at org.scalatest.Transformer.apply(Transformer.scala:22)
> 	at org.scalatest.Transformer.apply(Transformer.scala:20)
> 	at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
> 	at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:151)
> 	at org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
> 	at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
> 	at org.scalatest.SuperEngine.runTestImpl(Engine.scala:286)
> 	at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
> 	at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
> 	at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:58)
> 	at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221)
> 	at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214)
> 	at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:58)
> 	at org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229)
> 	at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:393)
> 	at scala.collection.immutable.List.foreach(List.scala:392)
> 	at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:381)
> 	at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:376)
> 	at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:458)
> 	at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229)
> 	at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228)
> 	at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
> 	at org.scalatest.Suite.run(Suite.scala:1124)
> 	at org.scalatest.Suite.run$(Suite.scala:1106)
> 	at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
> 	at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233)
> 	at org.scalatest.SuperEngine.runImpl(Engine.scala:518)
> 	at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233)
> 	at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232)
> 	at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:58)
> 	at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
> 	at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
> 	at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
> 	at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:58)
> 	at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:317)
> 	at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:510)
> 	at sbt.ForkMain$Run$2.call(ForkMain.java:296)
> 	at sbt.ForkMain$Run$2.call(ForkMain.java:286)
> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> 	at java.lang.Thread.run(Thread.java:748)
> {code}



--
This message was sent by Atlassian Jira
(v8.3.4#803005)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org


Mime
View raw message