arrow-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ALBERTO Bocchinfuso <>
Subject [Scala/Java] Writing a RecordBatch on a file using scala
Date Thu, 04 Oct 2018 09:05:53 GMT
Hi guys,

I’m trying to write a RecordBatch on a file using scala. To do so, I use the java API.
Aside from all the imports, my code is:

imports …
object Test {
             def main(args: Array[String]) {
                         val allocator = new RootAllocator(8192);

                         val fields = List[Field](new Field("names", new FieldType(false,
ArrowType.Utf8.INSTANCE, null), null), new Field("names", new FieldType(false, new ArrowType.Int(64,
false), null), null));

                         val fieldVectors = List[FieldVector](fields(0).getFieldType().createNewSingleVector("names",
allocator, null), fields(1).getFieldType().createNewSingleVector("numbers", allocator, null));


val names = List[String]("Name1", "Name2", "Name3", "Name4", "Name5", "Name6", "Name7", "Name8");
                         val numbers = List[Int](1, 2, 3, 4, 5, 6, 7, 8);
                         val bitmap: ArrowBuf = allocator.buffer(1);

                         var fos: FileOutputStream = new FileOutputStream("TEST");
          val vsr: VectorSchemaRoot = new VectorSchemaRoot(fields.asJava, fieldVectors.asJava,
                         val afw: ArrowFileWriter = new ArrowFileWriter(vsr, null , fos.getChannel());

                         val namesBuf: ArrowBuf = allocator.buffer(2048);
                         for(i <- names){

                         var numbersBuf: ArrowBuf = allocator.buffer(2048);
                         for(i <- numbers){

                         val btch = new ArrowRecordBatch(8192, List(new ArrowFieldNode(8,
0), new ArrowFieldNode(8, 0)).asJava, List(bitmap, namesBuf, bitmap, numbersBuf).asJava);

                         val loader: VectorLoader = new VectorLoader (vsr);


After many debugging messages I get:

[error] (run-main-e) java.util.NoSuchElementException: next on empty iterator
[error] java.util.NoSuchElementException: next on empty iterator
[error]             at scala.collection.Iterator$$anon$
[error]             at scala.collection.Iterator$$anon$
[error]             at scala.collection.LinearSeqLike$$anon$
[error]             at scala.collection.convert.Wrappers$
[error]             at org.apache.arrow.vector.VectorLoader.loadBuffers(
[error]             at org.apache.arrow.vector.VectorLoader.load(
[error]             at Test$.main(RecordBatchWriteTest.scala:53)
[error]             at Test.main(RecordBatchWriteTest.scala)
[error]             at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[error]             at sun.reflect.NativeMethodAccessorImpl.invoke(
[error]             at sun.reflect.DelegatingMethodAccessorImpl.invoke(
[error]             at java.lang.reflect.Method.invoke(
[error]             at sbt.Run.invokeMain(Run.scala:93)
[error]             at sbt.Run.run0(Run.scala:87)
[error]             at sbt.Run.execute$1(Run.scala:65)
[error]             at sbt.Run.$anonfun$run$4(Run.scala:77)
[error]             at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$
[error]             at sbt.util.InterfaceUtil$$anon$1.get(InterfaceUtil.scala:10)
[error]             at sbt.TrapExit$
[error]             at
[error] java.lang.RuntimeException: Nonzero exit code: 1
[error]             at sbt.Run$.executeTrapExit(Run.scala:124)
[error]             at
[error]             at sbt.Defaults$.$anonfun$bgRunTask$5(Defaults.scala:1185)
[error]             at sbt.Defaults$.$anonfun$bgRunTask$5$adapted(Defaults.scala:1180)
[error]             at sbt.internal.BackgroundThreadPool.$anonfun$run$1(DefaultBackgroundJobService.scala:366)
[error]             at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$
[error]             at scala.util.Try$.apply(Try.scala:209)
[error]             at sbt.internal.BackgroundThreadPool$
[error]             at java.util.concurrent.ThreadPoolExecutor.runWorker(
[error]             at java.util.concurrent.ThreadPoolExecutor$
[error]             at
[error] (Compile / run) Nonzero exit code: 1

To write the code I tried to follow step by step the procedure here:

Can someone help me?

Thanks in advance,

View raw message