beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Mike Lambert (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (BEAM-1791) Errors are truncated
Date Thu, 23 Mar 2017 08:51:41 GMT

    [ https://issues.apache.org/jira/browse/BEAM-1791?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15937945#comment-15937945
] 

Mike Lambert commented on BEAM-1791:
------------------------------------

Oh, and I should add that the final error message printed to the console when running {{python
-m my_dataflow}} is similarly truncated.

> Errors are truncated
> --------------------
>
>                 Key: BEAM-1791
>                 URL: https://issues.apache.org/jira/browse/BEAM-1791
>             Project: Beam
>          Issue Type: Bug
>          Components: sdk-py
>            Reporter: Mike Lambert
>            Assignee: Ahmet Altay
>            Priority: Minor
>
> When I run my code in the Google Cloud Dataflow, I am getting some tracebacks in the
the "Errors" view, but they are truncated. (Not sure if this is a bug against this Jira component,
or should be filed somewhere else...)
> {noformat}
> (cee7eb663424a856): Traceback (most recent call last):
>   File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/batchworker.py", line
578, in do_work
>     work_executor.execute()
>   File "/usr/local/lib/python2.7/dist-packages/dataflow_worker/executor.py", line 165,
in execute
>     op.start()
>   File "dataflow_worker/shuffle_operations.py", line 49, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
(dataflow_worker/shuffle_operations.c:3442)
>     def start(self):
>   File "dataflow_worker/shuffle_operations.py", line 50, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
(dataflow_worker/shuffle_operations.c:3346)
>     with self.scoped_start_state:
>   File "dataflow_worker/shuffle_operations.py", line 63, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
(dataflow_worker/shuffle_operations.c:3261)
>     with self.shuffle_source.reader() as reader:
>   File "dataflow_worker/shuffle_operations.py", line 67, in dataflow_worker.shuffle_operations.GroupedShuffleReadOperation.start
(dataflow_worker/shuffle_operations.c:3205)
>     self.output(windowed_value)
>   File "dataflow_worker/operations.py", line 139, in dataflow_worker.operations.Operation.output
(dataflow_worker/operations.c:5717)
>     cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
>   File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive
(dataflow_worker/operations.c:3421)
>     cython.cast(Operation, consumer).process(windowed_value)
>   File "dataflow_worker/shuffle_operations.py", line 220, in dataflow_worker.shuffle_operations.BatchGroupAlsoByWindowsOperation.process
(dataflow_worker/shuffle_operations.c:8094)
>     self.output(wvalue.with_value((k, wvalue.value)))
>   File "dataflow_worker/operations.py", line 139, in dataflow_worker.operations.Operation.output
(dataflow_worker/operations.c:5717)
>     cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value)
>   File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive
(dataflow_worker/operations.c:3421)
>     cython.cast(Operation, consumer).process(windowed_value)
>   File "dataflow_worker/operations.py", line 395, in dataflow_worker.operations.DoOperation.process
(dataflow_worker/operations.c:13546)
>     with self.scoped_process_state:
>   File "dataflow_worker/operations.py", line 396, in dataflow_worker.operations.DoOperation.process
(dataflow_worker/operations.c:13500)
>     self.dofn_receiver.receive(o)
>   File "apache_beam/runners/common.py", line 195, in apache_beam.runners.common.DoFnRunner.receive
(apache_beam/runners/common.c:5142)
>     self.process(windowed_value)
>   File "apache_beam/runners/common.py", line 267, in apache_beam.runners.common.DoFnRunner.process
(apache_beam/runners/common.c:7201)
>     self.reraise_augmented(exn)
>   File "apache_beam/runners/common.py", line 281, in apache_beam.runners.common.DoFnRunner.reraise_augmented
(apache_beam/runners/common.c:7609)
>     raise
>   File "apache_beam/runners/common.py", line 263, in apache_beam.runners.common.DoFnRunner.process
(apache_beam/runners/common.c:7090)
>     self._dofn_simple_invoker(element)
>   File "apache_beam/runners/common.py", line 198, in apache_beam.runners.common.DoFnRunner._dofn_simple_invoker
(apache_beam/runners/common.c:5288)
>     self._process_outputs(element, self.dofn_process(element.value))
>   File "apache_beam/runners/common.py", line 326, in apache_beam.runners.common.DoFnRunner._process_outputs
(apache_beam/runners/common.c:8563)
>     self.main_receivers.receive(windowed_value)
>   File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive
(dataflow_worker/operations.c:3421)
>     cython.cast(Operation, consumer).process(windowed_value)
>   File "dataflow_worker/operations.py", line 395, in dataflow_worker.operations.DoOperation.process
(dataflow_worker/operations.c:13546)
>     with self.scoped_process_state:
>   File "dataflow_worker/operations.py", line 396, in dataflow_worker.operations.DoOperation.process
(dataflow_worker/operations.c:13500)
>     self.dofn_receiver.receive(o)
>   File "apache_beam/runners/common.py", line 195, in apache_beam.runners.common.DoFnRunner.receive
(apache_beam/runners/common.c:5142)
>     self.process(windowed_value)
>   File "apache_beam/runners/common.py", line 267, in apache_beam.runners.common.DoFnRunner.process
(apache_beam/runners/common.c:7201)
>     self.reraise_augmented(exn)
>   File "apache_beam/runners/common.py", line 281, in apache_beam.runners.common.DoFnRunner.reraise_augmented
(apache_beam/runners/common.c:7609)
>     raise
>   File "apache_beam/runners/common.py", line 263, in apache_beam.runners.common.DoFnRunner.process
(apache_beam/runners/common.c:7090)
>     self._dofn_simple_invoker(element)
>   File "apache_beam/runners/common.py", line 198, in apache_beam.runners.common.DoFnRunner._dofn_simple_invoker
(apache_beam/runners/common.c:5288)
>     self._process_outputs(element, self.dofn_process(element.value))
>   File "apache_beam/runners/common.py", line 326, in apache_beam.runners.common.DoFnRunner._process_outputs
(apache_beam/runners/common.c:8563)
>     self.main_receivers.receive(windowed_value)
>   File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive
(dataflow_worker/operations.c:3421)
>     cython.cast(Operation, consumer).process(windowed_value)
>   File "dataflow_worker/operations.py", line 395, in dataflow_worker.operations.DoOperation.process
(dataflow_worker/operations.c:13546)
>     with self.scoped_process_state:
>   File "dataflow_worker/operations.py", line 396, in dataflow_worker.operations.DoOperation.process
(dataflow_worker/operations.c:13500)
>     self.dofn_receiver.receive(o)
>   File "apache_beam/runners/common.py", line 195, in apache_beam.runners.common.DoFnRunner.receive
(apache_beam/runners/common.c:5142)
>     self.process(windowed_value)
>   File "apache_beam/runners/common.py", line 267, in apache_beam.runners.common.DoFnRunner.process
(apache_beam/runners/common.c:7201)
>     self.reraise_augmented(exn)
>   File "apache_beam/runners/common.py", line 281, in apache_beam.runners.common.DoFnRunner.reraise_augmented
(apache_beam/runners/common.c:7609)
>     raise
>   File "apache_beam/runners/common.py", line 263, in apache_beam.runners.common.DoFnRunner.process
(apache_beam/runners/common.c:7090)
>     self._dofn_simple_invoker(element)
>   File "apache_beam/runners/common.py", line 198, in apache_beam.runners.common.DoFnRunner._dofn_simple_invoker
(apache_beam/runners/common.c:5288)
>     self._process_outputs(element, self.dofn_process(element.value))
>   File "apache_beam/runners/common.py", line 326, in apache_beam.runners.common.DoFnRunner._process_outputs
(apache_beam/runners/common.c:8563)
>     self.main_receivers.receive(windowed_value)
>   File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive
(dataflow_worker/operations.c:3421)
>     cython.cast(Operation, consumer).process(windowed_value)
>   File "dataflow_worker/operations.py", line 395, in dataflow_worker.operations.DoOperation.process
(dataflow_worker/operations.c:13546)
>     with self.scoped_process_state:
>   File "dataflow_worker/operations.py", line 396, in dataflow_worker.operations.DoOperation.process
(dataflow_worker/operations.c:13500)
>     self.dofn_receiver.receive(o)
>   File "apache_beam/runners/common.py", line 195, in apache_beam.runners.common.DoFnRunner.receive
(apache_beam/runners/common.c:5142)
>     self.process(windowed_value)
>   File "apache_beam/runners/common.py", line 267, in apache_beam.runners.common.DoFnRunner.process
(apache_beam/runners/common.c:7201)
>     self.reraise_augmented(exn)
>   File "apache_beam/runners/common.py", line 281, in apache_beam.runners.common.DoFnRunner.reraise_augmented
(apache_beam/runners/common.c:7609)
>     raise
>   File "apache_beam/runners/common.py", line 263, in apache_beam.runners.common.DoFnRunner.process
(apache_beam/runners/common.c:7090)
>     self._dofn_simple_invoker(element)
>   File "apache_beam/runners/common.py", line 198, in apache_beam.runners.common.DoFnRunner._dofn_simple_invoker
(apache_beam/runners/common.c:5288)
>     self._process_outputs(element, self.dofn_process(element.value))
>   File "apache_beam/runners/common.py", line 326, in apache_beam.runners.common.DoFnRunner._process_outputs
(apache_beam/runners/common.c:8563)
>     self.main_receivers.receive(windowed_value)
>   File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive
(dataflow_worker/operations.c:3421)
>     cython.cast(Operation, consumer).process(windowed_value)
>   File "dataflow_worker/operations.py", line 395, in dataflow_worker.operations.DoOperation.process
(dataflow_worker/operations.c:13546)
>     with self.scoped_process_state:
>   File "dataflow_worker/operations.py", line 396, in dataflow_worker.operations.DoOperation.process
(dataflow_worker/operations.c:13500)
>     self.dofn_receiver.receive(o)
>   File "apache_beam/runners/common.py", line 195, in apache_beam.runners.common.DoFnRunner.receive
(apache_beam/runners/common.c:5142)
>     self.process(windowed_value)
>   File "apache_beam/runners/common.py", line 267, in apache_beam.runners.common.DoFnRunner.process
(apache_beam/runners/common.c:7201)
>     self.reraise_augmented(exn)
>   File "apache_beam/runners/common.py", line 281, in apache_beam.runners.common.DoFnRunner.reraise_augmented
(apache_beam/runners/common.c:7609)
>     raise
>   File "apache_beam/runners/common.py", line 263, in apache_beam.runners.common.DoFnRunner.process
(apache_beam/runners/common.c:7090)
>     self._dofn_simple_invoker(element)
>   File "apache_beam/runners/common.py", line 198, in apache_beam.runners.common.DoFnRunner._dofn_simple_invoker
(apache_beam/runners/common.c:5288)
>     self._process_outputs(element, self.dofn_process(element.value))
>   File "apache_beam/runners/common.py", line 326, in apache_beam.runners.common.DoFnRunner._process_outputs
(apache_beam/runners/common.c:8563)
>     self.main_receivers.receive(windowed_value)
>   File "dataflow_worker/operations.py", line 72, in dataflow_worker.operations.ConsumerSet.receive
(dataflow_worker/operations.c:3421)
>     cython.cast(Operation, consumer).process(windowed_valu
> {noformat}
> Notice how it is truncated at the very end...after around 10220 characters, and it hasn't
even gotten to my actual code's stacktrace yet!
> If I open the logs panel and go look at the logs, I'm luckily able to see the full thing...including
the very last line that is pointing at my code, as well as the actual exception class/message
itself.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Mime
View raw message