beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From echauc...@apache.org
Subject [beam] 16/45: Add TODO in Combine translations
Date Tue, 09 Jul 2019 13:18:33 GMT
This is an automated email from the ASF dual-hosted git repository.

echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 1a85496dd533364f281ec15cf9aa7f8391405823
Author: Etienne Chauchot <echauchot@apache.org>
AuthorDate: Tue May 28 16:57:47 2019 +0200

    Add TODO in Combine translations
---
 .../translation/batch/CombineGloballyTranslatorBatch.java               | 1 +
 .../translation/batch/CombinePerKeyTranslatorBatch.java                 | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineGloballyTranslatorBatch.java
b/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineGloballyTranslatorBatch.java
index fd66002..53651cf 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineGloballyTranslatorBatch.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineGloballyTranslatorBatch.java
@@ -53,6 +53,7 @@ class CombineGloballyTranslatorBatch<InputT, AccumT, OutputT>
 
     Dataset<WindowedValue<InputT>> inputDataset = context.getDataset(input);
 
+    //TODO merge windows instead of doing unwindow/window to comply with beam model
     Dataset<InputT> unWindowedDataset =
         inputDataset.map(WindowingHelpers.unwindowMapFunction(), EncoderHelpers.genericEncoder());
 
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
b/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
index 5ff0048..3d0ee8b 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
@@ -52,9 +52,11 @@ class CombinePerKeyTranslatorBatch<K, InputT, AccumT, OutputT>
 
     Dataset<WindowedValue<KV<K, InputT>>> inputDataset = context.getDataset(input);
 
+    //TODO merge windows instead of doing unwindow/window to comply with beam model
     Dataset<KV<K, InputT>> keyedDataset =
         inputDataset.map(WindowingHelpers.unwindowMapFunction(), EncoderHelpers.kvEncoder());
 
+    // TODO change extractKey impl to deal with WindowedVAlue and use it in GBK
     KeyValueGroupedDataset<K, KV<K, InputT>> groupedDataset =
         keyedDataset.groupByKey(KVHelpers.extractKey(), EncoderHelpers.genericEncoder());
 


Mime
View raw message