beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pabl...@apache.org
Subject [beam] branch master updated: Add hot key detection protos to Windmill
Date Thu, 25 Jul 2019 20:54:51 GMT
This is an automated email from the ASF dual-hosted git repository.

pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 0ae24ea  Add hot key detection protos to Windmill
     new f03b6ba  Merge pull request #9155 from rohdesamuel/update-versions
0ae24ea is described below

commit 0ae24eaca73fc38e245f5328a802d87198aedafc
Author: Sam Rohde <rohde.samuel@gmail.com>
AuthorDate: Thu Jul 25 10:33:48 2019 -0700

    Add hot key detection protos to Windmill
---
 .../worker/windmill/src/main/proto/windmill.proto             | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill.proto
b/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill.proto
index 29c664d..5310902 100644
--- a/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill.proto
+++ b/runners/google-cloud-dataflow-java/worker/windmill/src/main/proto/windmill.proto
@@ -134,6 +134,12 @@ message WatermarkHold {
   optional string state_family = 4;
 }
 
+// Proto describing a hot key detected on a given WorkItem.
+message HotKeyInfo {
+  // The age of the hot key measured from when it was first detected.
+  optional int64 hot_key_age_usec = 1;
+}
+
 message WorkItem {
   required bytes key = 1;
   required fixed64 work_token = 2;
@@ -149,6 +155,11 @@ message WorkItem {
   // Indicates that this is a new key with no data associated. This allows
   // the harness to optimize data fetching.
   optional bool is_new_key = 10;
+
+  // A hot key is a symptom of poor data distribution in which there are enough
+  // elements mapped to a single key to impact pipeline performance. When
+  // present, this field includes metadata associated with any hot key.
+  optional HotKeyInfo hot_key_info = 11;
 }
 
 message ComputationWorkItems {


Mime
View raw message