beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From al...@apache.org
Subject [beam] branch master updated: [BEAM-5378] Remove go wordcap example Dataflow runner (#6395)
Date Tue, 25 Sep 2018 00:02:06 GMT
This is an automated email from the ASF dual-hosted git repository.

altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new f048354  [BEAM-5378] Remove go wordcap example Dataflow runner (#6395)
f048354 is described below

commit f048354619f30c1dfaf7207a12c1f4ec2b10d4c6
Author: Ahmet Altay <aaltay@gmail.com>
AuthorDate: Mon Sep 24 17:01:58 2018 -0700

    [BEAM-5378] Remove go wordcap example Dataflow runner (#6395)
    
    It is no longer needed and wordcount serves as a full example.
---
 sdks/go/README.md                   | 25 ++++++++++++-
 sdks/go/examples/build.gradle       |  1 -
 sdks/go/examples/wordcap/wordcap.go | 75 -------------------------------------
 3 files changed, 24 insertions(+), 77 deletions(-)

diff --git a/sdks/go/README.md b/sdks/go/README.md
index 247c287..50e9456 100644
--- a/sdks/go/README.md
+++ b/sdks/go/README.md
@@ -31,7 +31,7 @@ most examples), follow the setup
 verify that it works by running the corresponding Java example.
 
 The examples are normal Go programs and are most easily run directly. They
-are parameterized by Go flags. For example, to run wordcount do:
+are parameterized by Go flags. For example, to run wordcount on direct runner do:
 
 ```
 $ pwd
@@ -84,6 +84,29 @@ sentence: 1
 purse: 6
 ```
 
+To run wordcount on dataflow runner do:
+
+```
+$  go run wordcount.go --runner=dataflow --project=<YOUR_GCP_PROJECT> --staging_location=<YOUR_GCS_LOCATION>/staging
--worker_harness_container_image=<YOUR_SDK_HARNESS_IMAGE_LOCATION> --output=<YOUR_GCS_LOCATION>/output
+```
+
+The output is a GCS file in this case:
+
+```
+$ gsutil cat <YOUR_GCS_LOCATION>/output* | head
+Blanket: 1
+blot: 1
+Kneeling: 3
+cautions: 1
+appears: 4
+Deserved: 1
+nettles: 1
+OSWALD: 53
+sport: 3
+Crown'd: 1
+```
+
+
 See [BUILD.md](./BUILD.md) for how to build Go code in general. See
 [CONTAINERS.md](../CONTAINERS.md) for how to build and push the Go
 SDK harness container image.
diff --git a/sdks/go/examples/build.gradle b/sdks/go/examples/build.gradle
index a101bd6..05a9245 100644
--- a/sdks/go/examples/build.gradle
+++ b/sdks/go/examples/build.gradle
@@ -66,7 +66,6 @@ golang {
     go 'build -o ./build/bin/${GOOS}_${GOARCH}/pingpong github.com/apache/beam/sdks/go/examples/pingpong'
     go 'build -o ./build/bin/${GOOS}_${GOARCH}/tornadoes github.com/apache/beam/sdks/go/examples/cookbook/tornadoes'
     go 'build -o ./build/bin/${GOOS}_${GOARCH}/windowed_wordcount github.com/apache/beam/sdks/go/examples/windowed_wordcount'
-    go 'build -o ./build/bin/${GOOS}_${GOARCH}/wordcap github.com/apache/beam/sdks/go/examples/wordcap'
     go 'build -o ./build/bin/${GOOS}_${GOARCH}/wordcount github.com/apache/beam/sdks/go/examples/wordcount'
     go 'build -o ./build/bin/${GOOS}_${GOARCH}/yatzy github.com/apache/beam/sdks/go/examples/yatzy'
   }
diff --git a/sdks/go/examples/wordcap/wordcap.go b/sdks/go/examples/wordcap/wordcap.go
deleted file mode 100644
index 096335e..0000000
--- a/sdks/go/examples/wordcap/wordcap.go
+++ /dev/null
@@ -1,75 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more
-// contributor license agreements.  See the NOTICE file distributed with
-// this work for additional information regarding copyright ownership.
-// The ASF licenses this file to You under the Apache License, Version 2.0
-// (the "License"); you may not use this file except in compliance with
-// the License.  You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"context"
-	"flag"
-	"os"
-	"regexp"
-	"strings"
-
-	"github.com/apache/beam/sdks/go/pkg/beam"
-	"github.com/apache/beam/sdks/go/pkg/beam/io/textio"
-	"github.com/apache/beam/sdks/go/pkg/beam/log"
-	"github.com/apache/beam/sdks/go/pkg/beam/transforms/filter"
-	"github.com/apache/beam/sdks/go/pkg/beam/x/beamx"
-	"github.com/apache/beam/sdks/go/pkg/beam/x/debug"
-)
-
-var (
-	input = flag.String("input", os.ExpandEnv("$GOPATH/src/github.com/apache/beam/sdks/go/data/haiku/old_pond.txt"),
"Files to read.")
-	short = flag.Bool("short", false, "Filter out long words.")
-)
-
-var wordRE = regexp.MustCompile(`[a-zA-Z]+('[a-z])?`)
-
-func extractFn(line string, emit func(string)) {
-	for _, word := range wordRE.FindAllString(line, -1) {
-		emit(word)
-	}
-}
-
-func main() {
-	flag.Parse()
-	beam.Init()
-
-	ctx := context.Background()
-
-	log.Info(ctx, "Running wordcap")
-
-	// Construct an I/O-free, linear pipeline.
-	p := beam.NewPipeline()
-	s := p.Root()
-
-	lines, err := textio.Immediate(s, *input) // Embedded data. Go flags as parameters.
-	if err != nil {
-		log.Exitf(ctx, "Failed to read %v: %v", *input, err)
-	}
-	words := beam.ParDo(s, extractFn, lines)     // Named function.
-	cap := beam.ParDo(s, strings.ToUpper, words) // Library function.
-	if *short {
-		// Conditional pipeline construction. Function literals.
-		cap = filter.Include(s, cap, func(s string) bool {
-			return len(s) < 5
-		})
-	}
-	debug.Print(s, cap) // Debug helper.
-
-	if err := beamx.Run(context.Background(), p); err != nil {
-		log.Exitf(ctx, "Failed to execute job: %v", err)
-	}
-}


Mime
View raw message