Return-Path: X-Original-To: apmail-spark-commits-archive@minotaur.apache.org Delivered-To: apmail-spark-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4FAE411B83 for ; Thu, 10 Jul 2014 23:03:34 +0000 (UTC) Received: (qmail 2213 invoked by uid 500); 10 Jul 2014 23:03:34 -0000 Delivered-To: apmail-spark-commits-archive@spark.apache.org Received: (qmail 2173 invoked by uid 500); 10 Jul 2014 23:03:34 -0000 Mailing-List: contact commits-help@spark.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@spark.apache.org Delivered-To: mailing list commits@spark.apache.org Received: (qmail 2160 invoked by uid 99); 10 Jul 2014 23:03:34 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 Jul 2014 23:03:34 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id F31E29AC26F; Thu, 10 Jul 2014 23:03:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: rxin@apache.org To: commits@spark.apache.org Message-Id: <357c22edae0e4131a0fcba07d793f33a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: SPARK-2427: Fix Scala examples that use the wrong command line arguments index Date: Thu, 10 Jul 2014 23:03:33 +0000 (UTC) Repository: spark Updated Branches: refs/heads/master 2dd672485 -> ae8ca4dfb SPARK-2427: Fix Scala examples that use the wrong command line arguments index The Scala examples HBaseTest and HdfsTest don't use the correct indexes for the command line arguments. This due to to the fix of JIRA 1565, where these examples were not correctly adapted to the new usage of the submit script. Author: Artjom-Metro Author: Artjom-Metro Closes #1353 from Artjom-Metro/fix_examples and squashes the following commits: 6111801 [Artjom-Metro] Reduce the default number of iterations cfaa73c [Artjom-Metro] Fix some examples that use the wrong index to access the command line arguments Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ae8ca4df Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ae8ca4df Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ae8ca4df Branch: refs/heads/master Commit: ae8ca4dfbacd5a5197fb41722607ad99c190f768 Parents: 2dd6724 Author: Artjom-Metro Authored: Thu Jul 10 16:03:30 2014 -0700 Committer: Reynold Xin Committed: Thu Jul 10 16:03:30 2014 -0700 ---------------------------------------------------------------------- .../main/scala/org/apache/spark/examples/HBaseTest.scala | 6 +++--- .../main/scala/org/apache/spark/examples/HdfsTest.scala | 10 ++++++++-- .../scala/org/apache/spark/examples/SparkPageRank.scala | 6 +++++- 3 files changed, 16 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/ae8ca4df/examples/src/main/scala/org/apache/spark/examples/HBaseTest.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/spark/examples/HBaseTest.scala b/examples/src/main/scala/org/apache/spark/examples/HBaseTest.scala index 4893b01..8226733 100644 --- a/examples/src/main/scala/org/apache/spark/examples/HBaseTest.scala +++ b/examples/src/main/scala/org/apache/spark/examples/HBaseTest.scala @@ -31,12 +31,12 @@ object HBaseTest { val conf = HBaseConfiguration.create() // Other options for configuring scan behavior are available. More information available at // http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/mapreduce/TableInputFormat.html - conf.set(TableInputFormat.INPUT_TABLE, args(1)) + conf.set(TableInputFormat.INPUT_TABLE, args(0)) // Initialize hBase table if necessary val admin = new HBaseAdmin(conf) - if(!admin.isTableAvailable(args(1))) { - val tableDesc = new HTableDescriptor(args(1)) + if (!admin.isTableAvailable(args(0))) { + val tableDesc = new HTableDescriptor(args(0)) admin.createTable(tableDesc) } http://git-wip-us.apache.org/repos/asf/spark/blob/ae8ca4df/examples/src/main/scala/org/apache/spark/examples/HdfsTest.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/spark/examples/HdfsTest.scala b/examples/src/main/scala/org/apache/spark/examples/HdfsTest.scala index 331de3a..ed2b38e 100644 --- a/examples/src/main/scala/org/apache/spark/examples/HdfsTest.scala +++ b/examples/src/main/scala/org/apache/spark/examples/HdfsTest.scala @@ -19,16 +19,22 @@ package org.apache.spark.examples import org.apache.spark._ + object HdfsTest { + + /** Usage: HdfsTest [file] */ def main(args: Array[String]) { + if (args.length < 1) { + System.err.println("Usage: HdfsTest ") + System.exit(1) + } val sparkConf = new SparkConf().setAppName("HdfsTest") val sc = new SparkContext(sparkConf) - val file = sc.textFile(args(1)) + val file = sc.textFile(args(0)) val mapped = file.map(s => s.length).cache() for (iter <- 1 to 10) { val start = System.currentTimeMillis() for (x <- mapped) { x + 2 } - // println("Processing: " + x) val end = System.currentTimeMillis() println("Iteration " + iter + " took " + (end-start) + " ms") } http://git-wip-us.apache.org/repos/asf/spark/blob/ae8ca4df/examples/src/main/scala/org/apache/spark/examples/SparkPageRank.scala ---------------------------------------------------------------------- diff --git a/examples/src/main/scala/org/apache/spark/examples/SparkPageRank.scala b/examples/src/main/scala/org/apache/spark/examples/SparkPageRank.scala index 40b36c7..4c7e006 100644 --- a/examples/src/main/scala/org/apache/spark/examples/SparkPageRank.scala +++ b/examples/src/main/scala/org/apache/spark/examples/SparkPageRank.scala @@ -31,8 +31,12 @@ import org.apache.spark.{SparkConf, SparkContext} */ object SparkPageRank { def main(args: Array[String]) { + if (args.length < 1) { + System.err.println("Usage: SparkPageRank ") + System.exit(1) + } val sparkConf = new SparkConf().setAppName("PageRank") - var iters = args(1).toInt + val iters = if (args.length > 0) args(1).toInt else 10 val ctx = new SparkContext(sparkConf) val lines = ctx.textFile(args(0), 1) val links = lines.map{ s =>