Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 09F142007D0 for ; Tue, 10 May 2016 23:22:22 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 08AEF16098A; Tue, 10 May 2016 21:22:22 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 074C7160877 for ; Tue, 10 May 2016 23:22:20 +0200 (CEST) Received: (qmail 34413 invoked by uid 500); 10 May 2016 21:22:20 -0000 Mailing-List: contact commits-help@spark.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list commits@spark.apache.org Received: (qmail 34404 invoked by uid 99); 10 May 2016 21:22:20 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 May 2016 21:22:20 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 2340EE02D3; Tue, 10 May 2016 21:22:20 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: andrewor14@apache.org To: commits@spark.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: spark git commit: [SPARK-15249][SQL] Use FunctionResource instead of (String, String) in CreateFunction and CatalogFunction for resource Date: Tue, 10 May 2016 21:22:20 +0000 (UTC) archived-at: Tue, 10 May 2016 21:22:22 -0000 Repository: spark Updated Branches: refs/heads/branch-2.0 95f254994 -> 1db027d11 [SPARK-15249][SQL] Use FunctionResource instead of (String, String) in CreateFunction and CatalogFunction for resource Use FunctionResource instead of (String, String) in CreateFunction and CatalogFunction for resource see: TODO's here https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala#L36 https://github.com/apache/spark/blob/master/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala#L42 Existing tests Author: Sandeep Singh Closes #13024 from techaddict/SPARK-15249. (cherry picked from commit da02d006bbb5c4fe62abd5542b9fff7d1c58603c) Signed-off-by: Andrew Or Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1db027d1 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1db027d1 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1db027d1 Branch: refs/heads/branch-2.0 Commit: 1db027d113ada12d0f11cbd7e6da8a90b4ff027b Parents: 95f2549 Author: Sandeep Singh Authored: Tue May 10 14:21:47 2016 -0700 Committer: Andrew Or Committed: Tue May 10 14:22:12 2016 -0700 ---------------------------------------------------------------------- .../apache/spark/sql/catalyst/catalog/SessionCatalog.scala | 8 ++------ .../spark/sql/catalyst/catalog/functionResources.scala | 8 ++++---- .../org/apache/spark/sql/catalyst/catalog/interface.scala | 3 +-- .../spark/sql/catalyst/catalog/ExternalCatalogSuite.scala | 4 ++-- .../spark/sql/catalyst/catalog/SessionCatalogSuite.scala | 2 +- .../org/apache/spark/sql/execution/SparkSqlParser.scala | 4 ++-- .../org/apache/spark/sql/execution/command/functions.scala | 5 ++--- .../spark/sql/execution/command/DDLCommandSuite.scala | 9 +++++++-- .../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 7 ++++--- .../org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala | 4 ++-- 10 files changed, 27 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/1db027d1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index 7505e2c..f53311c5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -687,12 +687,8 @@ class SessionCatalog( * Loads resources such as JARs and Files for a function. Every resource is represented * by a tuple (resource type, resource uri). */ - def loadFunctionResources(resources: Seq[(String, String)]): Unit = { - resources.foreach { case (resourceType, uri) => - val functionResource = - FunctionResource(FunctionResourceType.fromString(resourceType.toLowerCase), uri) - functionResourceLoader.loadResource(functionResource) - } + def loadFunctionResources(resources: Seq[FunctionResource]): Unit = { + resources.foreach(functionResourceLoader.loadResource) } /** http://git-wip-us.apache.org/repos/asf/spark/blob/1db027d1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/functionResources.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/functionResources.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/functionResources.scala index 5adcc89..7da1fe9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/functionResources.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/functionResources.scala @@ -20,16 +20,16 @@ package org.apache.spark.sql.catalyst.catalog import org.apache.spark.sql.AnalysisException /** An trait that represents the type of a resourced needed by a function. */ -sealed trait FunctionResourceType +abstract class FunctionResourceType(val resourceType: String) -object JarResource extends FunctionResourceType +object JarResource extends FunctionResourceType("jar") -object FileResource extends FunctionResourceType +object FileResource extends FunctionResourceType("file") // We do not allow users to specify a archive because it is YARN specific. // When loading resources, we will throw an exception and ask users to // use --archive with spark submit. -object ArchiveResource extends FunctionResourceType +object ArchiveResource extends FunctionResourceType("archive") object FunctionResourceType { def fromString(resourceType: String): FunctionResourceType = { http://git-wip-us.apache.org/repos/asf/spark/blob/1db027d1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index 2c6e9f5..fc2068c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -33,11 +33,10 @@ import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan} * @param className fully qualified class name, e.g. "org.apache.spark.util.MyFunc" * @param resources resource types and Uris used by the function */ -// TODO: Use FunctionResource instead of (String, String) as the element type of resources. case class CatalogFunction( identifier: FunctionIdentifier, className: String, - resources: Seq[(String, String)]) + resources: Seq[FunctionResource]) /** http://git-wip-us.apache.org/repos/asf/spark/blob/1db027d1/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala index 651be26..ae190c0 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/ExternalCatalogSuite.scala @@ -466,7 +466,7 @@ abstract class ExternalCatalogSuite extends SparkFunSuite with BeforeAndAfterEac val catalog = newBasicCatalog() assert(catalog.getFunction("db2", "func1") == CatalogFunction(FunctionIdentifier("func1", Some("db2")), funcClass, - Seq.empty[(String, String)])) + Seq.empty[FunctionResource])) intercept[AnalysisException] { catalog.getFunction("db2", "does_not_exist") } @@ -679,7 +679,7 @@ abstract class CatalogTestUtils { } def newFunc(name: String, database: Option[String] = None): CatalogFunction = { - CatalogFunction(FunctionIdentifier(name, database), funcClass, Seq.empty[(String, String)]) + CatalogFunction(FunctionIdentifier(name, database), funcClass, Seq.empty[FunctionResource]) } /** http://git-wip-us.apache.org/repos/asf/spark/blob/1db027d1/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala index f2d2e99..80422c2 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala @@ -798,7 +798,7 @@ class SessionCatalogSuite extends SparkFunSuite { val catalog = new SessionCatalog(newBasicCatalog()) val expected = CatalogFunction(FunctionIdentifier("func1", Some("db2")), funcClass, - Seq.empty[(String, String)]) + Seq.empty[FunctionResource]) assert(catalog.getFunctionMetadata(FunctionIdentifier("func1", Some("db2"))) == expected) // Get function without explicitly specifying database catalog.setCurrentDatabase("db2") http://git-wip-us.apache.org/repos/asf/spark/blob/1db027d1/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 086282d..87e6f90 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -25,7 +25,7 @@ import org.antlr.v4.runtime.tree.TerminalNode import org.apache.spark.sql.SaveMode import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.catalog.{CatalogColumn, CatalogStorageFormat, CatalogTable, CatalogTableType} +import org.apache.spark.sql.catalyst.catalog._ import org.apache.spark.sql.catalyst.parser._ import org.apache.spark.sql.catalyst.parser.SqlBaseParser._ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, OneRowRelation, ScriptInputOutputSchema} @@ -430,7 +430,7 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder { val resourceType = resource.identifier.getText.toLowerCase resourceType match { case "jar" | "file" | "archive" => - resourceType -> string(resource.STRING) + FunctionResource(FunctionResourceType.fromString(resourceType), string(resource.STRING)) case other => throw operationNotAllowed(s"CREATE FUNCTION with resource type '$resourceType'", ctx) } http://git-wip-us.apache.org/repos/asf/spark/blob/1db027d1/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala index a9aa8d7..1ea9bc5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.execution.command import org.apache.spark.sql.{AnalysisException, Row, SparkSession} import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, NoSuchFunctionException} -import org.apache.spark.sql.catalyst.catalog.CatalogFunction +import org.apache.spark.sql.catalyst.catalog.{CatalogFunction, FunctionResource} import org.apache.spark.sql.catalyst.expressions.{Attribute, ExpressionInfo} import org.apache.spark.sql.types.{StringType, StructField, StructType} @@ -39,12 +39,11 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType} * AS className [USING JAR\FILE 'uri' [, JAR|FILE 'uri']] * }}} */ -// TODO: Use Seq[FunctionResource] instead of Seq[(String, String)] for resources. case class CreateFunction( databaseName: Option[String], functionName: String, className: String, - resources: Seq[(String, String)], + resources: Seq[FunctionResource], isTemp: Boolean) extends RunnableCommand { http://git-wip-us.apache.org/repos/asf/spark/blob/1db027d1/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala index bd428a0..a728ac3 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.execution.command import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{FunctionResource, FunctionResourceType} import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.catalyst.plans.PlanTest import org.apache.spark.sql.catalyst.plans.logical.Project @@ -156,13 +157,17 @@ class DDLCommandSuite extends PlanTest { None, "helloworld", "com.matthewrathbone.example.SimpleUDFExample", - Seq(("jar", "/path/to/jar1"), ("jar", "/path/to/jar2")), + Seq( + FunctionResource(FunctionResourceType.fromString("jar"), "/path/to/jar1"), + FunctionResource(FunctionResourceType.fromString("jar"), "/path/to/jar2")), isTemp = true) val expected2 = CreateFunction( Some("hello"), "world", "com.matthewrathbone.example.SimpleUDFExample", - Seq(("archive", "/path/to/archive"), ("file", "/path/to/file")), + Seq( + FunctionResource(FunctionResourceType.fromString("archive"), "/path/to/archive"), + FunctionResource(FunctionResourceType.fromString("file"), "/path/to/file")), isTemp = false) comparePlans(parsed1, expected1) comparePlans(parsed2, expected2) http://git-wip-us.apache.org/repos/asf/spark/blob/1db027d1/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index cddc0b6..bb32459 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -677,8 +677,9 @@ private[hive] class HiveClientImpl( .asInstanceOf[Class[_ <: org.apache.hadoop.hive.ql.io.HiveOutputFormat[_, _]]] private def toHiveFunction(f: CatalogFunction, db: String): HiveFunction = { - val resourceUris = f.resources.map { case (resourceType, resourcePath) => - new ResourceUri(ResourceType.valueOf(resourceType.toUpperCase), resourcePath) + val resourceUris = f.resources.map { resource => + new ResourceUri( + ResourceType.valueOf(resource.resourceType.resourceType.toUpperCase()), resource.uri) } new HiveFunction( f.identifier.funcName, @@ -700,7 +701,7 @@ private[hive] class HiveClientImpl( case ResourceType.JAR => "jar" case r => throw new AnalysisException(s"Unknown resource type: $r") } - (resourceType, uri.getUri()) + FunctionResource(FunctionResourceType.fromString(resourceType), uri.getUri()) } new CatalogFunction(name, hf.getClassName, resources) } http://git-wip-us.apache.org/repos/asf/spark/blob/1db027d1/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala index bfe559f..d05a362 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala @@ -33,7 +33,7 @@ import org.apache.spark._ import org.apache.spark.internal.Logging import org.apache.spark.sql.{QueryTest, Row, SparkSession, SQLContext} import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} -import org.apache.spark.sql.catalyst.catalog.CatalogFunction +import org.apache.spark.sql.catalyst.catalog.{CatalogFunction, FunctionResource, JarResource} import org.apache.spark.sql.expressions.Window import org.apache.spark.sql.hive.test.{TestHive, TestHiveContext} import org.apache.spark.sql.test.ProcessTestUtils.ProcessOutputCapturer @@ -425,7 +425,7 @@ object PermanentHiveUDFTest2 extends Logging { val function = CatalogFunction( FunctionIdentifier("example_max"), "org.apache.hadoop.hive.contrib.udaf.example.UDAFExampleMax", - ("JAR" -> jar) :: Nil) + FunctionResource(JarResource, jar) :: Nil) hiveContext.sessionState.catalog.createFunction(function, ignoreIfExists = false) val source = hiveContext.createDataFrame((1 to 10).map(i => (i, s"str$i"))).toDF("key", "val") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org For additional commands, e-mail: commits-help@spark.apache.org