spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wenc...@apache.org
Subject spark git commit: [SPARK-17927][SQL] Remove dead code in WriterContainer.
Date Fri, 14 Oct 2016 04:36:11 GMT
Repository: spark
Updated Branches:
  refs/heads/master 44cbb61b3 -> 8543996c3


[SPARK-17927][SQL] Remove dead code in WriterContainer.

## What changes were proposed in this pull request?
speculationEnabled and DATASOURCE_OUTPUTPATH seem like just dead code.

## How was this patch tested?
Tests should fail if they are not dead code.

Author: Reynold Xin <rxin@databricks.com>

Closes #15477 from rxin/SPARK-17927.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8543996c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8543996c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8543996c

Branch: refs/heads/master
Commit: 8543996c3f44098a521fc6b90ca0bb575f606e2a
Parents: 44cbb61
Author: Reynold Xin <rxin@databricks.com>
Authored: Fri Oct 14 12:35:59 2016 +0800
Committer: Wenchen Fan <wenchen@databricks.com>
Committed: Fri Oct 14 12:35:59 2016 +0800

----------------------------------------------------------------------
 .../sql/execution/datasources/WriterContainer.scala      | 11 -----------
 1 file changed, 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/8543996c/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriterContainer.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriterContainer.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriterContainer.scala
index 7880c7c..253aa44 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriterContainer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriterContainer.scala
@@ -49,7 +49,6 @@ private[datasources] case class WriteRelation(
 
 object WriterContainer {
   val DATASOURCE_WRITEJOBUUID = "spark.sql.sources.writeJobUUID"
-  val DATASOURCE_OUTPUTPATH = "spark.sql.sources.output.path"
 }
 
 private[datasources] abstract class BaseWriterContainer(
@@ -73,9 +72,6 @@ private[datasources] abstract class BaseWriterContainer(
   // This is only used on driver side.
   @transient private val jobContext: JobContext = job
 
-  private val speculationEnabled: Boolean =
-    relation.sparkSession.sparkContext.conf.getBoolean("spark.speculation", defaultValue
= false)
-
   // The following fields are initialized and used on both driver and executor side.
   @transient protected var outputCommitter: OutputCommitter = _
   @transient private var jobId: JobID = _
@@ -247,8 +243,6 @@ private[datasources] class DefaultWriterContainer(
 
   def writeRows(taskContext: TaskContext, iterator: Iterator[InternalRow]): Unit = {
     executorSideSetup(taskContext)
-    val configuration = taskAttemptContext.getConfiguration
-    configuration.set(WriterContainer.DATASOURCE_OUTPUTPATH, outputPath)
     var writer = newOutputWriter(getWorkPath)
     writer.initConverter(dataSchema)
 
@@ -353,15 +347,10 @@ private[datasources] class DynamicPartitionWriterContainer(
   private def newOutputWriter(
       key: InternalRow,
       getPartitionString: UnsafeProjection): OutputWriter = {
-    val configuration = taskAttemptContext.getConfiguration
     val path = if (partitionColumns.nonEmpty) {
       val partitionPath = getPartitionString(key).getString(0)
-      configuration.set(
-        WriterContainer.DATASOURCE_OUTPUTPATH,
-        new Path(outputPath, partitionPath).toString)
       new Path(getWorkPath, partitionPath).toString
     } else {
-      configuration.set(WriterContainer.DATASOURCE_OUTPUTPATH, outputPath)
       getWorkPath
     }
     val bucketId = getBucketIdFromKey(key)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message