carbondata-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenerlu <...@git.apache.org>
Subject [GitHub] carbondata pull request #964: [CARBONDATA-1099] Fixed bug for carbon-spark-s...
Date Tue, 06 Jun 2017 16:00:52 GMT
Github user chenerlu commented on a diff in the pull request:

    https://github.com/apache/carbondata/pull/964#discussion_r120403537
  
    --- Diff: integration/spark2/src/main/scala/org/apache/spark/repl/CarbonSparkILoop.scala
---
    @@ -19,54 +19,66 @@ package org.apache.spark.repl
     
     class CarbonSparkILoop extends SparkILoop {
     
    -  override def initializeSpark() {
    -    intp.beQuietDuring {
    -      command("""
    -         if(org.apache.spark.repl.carbon.Main.interp == null) {
    -           org.apache.spark.repl.carbon.Main.main(Array[String]())
    -         }
    -              """)
    -      command("val i1 = org.apache.spark.repl.carbon.Main.interp")
    -      command("import i1._")
    -      command("""
    -         @transient val sc = {
    -           val _sc = i1.createSparkContext()
    -           println("Spark context available as sc.")
    -           _sc
    -         }
    -              """)
    -      command("import org.apache.spark.SparkContext._")
    -      command("import org.apache.spark.sql.CarbonContext")
    -      command("""
    -         @transient val cc = {
    -           val _cc = {
    -             import java.io.File
    -             val path = System.getenv("CARBON_HOME") + "/bin/carbonshellstore"
    -             val store = new File(path)
    -             store.mkdirs()
    -             val storePath = sc.getConf.getOption("spark.carbon.storepath")
    -                  .getOrElse(store.getCanonicalPath)
    -             new CarbonContext(sc, storePath, store.getCanonicalPath)
    -           }
    -           println("Carbon context available as cc.")
    -           _cc
    -         }
    -              """)
    +  private def initOriginSpark(): Unit = {
    +    processLine("""
    +        @transient val spark = if (org.apache.spark.repl.Main.sparkSession != null) {
    +            org.apache.spark.repl.Main.sparkSession
    +          } else {
    +            org.apache.spark.repl.Main.createSparkSession()
    +          }
    +        @transient val sc = {
    +          val _sc = spark.sparkContext
    +          if (_sc.getConf.getBoolean("spark.ui.reverseProxy", false)) {
    +            val proxyUrl = _sc.getConf.get("spark.ui.reverseProxyUrl", null)
    +            if (proxyUrl != null) {
    +              println(s"Spark Context Web UI is available at " +
    +                s"${proxyUrl}/proxy/${_sc.applicationId}")
    +            } else {
    +              println(s"Spark Context Web UI is available at Spark Master Public URL")
    +            }
    +          } else {
    +            _sc.uiWebUrl.foreach {
    +              webUrl => println(s"Spark context Web UI available at ${webUrl}")
    +            }
    +          }
    +          println("Spark context available as 'sc' " +
    +            s"(master = ${_sc.master}, app id = ${_sc.applicationId}).")
    +          println("Spark session available as 'spark'.")
    +          _sc
    +        }
    +        """)
    +    processLine("import org.apache.spark.SparkContext._")
    +    processLine("import spark.implicits._")
    +    processLine("import spark.sql")
    +    processLine("import org.apache.spark.sql.functions._")
    +  }
     
    -      command("import org.apache.spark.sql.SQLContext")
    -      command("""
    -         @transient val sqlContext = {
    -           val _sqlContext = new SQLContext(sc)
    -           println("SQL context available as sqlContext.")
    -           _sqlContext
    -         }
    -              """)
    -      command("import sqlContext.implicits._")
    -      command("import sqlContext.sql")
    +  private def initCarbon(): Unit = {
    +    processLine("""
    +      import org.apache.spark.sql.SparkSession
    +      import org.apache.spark.sql.CarbonSession._
    +      @transient val carbon = {
    +        val _carbon = {
    +          import java.io.File
    +          val path = System.getenv("CARBON_HOME") + "/bin/carbonshellstore"
    +          val store = new File(path)
    +          store.mkdirs()
    +          val storePath = sc.getConf.getOption("spark.carbon.storepath")
    +                 .getOrElse(store.getCanonicalPath)
    +          SparkSession.builder().config(sc.getConf).getOrCreateCarbonSession(storePath)
    --- End diff --
    
    If you do not specify the metastore path, it will generate carbon.metastore which is in
same level with carbondata project.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message