spark-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Hyukjin Kwon (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (SPARK-26408) java.util.NoSuchElementException: None.get at scala.None$.get(Option.scala:347)
Date Mon, 24 Dec 2018 04:56:00 GMT

    [ https://issues.apache.org/jira/browse/SPARK-26408?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16728140#comment-16728140
] 

Hyukjin Kwon commented on SPARK-26408:
--------------------------------------

Yes, it doesn't looks like Spark's problem.

> java.util.NoSuchElementException: None.get at scala.None$.get(Option.scala:347)
> -------------------------------------------------------------------------------
>
>                 Key: SPARK-26408
>                 URL: https://issues.apache.org/jira/browse/SPARK-26408
>             Project: Spark
>          Issue Type: Bug
>          Components: SQL
>    Affects Versions: 2.3.2
>         Environment: Spark version 2.3.2
> Scala version 2.11.8
> Hbase version 1.4.7
>            Reporter: Amit Siddhu
>            Priority: Major
>
> {code:java}
> sudo spark-shell --packages com.hortonworks:shc-core:1.1.1-2.1-s_2.11 --repositories
http://repo.hortonworks.com/content/groups/public/
> {code}
> {code:java}
> import org.apache.spark.sql.{SQLContext, _}
> import org.apache.spark.sql.execution.datasources.hbase._
> import org.apache.spark.{SparkConf, SparkContext}
> import spark.sqlContext.implicits._
> {code}
> {code:java}
> def withCatalog(cat: String): DataFrame = {
>   spark.sqlContext
>   .read
>   .options(Map(HBaseTableCatalog.tableCatalog->cat))
>   .format("org.apache.spark.sql.execution.datasources.hbase")
>   .load()
> }
> {code}
> {code:java}
> def motorQuoteCatatog = s"""{ |"table":{"namespace":"default", "name":"public.motor_product_quote",
"tableCoder":"PrimitiveType"}, |"rowkey":"id", |"columns":{ |"id":{"cf":"rowkey", "col":"id",
"type":"string"}, |"quote_id":{"cf":"motor_product_quote", "col":"quote_id", "type":"string"},
|"vehicle_id":{"cf":"motor_product_quote", "col":"vehicle_id", "type":"bigint"}, |"is_new":{"cf":"motor_product_quote",
"col":"is_new", "type":"boolean"}, |"date_of_manufacture":{"cf":"motor_product_quote", "col":"date_of_manufacture",
"type":"string"}, |"raw_data":{"cf":"motor_product_quote", "col":"raw_data", "type":"string"},
|"is_processed":{"cf":"motor_product_quote", "col":"is_processed", "type":"boolean"}, |"created_on":{"cf":"motor_product_quote",
"col":"created_on", "type":"string"}, |"type":{"cf":"motor_product_quote", "col":"type", "type":"string"},
|"requirement_id":{"cf":"motor_product_quote", "col":"requirement_id", "type":"int"}, |"previous_policy_id":{"cf":"motor_product_quote",
"col":"type", "previous_policy_id":"int"}, |"parent_quote_id":{"cf":"motor_product_quote",
"col":"type", "parent_quote_id":"int"}, |"ticket_id":{"cf":"motor_product_quote", "col":"type",
"ticket_id":"int"}, |"tracker_id":{"cf":"motor_product_quote", "col":"tracker_id", "type":"int"},
|"category":{"cf":"motor_product_quote", "col":"category", "type":"string"}, |"sales_channel_id":{"cf":"motor_product_quote",
"col":"sales_channel_id", "type":"int"}, |"policy_type":{"cf":"motor_product_quote", "col":"policy_type",
"type":"string"}, |"original_quote_created_by_id":{"cf":"motor_product_quote", "col":"type",
"original_quote_created_by_id":"int"}, |"created_by_id":{"cf":"motor_product_quote", "col":"created_by_id",
"type":"int"}, |"mobile":{"cf":"motor_product_quote", "col":"mobile", "type":"string"}, |"registration_number":{"cf":"motor_product_quote",
"col":"registration_number", "type":"string"} |} |}""".stripMargin
> {code}
>  
> {code:java}
> val df = withCatalog(motorQuoteCatatog){code}
> {code:java}
> java.util.NoSuchElementException: None.get
>  at scala.None$.get(Option.scala:347)
>  at scala.None$.get(Option.scala:345)
>  at org.apache.spark.sql.execution.datasources.hbase.Field.<init>       (HBaseTableCatalog.scala:102)
>  at  org.apache.spark.sql.execution.datasources.hbase.HBaseTableCatalog$$anonfun$ap ply$3.apply(HBaseTableCatalog.scala:286)
>  at org.apache.spark.sql.execution.datasources.hbase.HBaseTableCatalog$$anonfun$apply$3.apply(HBaseTableCatalog.scala:281)
> at scala.collection.immutable.List.foreach(List.scala:381)
>  at org.apache.spark.sql.execution.datasources.hbase.HBaseTableCatalog$.apply(HBaseTableCatalog.scala:281)
>  at org.apache.spark.sql.execution.datasources.hbase.HBaseRelation.<init>(HBaseRelation.scala:80)
>  at org.apache.spark.sql.execution.datasources.hbase.DefaultSource.createRelation(HBaseRelation.scala:51)
>  at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:341)
>  at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:239)
>  at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:227)
>  at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:164)
>  at withCatalog(<console>:38)
>  ... 55 elided
> {code}



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org


Mime
View raw message