hive-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Sanjay Subramanian <Sanjay.Subraman...@wizecommerce.com>
Subject need help with an error - script used to work and now it does not :-(
Date Fri, 17 May 2013 02:34:04 GMT
2013-05-16 18:57:21,094 FATAL [IPC Server handler 19 on 40222] org.apache.hadoop.mapred.TaskAttemptListenerImpl:
Task: attempt_1368666339740_0135_m_000104_1 - exited : java.lang.RuntimeException: Error in
configuring object
        at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106)
        at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:72)
        at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:130)
        at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:395)
        at org.apache.hadoop.mapred.MapTask.run(MapTask.java:334)
        at org.apache.hadoop.mapred.YarnChild$2.run(YarnChild.java:152)
        at java.security.AccessController.doPrivileged(Native Method)
        at javax.security.auth.Subject.doAs(Subject.java:396)
        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1332)
        at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:147)
Caused by: java.lang.reflect.InvocationTargetException
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:103)
        ... 9 more
Caused by: java.lang.RuntimeException: Error in configuring object
        at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:106)
        at org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:72)
        at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:130)
        at org.apache.hadoop.mapred.MapRunner.configure(MapRunner.java:38)
        ... 14 more
Caused by: java.lang.reflect.InvocationTargetException
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at org.apache.hadoop.util.ReflectionUtils.setJobConf(ReflectionUtils.java:103)
        ... 17 more
Caused by: java.lang.RuntimeException: Map operator initialization failed
        at org.apache.hadoop.hive.ql.exec.ExecMapper.configure(ExecMapper.java:121)
        ... 22 more
Caused by: java.lang.RuntimeException: cannot find field header_date from [org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector$MyField@2add5681,
org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector$MyField@295a4523,
org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector$MyField@6571120a,
org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector$MyField@6257828d,
org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector$MyField@5f3c296b,
org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector$MyField@66c360a5,
org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector$MyField@24fe2558,
org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector$MyField@2945c761,
org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector$MyField@2424c672]
        at org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.getStandardStructFieldRef(ObjectInspectorUtils.java:345)
        at org.apache.hadoop.hive.serde2.objectinspector.UnionStructObjectInspector.getStructFieldRef(UnionStructObjectInspector.java:100)
        at org.apache.hadoop.hive.ql.exec.ExprNodeColumnEvaluator.initialize(ExprNodeColumnEvaluator.java:57)
        at org.apache.hadoop.hive.ql.exec.Operator.initEvaluators(Operator.java:896)
        at org.apache.hadoop.hive.ql.exec.Operator.initEvaluatorsAndReturnStruct(Operator.java:922)
        at org.apache.hadoop.hive.ql.exec.SelectOperator.initializeOp(SelectOperator.java:60)
        at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:357)
        at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:433)
        at org.apache.hadoop.hive.ql.exec.Operator.initializeChildren(Operator.java:389)
        at org.apache.hadoop.hive.ql.exec.FilterOperator.initializeOp(FilterOperator.java:78)
        at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:357)
        at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:433)
        at org.apache.hadoop.hive.ql.exec.Operator.initializeChildren(Operator.java:389)
        at org.apache.hadoop.hive.ql.exec.TableScanOperator.initializeOp(TableScanOperator.java:166)
        at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:357)
        at org.apache.hadoop.hive.ql.exec.MapOperator.initializeOp(MapOperator.java:427)
        at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:357)
        at org.apache.hadoop.hive.ql.exec.ExecMapper.configure(ExecMapper.java:98)
        ... 22 more

MY SCRIPT is given below
=====================
hive -hiveconf hive.root.logger=INFO,console -hiveconf mapred.job.priority=VERY_HIGH -e "
SET hive.exec.compress.output=true;
SET mapred.reduce.tasks=16;
SET mapred.output.compression.codec=org.apache.hadoop.io.compress.GzipCodec;
add jar ${JAR_NAME_AND_PATH};
create temporary function collect  as 'com.wizecommerce.utils.hive.udf.GenericUDAFCollect';
create temporary function isnextagip  as 'com.wizecommerce.utils.hive.udf.IsNextagIP';
create temporary function isfrombot  as 'com.wizecommerce.utils.hive.udf.IsFromBot';
create temporary function processblankkeyword  as 'com.wizecommerce.utils.hive.udf.ProcessBlankKeyword';
create temporary function getSellersProdImpr as 'com.wizecommerce.utils.hive.udf.GetSellersWithValidSellerIdsProdImpr';
create temporary function getProgramCode as 'com.wizecommerce.utils.hive.udf.GetProgramCodeFromSellerClickContext';
INSERT OVERWRITE DIRECTORY '/user/beeswax/warehouse/${HIVE_OUTPUT_TBL}/${DATE_STR}'
SELECT
    h.header_date_donotquery as date_,
    h.header_id as impression_id,
    h.header_searchsessionid as search_session_id,
    h.cached_visitid as visit_id ,
    split(h.server_name_donotquery,'[\.]')[0] as server,
    h.cached_ip ip,
    h.header_adnodeid ad_nodes,
    if(concat_ws(',' , getSellersProdImpr(collect_set(concat_ws('|',
                                       if(h.seller_sellerid is null, 'null',cast(h.seller_sellerid
as STRING)),
                                       if(h.seller_tagid is null,'null',cast(h.seller_tagid
as STRING)),
                                       cast(IF(h.seller_subtotal IS NULL, -1, h.seller_subtotal)
 as STRING),
                                       cast(IF(h.seller_pricetier IS NULL, -1, h.seller_pricetier)
as STRING),
                                       cast(IF(h.seller_pricerank IS  NULL, -1, h.seller_pricerank)
as STRING),
                                       cast(IF(h.seller_cpc IS NULL, -1, h.seller_cpc) as
STRING),
                                       h.program_code_notnull)))) = '', NULL, concat_ws(','
, getSellersProdImpr(collect_set(concat_ws('|',
                                       if(h.seller_sellerid is null, 'null',cast(h.seller_sellerid
as STRING)),
                                       if(h.seller_tagid is null,'null',cast(h.seller_tagid
as STRING)),
                                       cast(IF(h.seller_subtotal IS NULL, -1, h.seller_subtotal)
 as STRING),
                                       cast(IF(h.seller_pricetier IS NULL, -1, h.seller_pricetier)
as STRING),
                                       cast(IF(h.seller_pricerank IS  NULL, -1, h.seller_pricerank)
as STRING),
                                       cast(IF(h.seller_cpc IS NULL, -1, h.seller_cpc) as
STRING),
                                       h.program_code_notnull))))) as visible_sellers,

    if(concat_ws(',' , getSellersProdImpr(collect_set(concat_ws('|',
                                       if(sh.seller_id is null,'null',cast(sh.seller_id as
STRING)),
                                       if(sh.tag_id is null, 'null', cast(sh.tag_id as STRING)),
                                       '-1.0',
                                       cast(IF(sh.price_tier IS NULL, -1, sh.price_tier) as
STRING),
                                       '-1',
                                       cast(IF(sh.price_tier IS NULL, -1.0, sh.price_tier*1.0)
as STRING),
                                       h.program_code_null)))) = '', NULL, concat_ws(',' ,
getSellersProdImpr(collect_set(concat_ws('|',
                                       if(sh.seller_id is null,'null',cast(sh.seller_id as
STRING)),
                                       if(sh.tag_id is null, 'null', cast(sh.tag_id as STRING)),
                                       '-1.0',
                                       cast(IF(sh.price_tier IS NULL, -1, sh.price_tier) as
STRING),
                                       '-1',
                                       cast(IF(sh.price_tier IS NULL, -1.0, sh.price_tier*1.0)
as STRING),
                                       h.program_code_null))))) as invisible_sellers
FROM
     (SELECT
          header_id,
          header_date,
          header_date_donotquery,
          header_searchsessionid,
          cached_visitid,
          cached_ip,
          header_adnodeid,
          server_name_donotquery,
          seller_sellerid,
          seller_tagid,
          cast (regexp_replace(seller_subtotal,',','.') as DOUBLE) as seller_subtotal,
          seller_pricetier,
          seller_pricerank,
          CAST(CAST(seller_cpc as INT) as DOUBLE) as seller_cpc,
          cast(getProgramCode('${THISHOST}', '${REST_API_SERVER_NAME}',seller_clickcontext)
as STRING) as program_code_notnull,
          cast(getProgramCode('${THISHOST}', '${REST_API_SERVER_NAME}', '') as STRING) as
program_code_null
      FROM
          product_impressions_hive_only
      WHERE
         header_date='${DATE_STR}'
      AND
         cached_recordid IS NOT NULL
      AND
         isnextagip(cached_ip) = FALSE
      AND
         isfrombot(cached_visitid) = FALSE
      AND
         header_skipsellerloggingflag = 0
     ) h

LEFT OUTER JOIN
     (SELECT
           *
      FROM
           prodimpr_seller_hidden
      WHERE
           date_seller = '${DATE_STR}'
     ) sh
ON
     h.header_id = sh.header_id
AND
     sh.date_seller=h.header_date
GROUP BY
     h.header_date_donotquery,
     h.header_id,
     h.header_searchsessionid,
     h.cached_visitid,
     h.server_name_donotquery,
     h.cached_ip,
     h.header_adnodeid
;
"


CONFIDENTIALITY NOTICE
======================
This email message and any attachments are for the exclusive use of the intended recipient(s)
and may contain confidential and privileged information. Any unauthorized review, use, disclosure
or distribution is prohibited. If you are not the intended recipient, please contact the sender
by reply email and destroy all copies of the original message along with any attachments,
from your computer system. If you are the intended recipient, please be advised that the content
of this message is subject to access, review and disclosure by the sender's Email System Administrator.

Mime
View raw message