atlas-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Vimal Sharma (JIRA)" <j...@apache.org>
Subject [jira] [Assigned] (ATLAS-1572) sqoop metadata ingest fails with "org.apache.atlas.exception.AtlasBaseException: expected type List or Set; found org.apache.atlas.typesystem.Referenceable"
Date Tue, 21 Feb 2017 11:34:44 GMT

     [ https://issues.apache.org/jira/browse/ATLAS-1572?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Vimal Sharma reassigned ATLAS-1572:
-----------------------------------

    Assignee: Vimal Sharma

> sqoop metadata ingest fails with "org.apache.atlas.exception.AtlasBaseException: expected
type List or Set; found org.apache.atlas.typesystem.Referenceable"
> ------------------------------------------------------------------------------------------------------------------------------------------------------------
>
>                 Key: ATLAS-1572
>                 URL: https://issues.apache.org/jira/browse/ATLAS-1572
>             Project: Atlas
>          Issue Type: Bug
>          Components: atlas-intg
>    Affects Versions: trunk, 0.8-incubating
>            Reporter: Ayub Khan
>            Assignee: Vimal Sharma
>            Priority: Blocker
>             Fix For: trunk, 0.8-incubating
>
>         Attachments: ATLAS-1572.patch
>
>
> Sqoop import succeeds but the metadata ingest fails with below error
> {noformat}
> 2017-02-20 11:20:43,179 WARN  - [NotificationHookConsumer thread-0:] ~ Max retries exceeded
for message [{Id='(type: sqoop_dbdatastore, id: <unassigned>)', traits=[], values={owner=hrt_qa,
storeUri=jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull,
dbStoreType=mysql, qualifiedName=mysql --url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji, name=mysql --url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji, description=, source=createSqoopTablejmnwwf8rji, storeUse=TABLE}},
{Id='(type: hive_db, id: <unassigned>)', traits=[], values={name=default, qualifiedName=default@cl1,
clusterName=cl1}}, {Id='(type: hive_table, id: <unassigned>)', traits=[], values={name=createsqooptablejmnwwf8rji,
qualifiedName=default.createsqooptablejmnwwf8rji@cl1, db={Id='(type: hive_db, id: <unassigned>)',
traits=[], values={name=default, qualifiedName=default@cl1, clusterName=cl1}}}}, {Id='(type:
sqoop_process, id: <unassigned>)', traits=[], values={outputs={Id='(type: hive_table,
id: <unassigned>)', traits=[], values={name=createsqooptablejmnwwf8rji, qualifiedName=default.createsqooptablejmnwwf8rji@cl1,
db={Id='(type: hive_db, id: <unassigned>)', traits=[], values={name=default, qualifiedName=default@cl1,
clusterName=cl1}}}}, commandlineOpts={db.clear.staging.table=false, hive.import=true, db.require.password=true,
codegen.output.delimiters.enclose=0, codegen.input.delimiters.field=0, customtool.options.jsonmap={},
hive.compute.stats.table=false, db.connect.string=jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull,
incremental.mode=None, db.table=createSqoopTablejmnwwf8rji, verbose=false, codegen.output.delimiters.enclose.required=false,
mapreduce.num.mappers=4, hdfs.append.dir=false, direct.import=false, hive.drop.delims=false,
hive.overwrite.table=false, hbase.bulk.load.enabled=false, hive.fail.table.exists=false, relaxed.isolation=false,
hdfs.delete-target.dir=false, split.limit=null, db.username=sqoop, codegen.input.delimiters.enclose.required=false,
codegen.output.dir=., import.direct.split.size=0, reset.onemapper=false, codegen.output.delimiters.record=10,
temporary.dirRoot=_sqoop, hcatalog.create.table=false, db.batch=false, import.fetch.size=-2147483648,
accumulo.max.latency=5000, hdfs.file.format=TextFile, codegen.output.delimiters.field=1, mainframe.input.dataset.type=p,
codegen.output.delimiters.escape=0, hcatalog.drop.and.create.table=false, import.max.inline.lob.size=16777216,
hbase.create.table=false, codegen.auto.compile.dir=true, codegen.compile.dir=/tmp/sqoop-hrt_qa/compile/f7c5181710812973a20a3e4776c5a6ef,
codegen.input.delimiters.enclose=0, export.new.update=UpdateOnly, enable.compression=false,
skip.dist.cache=false, accumulo.batch.size=10240000, hdfs.warehouse.dir=/user/hrt_qa, codegen.input.delimiters.record=0,
codegen.input.delimiters.escape=0, accumulo.create.table=false}, inputs={Id='(type: sqoop_dbdatastore,
id: <unassigned>)', traits=[], values={owner=hrt_qa, storeUri=jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull,
dbStoreType=mysql, qualifiedName=mysql --url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji, name=mysql --url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji, description=, source=createSqoopTablejmnwwf8rji, storeUse=TABLE}},
qualifiedName=sqoop import --connect jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji --hive-import --hive-database default --hive-table createsqooptablejmnwwf8rji
--hive-cluster cl1, name=sqoop import --connect jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji --hive-import --hive-database default --hive-table createsqooptablejmnwwf8rji
--hive-cluster cl1, startTime=2017-02-20T11:20:04.901Z, endTime=2017-02-20T11:20:40.055Z,
userName=hrt_qa, operation=import}}] (NotificationHookConsumer:324)
> org.apache.atlas.exception.AtlasBaseException: expected type List or Set; found org.apache.atlas.typesystem.Referenceable
>         at org.apache.atlas.repository.converters.AtlasArrayFormatConverter.fromV1ToV2(AtlasArrayFormatConverter.java:50)
>         at org.apache.atlas.repository.converters.AtlasArrayFormatConverter.fromV1ToV2(AtlasArrayFormatConverter.java:34)
>         at org.apache.atlas.repository.converters.AtlasStructFormatConverter.fromV1ToV2(AtlasStructFormatConverter.java:169)
>         at org.apache.atlas.repository.converters.AtlasEntityFormatConverter.fromV1ToV2(AtlasEntityFormatConverter.java:74)
>         at org.apache.atlas.repository.converters.AtlasInstanceConverter.fromV1toV2Entity(AtlasInstanceConverter.java:222)
>         at org.apache.atlas.repository.converters.AtlasInstanceConverter.toAtlasEntities(AtlasInstanceConverter.java:204)
>         at org.apache.atlas.notification.NotificationHookConsumer$HookConsumer.handleMessage(NotificationHookConsumer.java:256)
>         at org.apache.atlas.notification.NotificationHookConsumer$HookConsumer.run(NotificationHookConsumer.java:230)
>         at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
>         at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>         at java.lang.Thread.run(Thread.java:745)
> 2017-02-20 11:21:06,094 ERROR - [NotificationHookConsumer thread-0:] ~ [DROPPED_NOTIFICATION]
{"version":{"version":"1.0.0"},"message":{"entities":[{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888062","version":0,"typeName":"sqoop_dbdatastore","state":"ACTIVE"},"typeName":"sqoop_dbdatastore","values":{"name":"mysql
--url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji","source":"createSqoopTablejmnwwf8rji","storeUse":"TABLE","description":"","storeUri":"jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull","qualifiedName":"mysql
--url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji","owner":"hrt_qa","dbStoreType":"mysql"},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888061","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"qualifiedName":"default@cl1","clusterName":"cl1","name":"default"},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888060","version":0,"typeName":"hive_table","state":"ACTIVE"},"typeName":"hive_table","values":{"qualifiedName":"default.createsqooptablejmnwwf8rji@cl1","db":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888061","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"qualifiedName":"default@cl1","clusterName":"cl1","name":"default"},"traitNames":[],"traits":{},"systemAttributes":{}},"name":"createsqooptablejmnwwf8rji"},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888059","version":0,"typeName":"sqoop_process","state":"ACTIVE"},"typeName":"sqoop_process","values":{"name":"sqoop
import --connect jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji --hive-import --hive-database default --hive-table createsqooptablejmnwwf8rji
--hive-cluster cl1","startTime":"2017-02-20T11:20:04.901Z","outputs":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888060","version":0,"typeName":"hive_table","state":"ACTIVE"},"typeName":"hive_table","values":{"qualifiedName":"default.createsqooptablejmnwwf8rji@cl1","db":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888061","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"qualifiedName":"default@cl1","clusterName":"cl1","name":"default"},"traitNames":[],"traits":{},"systemAttributes":{}},"name":"createsqooptablejmnwwf8rji"},"traitNames":[],"traits":{},"systemAttributes":{}},"commandlineOpts":{"db.clear.staging.table":"false","hive.import":"true","db.require.password":"true","codegen.output.delimiters.enclose":"0","codegen.input.delimiters.field":"0","customtool.options.jsonmap":"{}","hive.compute.stats.table":"false","db.connect.string":"jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull","incremental.mode":"None","db.table":"createSqoopTablejmnwwf8rji","verbose":"false","codegen.output.delimiters.enclose.required":"false","mapreduce.num.mappers":"4","hdfs.append.dir":"false","direct.import":"false","hive.drop.delims":"false","hive.overwrite.table":"false","hbase.bulk.load.enabled":"false","hive.fail.table.exists":"false","relaxed.isolation":"false","hdfs.delete-target.dir":"false","split.limit":"null","db.username":"sqoop","codegen.input.delimiters.enclose.required":"false","codegen.output.dir":".","import.direct.split.size":"0","reset.onemapper":"false","codegen.output.delimiters.record":"10","temporary.dirRoot":"_sqoop","hcatalog.create.table":"false","db.batch":"false","import.fetch.size":"-2147483648","accumulo.max.latency":"5000","hdfs.file.format":"TextFile","codegen.output.delimiters.field":"1","mainframe.input.dataset.type":"p","codegen.output.delimiters.escape":"0","hcatalog.drop.and.create.table":"false","import.max.inline.lob.size":"16777216","hbase.create.table":"false","codegen.auto.compile.dir":"true","codegen.compile.dir":"/tmp/sqoop-hrt_qa/compile/f7c5181710812973a20a3e4776c5a6ef","codegen.input.delimiters.enclose":"0","export.new.update":"UpdateOnly","enable.compression":"false","skip.dist.cache":"false","accumulo.batch.size":"10240000","hdfs.warehouse.dir":"/user/hrt_qa","codegen.input.delimiters.record":"0","codegen.input.delimiters.escape":"0","accumulo.create.table":"false"},"endTime":"2017-02-20T11:20:40.055Z","inputs":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888062","version":0,"typeName":"sqoop_dbdatastore","state":"ACTIVE"},"typeName":"sqoop_dbdatastore","values":{"name":"mysql
--url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji","source":"createSqoopTablejmnwwf8rji","storeUse":"TABLE","description":"","storeUri":"jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull","qualifiedName":"mysql
--url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji","owner":"hrt_qa","dbStoreType":"mysql"},"traitNames":[],"traits":{},"systemAttributes":{}},"operation":"import","qualifiedName":"sqoop
import --connect jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji --hive-import --hive-database default --hive-table createsqooptablejmnwwf8rji
--hive-cluster cl1","userName":"hrt_qa"},"traitNames":[],"traits":{},"systemAttributes":{}}],"type":"ENTITY_CREATE","user":"hrt_qa"}}
(FAILED:339)
> {noformat}
> Steps to repro:
> Run the below sqoop import command
> {noformat}
>  /usr/hdp/current/sqoop-client/bin/sqoop  import  --connect jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests
--username sqoop --password sqoop --hive-import  --warehouse-dir /user/hrt_qa --table createSqoopTablejmnwwf8rji
> {noformat}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Mime
View raw message