atlas-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Vimal Sharma (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (ATLAS-1572) sqoop metadata ingest fails with "org.apache.atlas.exception.AtlasBaseException: expected type List or Set; found org.apache.atlas.typesystem.Referenceable"
Date Mon, 27 Feb 2017 17:49:45 GMT

     [ https://issues.apache.org/jira/browse/ATLAS-1572?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Vimal Sharma updated ATLAS-1572:
--------------------------------
    Attachment: ATLAS-1572.1.patch

Updating the patch to handle the issue on the server side. When an Object is supplied to V1
to V2 Array converter, it handles the case where the given Object is not a Collection. 

> sqoop metadata ingest fails with "org.apache.atlas.exception.AtlasBaseException: expected
type List or Set; found org.apache.atlas.typesystem.Referenceable"
> ------------------------------------------------------------------------------------------------------------------------------------------------------------
>
>                 Key: ATLAS-1572
>                 URL: https://issues.apache.org/jira/browse/ATLAS-1572
>             Project: Atlas
>          Issue Type: Bug
>          Components: atlas-intg
>    Affects Versions: trunk, 0.8-incubating
>            Reporter: Ayub Khan
>            Assignee: Vimal Sharma
>            Priority: Blocker
>             Fix For: trunk, 0.8-incubating
>
>         Attachments: ATLAS-1572.1.patch, ATLAS-1572.patch
>
>
> Sqoop import succeeds but the metadata ingest fails with below error
> {noformat}
> 2017-02-20 11:20:43,179 WARN  - [NotificationHookConsumer thread-0:] ~ Max retries exceeded
for message [{Id='(type: sqoop_dbdatastore, id: <unassigned>)', traits=[], values={owner=hrt_qa,
storeUri=jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull,
dbStoreType=mysql, qualifiedName=mysql --url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji, name=mysql --url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji, description=, source=createSqoopTablejmnwwf8rji, storeUse=TABLE}},
{Id='(type: hive_db, id: <unassigned>)', traits=[], values={name=default, qualifiedName=default@cl1,
clusterName=cl1}}, {Id='(type: hive_table, id: <unassigned>)', traits=[], values={name=createsqooptablejmnwwf8rji,
qualifiedName=default.createsqooptablejmnwwf8rji@cl1, db={Id='(type: hive_db, id: <unassigned>)',
traits=[], values={name=default, qualifiedName=default@cl1, clusterName=cl1}}}}, {Id='(type:
sqoop_process, id: <unassigned>)', traits=[], values={outputs={Id='(type: hive_table,
id: <unassigned>)', traits=[], values={name=createsqooptablejmnwwf8rji, qualifiedName=default.createsqooptablejmnwwf8rji@cl1,
db={Id='(type: hive_db, id: <unassigned>)', traits=[], values={name=default, qualifiedName=default@cl1,
clusterName=cl1}}}}, commandlineOpts={db.clear.staging.table=false, hive.import=true, db.require.password=true,
codegen.output.delimiters.enclose=0, codegen.input.delimiters.field=0, customtool.options.jsonmap={},
hive.compute.stats.table=false, db.connect.string=jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull,
incremental.mode=None, db.table=createSqoopTablejmnwwf8rji, verbose=false, codegen.output.delimiters.enclose.required=false,
mapreduce.num.mappers=4, hdfs.append.dir=false, direct.import=false, hive.drop.delims=false,
hive.overwrite.table=false, hbase.bulk.load.enabled=false, hive.fail.table.exists=false, relaxed.isolation=false,
hdfs.delete-target.dir=false, split.limit=null, db.username=sqoop, codegen.input.delimiters.enclose.required=false,
codegen.output.dir=., import.direct.split.size=0, reset.onemapper=false, codegen.output.delimiters.record=10,
temporary.dirRoot=_sqoop, hcatalog.create.table=false, db.batch=false, import.fetch.size=-2147483648,
accumulo.max.latency=5000, hdfs.file.format=TextFile, codegen.output.delimiters.field=1, mainframe.input.dataset.type=p,
codegen.output.delimiters.escape=0, hcatalog.drop.and.create.table=false, import.max.inline.lob.size=16777216,
hbase.create.table=false, codegen.auto.compile.dir=true, codegen.compile.dir=/tmp/sqoop-hrt_qa/compile/f7c5181710812973a20a3e4776c5a6ef,
codegen.input.delimiters.enclose=0, export.new.update=UpdateOnly, enable.compression=false,
skip.dist.cache=false, accumulo.batch.size=10240000, hdfs.warehouse.dir=/user/hrt_qa, codegen.input.delimiters.record=0,
codegen.input.delimiters.escape=0, accumulo.create.table=false}, inputs={Id='(type: sqoop_dbdatastore,
id: <unassigned>)', traits=[], values={owner=hrt_qa, storeUri=jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull,
dbStoreType=mysql, qualifiedName=mysql --url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji, name=mysql --url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji, description=, source=createSqoopTablejmnwwf8rji, storeUse=TABLE}},
qualifiedName=sqoop import --connect jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji --hive-import --hive-database default --hive-table createsqooptablejmnwwf8rji
--hive-cluster cl1, name=sqoop import --connect jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior=convertToNull
--table createSqoopTablejmnwwf8rji --hive-import --hive-database default --hive-table createsqooptablejmnwwf8rji
--hive-cluster cl1, startTime=2017-02-20T11:20:04.901Z, endTime=2017-02-20T11:20:40.055Z,
userName=hrt_qa, operation=import}}] (NotificationHookConsumer:324)
> org.apache.atlas.exception.AtlasBaseException: expected type List or Set; found org.apache.atlas.typesystem.Referenceable
>         at org.apache.atlas.repository.converters.AtlasArrayFormatConverter.fromV1ToV2(AtlasArrayFormatConverter.java:50)
>         at org.apache.atlas.repository.converters.AtlasArrayFormatConverter.fromV1ToV2(AtlasArrayFormatConverter.java:34)
>         at org.apache.atlas.repository.converters.AtlasStructFormatConverter.fromV1ToV2(AtlasStructFormatConverter.java:169)
>         at org.apache.atlas.repository.converters.AtlasEntityFormatConverter.fromV1ToV2(AtlasEntityFormatConverter.java:74)
>         at org.apache.atlas.repository.converters.AtlasInstanceConverter.fromV1toV2Entity(AtlasInstanceConverter.java:222)
>         at org.apache.atlas.repository.converters.AtlasInstanceConverter.toAtlasEntities(AtlasInstanceConverter.java:204)
>         at org.apache.atlas.notification.NotificationHookConsumer$HookConsumer.handleMessage(NotificationHookConsumer.java:256)
>         at org.apache.atlas.notification.NotificationHookConsumer$HookConsumer.run(NotificationHookConsumer.java:230)
>         at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
>         at java.util.concurrent.FutureTask.run(FutureTask.java:266)
>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>         at java.lang.Thread.run(Thread.java:745)
> 2017-02-20 11:21:06,094 ERROR - [NotificationHookConsumer thread-0:] ~ [DROPPED_NOTIFICATION]
{"version":{"version":"1.0.0"},"message":{"entities":[{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888062","version":0,"typeName":"sqoop_dbdatastore","state":"ACTIVE"},"typeName":"sqoop_dbdatastore","values":{"name":"mysql
--url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji","source":"createSqoopTablejmnwwf8rji","storeUse":"TABLE","description":"","storeUri":"jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull","qualifiedName":"mysql
--url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji","owner":"hrt_qa","dbStoreType":"mysql"},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888061","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"qualifiedName":"default@cl1","clusterName":"cl1","name":"default"},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888060","version":0,"typeName":"hive_table","state":"ACTIVE"},"typeName":"hive_table","values":{"qualifiedName":"default.createsqooptablejmnwwf8rji@cl1","db":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888061","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"qualifiedName":"default@cl1","clusterName":"cl1","name":"default"},"traitNames":[],"traits":{},"systemAttributes":{}},"name":"createsqooptablejmnwwf8rji"},"traitNames":[],"traits":{},"systemAttributes":{}},{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888059","version":0,"typeName":"sqoop_process","state":"ACTIVE"},"typeName":"sqoop_process","values":{"name":"sqoop
import --connect jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji --hive-import --hive-database default --hive-table createsqooptablejmnwwf8rji
--hive-cluster cl1","startTime":"2017-02-20T11:20:04.901Z","outputs":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888060","version":0,"typeName":"hive_table","state":"ACTIVE"},"typeName":"hive_table","values":{"qualifiedName":"default.createsqooptablejmnwwf8rji@cl1","db":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888061","version":0,"typeName":"hive_db","state":"ACTIVE"},"typeName":"hive_db","values":{"qualifiedName":"default@cl1","clusterName":"cl1","name":"default"},"traitNames":[],"traits":{},"systemAttributes":{}},"name":"createsqooptablejmnwwf8rji"},"traitNames":[],"traits":{},"systemAttributes":{}},"commandlineOpts":{"db.clear.staging.table":"false","hive.import":"true","db.require.password":"true","codegen.output.delimiters.enclose":"0","codegen.input.delimiters.field":"0","customtool.options.jsonmap":"{}","hive.compute.stats.table":"false","db.connect.string":"jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull","incremental.mode":"None","db.table":"createSqoopTablejmnwwf8rji","verbose":"false","codegen.output.delimiters.enclose.required":"false","mapreduce.num.mappers":"4","hdfs.append.dir":"false","direct.import":"false","hive.drop.delims":"false","hive.overwrite.table":"false","hbase.bulk.load.enabled":"false","hive.fail.table.exists":"false","relaxed.isolation":"false","hdfs.delete-target.dir":"false","split.limit":"null","db.username":"sqoop","codegen.input.delimiters.enclose.required":"false","codegen.output.dir":".","import.direct.split.size":"0","reset.onemapper":"false","codegen.output.delimiters.record":"10","temporary.dirRoot":"_sqoop","hcatalog.create.table":"false","db.batch":"false","import.fetch.size":"-2147483648","accumulo.max.latency":"5000","hdfs.file.format":"TextFile","codegen.output.delimiters.field":"1","mainframe.input.dataset.type":"p","codegen.output.delimiters.escape":"0","hcatalog.drop.and.create.table":"false","import.max.inline.lob.size":"16777216","hbase.create.table":"false","codegen.auto.compile.dir":"true","codegen.compile.dir":"/tmp/sqoop-hrt_qa/compile/f7c5181710812973a20a3e4776c5a6ef","codegen.input.delimiters.enclose":"0","export.new.update":"UpdateOnly","enable.compression":"false","skip.dist.cache":"false","accumulo.batch.size":"10240000","hdfs.warehouse.dir":"/user/hrt_qa","codegen.input.delimiters.record":"0","codegen.input.delimiters.escape":"0","accumulo.create.table":"false"},"endTime":"2017-02-20T11:20:40.055Z","inputs":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Reference","id":{"jsonClass":"org.apache.atlas.typesystem.json.InstanceSerialization$_Id","id":"-16210205233888062","version":0,"typeName":"sqoop_dbdatastore","state":"ACTIVE"},"typeName":"sqoop_dbdatastore","values":{"name":"mysql
--url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji","source":"createSqoopTablejmnwwf8rji","storeUse":"TABLE","description":"","storeUri":"jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull","qualifiedName":"mysql
--url jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji","owner":"hrt_qa","dbStoreType":"mysql"},"traitNames":[],"traits":{},"systemAttributes":{}},"operation":"import","qualifiedName":"sqoop
import --connect jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests?zeroDateTimeBehavior\u003dconvertToNull
--table createSqoopTablejmnwwf8rji --hive-import --hive-database default --hive-table createsqooptablejmnwwf8rji
--hive-cluster cl1","userName":"hrt_qa"},"traitNames":[],"traits":{},"systemAttributes":{}}],"type":"ENTITY_CREATE","user":"hrt_qa"}}
(FAILED:339)
> {noformat}
> Steps to repro:
> Run the below sqoop import command
> {noformat}
>  /usr/hdp/current/sqoop-client/bin/sqoop  import  --connect jdbc:mysql://ctr-e129-1487033772569-11862-01-000002.hwx.site/sqoopTests
--username sqoop --password sqoop --hive-import  --warehouse-dir /user/hrt_qa --table createSqoopTablejmnwwf8rji
> {noformat}



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)

Mime
View raw message