hive-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Sahil Takiar (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (HIVE-15310) CTAS fails when target location contains multiple directories levels that don't exist
Date Tue, 29 Nov 2016 22:06:59 GMT

     [ https://issues.apache.org/jira/browse/HIVE-15310?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Sahil Takiar updated HIVE-15310:
--------------------------------
    Description: 
The second query below fails, if the {{/tmp/}} directory is empty:

{code}
create table test1 (id int) location '/tmp/test1/one/two/';
create table test2 location '/tmp/test2/one/two' as select * from test1;
{code}

The stacktrace is:

{code}
Error: org.apache.hive.service.cli.HiveSQLException: Error while processing statement: FAILED:
Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.MoveTask. Unable to move
source file:/var/folders/jb/350gyf853s91hk5xk06jyw2w0000gp/T/stakiar/f6c5c246-3209-4d76-aaf4-1ac91436f59a/hive_2016-11-29_14-02-33_047_219358498634208469-1/-mr-10002
to destination /tmp/test2/one/two
	at org.apache.hive.service.cli.operation.Operation.toSQLException(Operation.java:393)
	at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:250)
	at org.apache.hive.service.cli.operation.SQLOperation.access$800(SQLOperation.java:91)
	at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:340)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:422)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
	at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:353)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to move source file:/var/folders/jb/350gyf853s91hk5xk06jyw2w0000gp/T/stakiar/f6c5c246-3209-4d76-aaf4-1ac91436f59a/hive_2016-11-29_14-02-33_047_219358498634208469-1/-mr-10002
to destination /tmp/test2/one/two
	at org.apache.hadoop.hive.ql.exec.MoveTask.moveFile(MoveTask.java:104)
	at org.apache.hadoop.hive.ql.exec.MoveTask.execute(MoveTask.java:263)
	at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:199)
	at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:100)
	at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2166)
	at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1822)
	at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1510)
	at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1221)
	at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1216)
	at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:248)
	... 11 more
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to move source file:/var/folders/jb/350gyf853s91hk5xk06jyw2w0000gp/T/stakiar/f6c5c246-3209-4d76-aaf4-1ac91436f59a/hive_2016-11-29_14-02-33_047_219358498634208469-1/-mr-10002
to destination /tmp/test2/one/two
	at org.apache.hadoop.hive.ql.metadata.Hive.moveFile(Hive.java:3119)
	at org.apache.hadoop.hive.ql.exec.MoveTask.moveFileInDfs(MoveTask.java:120)
	at org.apache.hadoop.hive.ql.exec.MoveTask.moveFile(MoveTask.java:97)
	... 20 more
Caused by: java.io.FileNotFoundException: File /tmp/test2/one does not exist
	at org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:609)
	at org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:822)
	at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:599)
	at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:421)
	at org.apache.hadoop.hive.io.HdfsUtils$HadoopFileStatus.<init>(HdfsUtils.java:178)
	at org.apache.hadoop.hive.ql.metadata.Hive.moveFile(Hive.java:3029)
	... 22 more (state=08S01,code=1)
{code}

The second query works if the target location is simply {{/tmp/test2/}}

  was:
The following query fails, if the {{/tmp/}} directory is empty:

{code}
create table test1 (id int) location '/tmp/test1/one/two/';
create table test2 location '/tmp/test2/one/two' as select * from test1;
{code}

The stacktrace is:

{code}
Error: org.apache.hive.service.cli.HiveSQLException: Error while processing statement: FAILED:
Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.MoveTask. Unable to move
source file:/var/folders/jb/350gyf853s91hk5xk06jyw2w0000gp/T/stakiar/f6c5c246-3209-4d76-aaf4-1ac91436f59a/hive_2016-11-29_14-02-33_047_219358498634208469-1/-mr-10002
to destination /tmp/test2/one/two
	at org.apache.hive.service.cli.operation.Operation.toSQLException(Operation.java:393)
	at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:250)
	at org.apache.hive.service.cli.operation.SQLOperation.access$800(SQLOperation.java:91)
	at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:340)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:422)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
	at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:353)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to move source file:/var/folders/jb/350gyf853s91hk5xk06jyw2w0000gp/T/stakiar/f6c5c246-3209-4d76-aaf4-1ac91436f59a/hive_2016-11-29_14-02-33_047_219358498634208469-1/-mr-10002
to destination /tmp/test2/one/two
	at org.apache.hadoop.hive.ql.exec.MoveTask.moveFile(MoveTask.java:104)
	at org.apache.hadoop.hive.ql.exec.MoveTask.execute(MoveTask.java:263)
	at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:199)
	at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:100)
	at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2166)
	at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1822)
	at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1510)
	at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1221)
	at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1216)
	at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:248)
	... 11 more
Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to move source file:/var/folders/jb/350gyf853s91hk5xk06jyw2w0000gp/T/stakiar/f6c5c246-3209-4d76-aaf4-1ac91436f59a/hive_2016-11-29_14-02-33_047_219358498634208469-1/-mr-10002
to destination /tmp/test2/one/two
	at org.apache.hadoop.hive.ql.metadata.Hive.moveFile(Hive.java:3119)
	at org.apache.hadoop.hive.ql.exec.MoveTask.moveFileInDfs(MoveTask.java:120)
	at org.apache.hadoop.hive.ql.exec.MoveTask.moveFile(MoveTask.java:97)
	... 20 more
Caused by: java.io.FileNotFoundException: File /tmp/test2/one does not exist
	at org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:609)
	at org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:822)
	at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:599)
	at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:421)
	at org.apache.hadoop.hive.io.HdfsUtils$HadoopFileStatus.<init>(HdfsUtils.java:178)
	at org.apache.hadoop.hive.ql.metadata.Hive.moveFile(Hive.java:3029)
	... 22 more (state=08S01,code=1)
{code}


> CTAS fails when target location contains multiple directories levels that don't exist
> -------------------------------------------------------------------------------------
>
>                 Key: HIVE-15310
>                 URL: https://issues.apache.org/jira/browse/HIVE-15310
>             Project: Hive
>          Issue Type: Bug
>          Components: Hive
>            Reporter: Sahil Takiar
>
> The second query below fails, if the {{/tmp/}} directory is empty:
> {code}
> create table test1 (id int) location '/tmp/test1/one/two/';
> create table test2 location '/tmp/test2/one/two' as select * from test1;
> {code}
> The stacktrace is:
> {code}
> Error: org.apache.hive.service.cli.HiveSQLException: Error while processing statement:
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.MoveTask. Unable
to move source file:/var/folders/jb/350gyf853s91hk5xk06jyw2w0000gp/T/stakiar/f6c5c246-3209-4d76-aaf4-1ac91436f59a/hive_2016-11-29_14-02-33_047_219358498634208469-1/-mr-10002
to destination /tmp/test2/one/two
> 	at org.apache.hive.service.cli.operation.Operation.toSQLException(Operation.java:393)
> 	at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:250)
> 	at org.apache.hive.service.cli.operation.SQLOperation.access$800(SQLOperation.java:91)
> 	at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork$1.run(SQLOperation.java:340)
> 	at java.security.AccessController.doPrivileged(Native Method)
> 	at javax.security.auth.Subject.doAs(Subject.java:422)
> 	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
> 	at org.apache.hive.service.cli.operation.SQLOperation$BackgroundWork.run(SQLOperation.java:353)
> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> 	at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to move source file:/var/folders/jb/350gyf853s91hk5xk06jyw2w0000gp/T/stakiar/f6c5c246-3209-4d76-aaf4-1ac91436f59a/hive_2016-11-29_14-02-33_047_219358498634208469-1/-mr-10002
to destination /tmp/test2/one/two
> 	at org.apache.hadoop.hive.ql.exec.MoveTask.moveFile(MoveTask.java:104)
> 	at org.apache.hadoop.hive.ql.exec.MoveTask.execute(MoveTask.java:263)
> 	at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:199)
> 	at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:100)
> 	at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2166)
> 	at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1822)
> 	at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1510)
> 	at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1221)
> 	at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1216)
> 	at org.apache.hive.service.cli.operation.SQLOperation.runQuery(SQLOperation.java:248)
> 	... 11 more
> Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Unable to move source file:/var/folders/jb/350gyf853s91hk5xk06jyw2w0000gp/T/stakiar/f6c5c246-3209-4d76-aaf4-1ac91436f59a/hive_2016-11-29_14-02-33_047_219358498634208469-1/-mr-10002
to destination /tmp/test2/one/two
> 	at org.apache.hadoop.hive.ql.metadata.Hive.moveFile(Hive.java:3119)
> 	at org.apache.hadoop.hive.ql.exec.MoveTask.moveFileInDfs(MoveTask.java:120)
> 	at org.apache.hadoop.hive.ql.exec.MoveTask.moveFile(MoveTask.java:97)
> 	... 20 more
> Caused by: java.io.FileNotFoundException: File /tmp/test2/one does not exist
> 	at org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:609)
> 	at org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:822)
> 	at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:599)
> 	at org.apache.hadoop.fs.FilterFileSystem.getFileStatus(FilterFileSystem.java:421)
> 	at org.apache.hadoop.hive.io.HdfsUtils$HadoopFileStatus.<init>(HdfsUtils.java:178)
> 	at org.apache.hadoop.hive.ql.metadata.Hive.moveFile(Hive.java:3029)
> 	... 22 more (state=08S01,code=1)
> {code}
> The second query works if the target location is simply {{/tmp/test2/}}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Mime
View raw message