hudi-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "liujianhui (Jira)" <j...@apache.org>
Subject [jira] [Updated] (HUDI-518) compact error when hoodie.compact.inline is true
Date Fri, 10 Jan 2020 17:00:01 GMT

     [ https://issues.apache.org/jira/browse/HUDI-518?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

liujianhui updated HUDI-518:
----------------------------
    Description: 
# set the property [hoodie.compact.inline|http://hoodie.compact.inline/] as true
 # the duration of the write process is 1 second
 # the instant time of the compact is same to the commit instant time

 

{code}

java.lang.IllegalArgumentException: Following instants have timestamps >= compactionInstant
(20200110171526) Instants :[[20200110171526__deltacommit__COMPLETED]]
 at com.google.common.base.Preconditions.checkArgument(Preconditions.java:92)
 at org.apache.hudi.HoodieWriteClient.scheduleCompactionAtInstant(HoodieWriteClient.java:1043)
 at org.apache.hudi.HoodieWriteClient.scheduleCompaction(HoodieWriteClient.java:1018)
 at org.apache.hudi.HoodieWriteClient.forceCompact(HoodieWriteClient.java:1292)
 at org.apache.hudi.HoodieWriteClient.commit(HoodieWriteClient.java:510)
 at org.apache.hudi.HoodieWriteClient.commit(HoodieWriteClient.java:479)
 at org.apache.hudi.HoodieWriteClient.commit(HoodieWriteClient.java:470)
 at org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:152)
 at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:91)
 at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:45)
 at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)
 at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)
 at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:86)
 at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)
 at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)
 at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)
 at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
 at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)
 at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)
 at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80)
 at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80)
 at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676)
 at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676)
 at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:78)
 at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:125)
 at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:73)
 at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:676)
 at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:285)
 at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:271)

{code}

  was:
# set the property [hoodie.compact.inline|http://hoodie.compact.inline/] as true
 # the duration of the write process is 1 second
 # the instant time of the compact is same to the commit instant time

 

```

java.lang.IllegalArgumentException: Following instants have timestamps >= compactionInstant
(20200110171526) Instants :[[20200110171526__deltacommit__COMPLETED]]
 at com.google.common.base.Preconditions.checkArgument(Preconditions.java:92)
 at org.apache.hudi.HoodieWriteClient.scheduleCompactionAtInstant(HoodieWriteClient.java:1043)
 at org.apache.hudi.HoodieWriteClient.scheduleCompaction(HoodieWriteClient.java:1018)
 at org.apache.hudi.HoodieWriteClient.forceCompact(HoodieWriteClient.java:1292)
 at org.apache.hudi.HoodieWriteClient.commit(HoodieWriteClient.java:510)
 at org.apache.hudi.HoodieWriteClient.commit(HoodieWriteClient.java:479)
 at org.apache.hudi.HoodieWriteClient.commit(HoodieWriteClient.java:470)
 at org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:152)
 at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:91)
 at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:45)
 at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)
 at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)
 at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:86)
 at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)
 at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)
 at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)
 at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
 at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)
 at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)
 at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80)
 at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80)
 at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676)
 at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676)
 at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:78)
 at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:125)
 at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:73)
 at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:676)
 at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:285)
 at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:271)

```


> compact error when hoodie.compact.inline is true
> ------------------------------------------------
>
>                 Key: HUDI-518
>                 URL: https://issues.apache.org/jira/browse/HUDI-518
>             Project: Apache Hudi (incubating)
>          Issue Type: Bug
>          Components: Compaction
>            Reporter: liujianhui
>            Priority: Minor
>
> # set the property [hoodie.compact.inline|http://hoodie.compact.inline/] as true
>  # the duration of the write process is 1 second
>  # the instant time of the compact is same to the commit instant time
>  
> {code}
> java.lang.IllegalArgumentException: Following instants have timestamps >= compactionInstant
(20200110171526) Instants :[[20200110171526__deltacommit__COMPLETED]]
>  at com.google.common.base.Preconditions.checkArgument(Preconditions.java:92)
>  at org.apache.hudi.HoodieWriteClient.scheduleCompactionAtInstant(HoodieWriteClient.java:1043)
>  at org.apache.hudi.HoodieWriteClient.scheduleCompaction(HoodieWriteClient.java:1018)
>  at org.apache.hudi.HoodieWriteClient.forceCompact(HoodieWriteClient.java:1292)
>  at org.apache.hudi.HoodieWriteClient.commit(HoodieWriteClient.java:510)
>  at org.apache.hudi.HoodieWriteClient.commit(HoodieWriteClient.java:479)
>  at org.apache.hudi.HoodieWriteClient.commit(HoodieWriteClient.java:470)
>  at org.apache.hudi.HoodieSparkSqlWriter$.write(HoodieSparkSqlWriter.scala:152)
>  at org.apache.hudi.DefaultSource.createRelation(DefaultSource.scala:91)
>  at org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand.run(SaveIntoDataSourceCommand.scala:45)
>  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:70)
>  at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:68)
>  at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:86)
>  at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)
>  at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)
>  at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)
>  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
>  at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)
>  at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)
>  at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80)
>  at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80)
>  at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676)
>  at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:676)
>  at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:78)
>  at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:125)
>  at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:73)
>  at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:676)
>  at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:285)
>  at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:271)
> {code}



--
This message was sent by Atlassian Jira
(v8.3.4#803005)

Mime
View raw message