hudi-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (Jira)" <j...@apache.org>
Subject [jira] [Updated] (HUDI-573) Rolling stats written twice onto commit metadata
Date Thu, 20 Feb 2020 17:53:00 GMT

     [ https://issues.apache.org/jira/browse/HUDI-573?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

ASF GitHub Bot updated HUDI-573:
--------------------------------
    Labels: pull-request-available  (was: )

> Rolling stats written twice onto commit metadata
> ------------------------------------------------
>
>                 Key: HUDI-573
>                 URL: https://issues.apache.org/jira/browse/HUDI-573
>             Project: Apache Hudi (incubating)
>          Issue Type: Bug
>          Components: Writer Core
>            Reporter: Vinoth Chandar
>            Assignee: Nishith Agarwal
>            Priority: Major
>              Labels: pull-request-available
>             Fix For: 0.6.0
>
>
> To reproduce, simply write a commit and observe the file locally 
> {code:java}
>   "extraMetadataMap" : {
>     "ROLLING_STAT" : "{\n  \"partitionToRollingStats\" : {\n    \"date-0\" : {\n    
 \"b2404182-a20b-48f3-9386-1fd46e233aa6-1\" : {\n        \"fileId\" : \"b2404182-a20b-48f3-9386-1fd46e233aa6-1\",\n
       \"inserts\" : 86046,\n        \"upserts\" : 0,\n        \"deletes\" : 0,\n        \"totalInputWriteBytesToDisk\"
: 0,\n        \"totalInputWriteBytesOnDisk\" : 49691893\n      },\n      \"b2404182-a20b-48f3-9386-1fd46e233aa6-0\"
: {\n        \"fileId\" : \"b2404182-a20b-48f3-9386-1fd46e233aa6-0\",\n        \"inserts\"
: 214350,\n        \"upserts\" : 0,\n        \"deletes\" : 0,\n        \"totalInputWriteBytesToDisk\"
: 0,\n        \"totalInputWriteBytesOnDisk\" : 123138517\n      }\n    },\n    \"date-2\"
: {\n      \"f8924f32-09e6-4049-8830-9fb623d6c1e9-2\" : {\n        \"fileId\" : \"f8924f32-09e6-4049-8830-9fb623d6c1e9-2\",\n
       \"inserts\" : 186262,\n        \"upserts\" : 0,\n        \"deletes\" : 0,\n       
\"totalInputWriteBytesToDisk\" : 0,\n        \"totalInputWriteBytesOnDisk\" : 107012887\n
     },\n      \"f8924f32-09e6-4049-8830-9fb623d6c1e9-1\" : {\n        \"fileId\" : \"f8924f32-09e6-4049-8830-9fb623d6c1e9-1\",\n
       \"inserts\" : 214350,\n        \"upserts\" : 0,\n        \"deletes\" : 0,\n       
\"totalInputWriteBytesToDisk\" : 0,\n        \"totalInputWriteBytesOnDisk\" : 123081042\n
     }\n    },\n    \"date-1\" : {\n      \"f8924f32-09e6-4049-8830-9fb623d6c1e9-0\" : {\n
       \"fileId\" : \"f8924f32-09e6-4049-8830-9fb623d6c1e9-0\",\n        \"inserts\" : 63043,\n
       \"upserts\" : 0,\n        \"deletes\" : 0,\n        \"totalInputWriteBytesToDisk\"
: 0,\n        \"totalInputWriteBytesOnDisk\" : 36528903\n      },\n      \"b2404182-a20b-48f3-9386-1fd46e233aa6-3\"
: {\n        \"fileId\" : \"b2404182-a20b-48f3-9386-1fd46e233aa6-3\",\n        \"inserts\"
: 21632,\n        \"upserts\" : 0,\n        \"deletes\" : 0,\n        \"totalInputWriteBytesToDisk\"
: 0,\n        \"totalInputWriteBytesOnDisk\" : 12820469\n      },\n      \"b2404182-a20b-48f3-9386-1fd46e233aa6-2\"
: {\n        \"fileId\" : \"b2404182-a20b-48f3-9386-1fd46e233aa6-2\",\n        \"inserts\"
: 214319,\n        \"upserts\" : 0,\n        \"deletes\" : 0,\n        \"totalInputWriteBytesToDisk\"
: 0,\n        \"totalInputWriteBytesOnDisk\" : 123119553\n      }\n    }\n  },\n  \"actionType\"
: \"commit\"\n}",
>     "schema" : "{\"type\":\"record\",\"name\":\"hoodie_benchmark_record\",\"namespace\":\"hoodie.hoodie_benchmark\",\"fields\":[{\"name\":\"key\",\"type\":[\"string\",\"null\"]},{\"name\":\"partition\",\"type\":[\"string\",\"null\"]},{\"name\":\"ts\",\"type\":[\"long\",\"null\"]},{\"name\":\"textField\",\"type\":[\"string\",\"null\"]},{\"name\":\"decimalField\",\"type\":[\"float\",\"null\"]},{\"name\":\"longField\",\"type\":[\"long\",\"null\"]},{\"name\":\"arrayField\",\"type\":[{\"type\":\"array\",\"items\":[\"int\",\"null\"]},\"null\"]},{\"name\":\"mapField\",\"type\":[{\"type\":\"map\",\"values\":[\"int\",\"null\"]},\"null\"]}]}"
>   },
>   "extraMetadata" : {
>     "ROLLING_STAT" : "{\n  \"partitionToRollingStats\" : {\n    \"date-0\" : {\n    
 \"b2404182-a20b-48f3-9386-1fd46e233aa6-1\" : {\n        \"fileId\" : \"b2404182-a20b-48f3-9386-1fd46e233aa6-1\",\n
       \"inserts\" : 86046,\n        \"upserts\" : 0,\n        \"deletes\" : 0,\n        \"totalInputWriteBytesToDisk\"
: 0,\n        \"totalInputWriteBytesOnDisk\" : 49691893\n      },\n      \"b2404182-a20b-48f3-9386-1fd46e233aa6-0\"
: {\n        \"fileId\" : \"b2404182-a20b-48f3-9386-1fd46e233aa6-0\",\n        \"inserts\"
: 214350,\n        \"upserts\" : 0,\n        \"deletes\" : 0,\n        \"totalInputWriteBytesToDisk\"
: 0,\n        \"totalInputWriteBytesOnDisk\" : 123138517\n      }\n    },\n    \"date-2\"
: {\n      \"f8924f32-09e6-4049-8830-9fb623d6c1e9-2\" : {\n        \"fileId\" : \"f8924f32-09e6-4049-8830-9fb623d6c1e9-2\",\n
       \"inserts\" : 186262,\n        \"upserts\" : 0,\n        \"deletes\" : 0,\n       
\"totalInputWriteBytesToDisk\" : 0,\n        \"totalInputWriteBytesOnDisk\" : 107012887\n
     },\n      \"f8924f32-09e6-4049-8830-9fb623d6c1e9-1\" : {\n        \"fileId\" : \"f8924f32-09e6-4049-8830-9fb623d6c1e9-1\",\n
       \"inserts\" : 214350,\n        \"upserts\" : 0,\n        \"deletes\" : 0,\n       
\"totalInputWriteBytesToDisk\" : 0,\n        \"totalInputWriteBytesOnDisk\" : 123081042\n
     }\n    },\n    \"date-1\" : {\n      \"f8924f32-09e6-4049-8830-9fb623d6c1e9-0\" : {\n
       \"fileId\" : \"f8924f32-09e6-4049-8830-9fb623d6c1e9-0\",\n        \"inserts\" : 63043,\n
       \"upserts\" : 0,\n        \"deletes\" : 0,\n        \"totalInputWriteBytesToDisk\"
: 0,\n        \"totalInputWriteBytesOnDisk\" : 36528903\n      },\n      \"b2404182-a20b-48f3-9386-1fd46e233aa6-3\"
: {\n        \"fileId\" : \"b2404182-a20b-48f3-9386-1fd46e233aa6-3\",\n        \"inserts\"
: 21632,\n        \"upserts\" : 0,\n        \"deletes\" : 0,\n        \"totalInputWriteBytesToDisk\"
: 0,\n        \"totalInputWriteBytesOnDisk\" : 12820469\n      },\n      \"b2404182-a20b-48f3-9386-1fd46e233aa6-2\"
: {\n        \"fileId\" : \"b2404182-a20b-48f3-9386-1fd46e233aa6-2\",\n        \"inserts\"
: 214319,\n        \"upserts\" : 0,\n        \"deletes\" : 0,\n        \"totalInputWriteBytesToDisk\"
: 0,\n        \"totalInputWriteBytesOnDisk\" : 123119553\n      }\n    }\n  },\n  \"actionType\"
: \"commit\"\n}",
>     "schema" : "{\"type\":\"record\",\"name\":\"hoodie_benchmark_record\",\"namespace\":\"hoodie.hoodie_benchmark\",\"fields\":[{\"name\":\"key\",\"type\":[\"string\",\"null\"]},{\"name\":\"partition\",\"type\":[\"string\",\"null\"]},{\"name\":\"ts\",\"type\":[\"long\",\"null\"]},{\"name\":\"textField\",\"type\":[\"string\",\"null\"]},{\"name\":\"decimalField\",\"type\":[\"float\",\"null\"]},{\"name\":\"longField\",\"type\":[\"long\",\"null\"]},{\"name\":\"arrayField\",\"type\":[{\"type\":\"array\",\"items\":[\"int\",\"null\"]},\"null\"]},{\"name\":\"mapField\",\"type\":[{\"type\":\"map\",\"values\":[\"int\",\"null\"]},\"null\"]}]}"
>   },
>   "fileIdAndRelativePaths" : {
>  {code}



--
This message was sent by Atlassian Jira
(v8.3.4#803005)

Mime
View raw message