hudi-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] [incubator-hudi] n3nash commented on a change in pull request #1318: [HUDI-571] CLI - add option to print additional commit metadata
Date Wed, 12 Feb 2020 18:33:12 GMT
n3nash commented on a change in pull request #1318: [HUDI-571] CLI - add option to print additional
commit metadata
URL: https://github.com/apache/incubator-hudi/pull/1318#discussion_r378435736
 
 

 ##########
 File path: hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java
 ##########
 @@ -100,8 +100,58 @@ private String printCommits(HoodieDefaultTimeline timeline,
     return HoodiePrintHelper.print(header, fieldNameToConverterMap, sortByField, descending,
limit, headerOnly, rows);
   }
 
+  private String printCommitsWithMetadata(HoodieDefaultTimeline timeline,
+                              final Integer limit, final String sortByField,
+                              final boolean descending,
+                              final boolean headerOnly) throws IOException {
+    final List<Comparable[]> rows = new ArrayList<>();
+
+    final List<HoodieInstant> commits = timeline.getCommitsTimeline().filterCompletedInstants()
+            .getInstants().collect(Collectors.toList());
+    // timeline can be read from multiple files. So sort is needed instead of reversing the
collection
+    Collections.sort(commits, HoodieInstant.COMPARATOR.reversed());
+
+    for (int i = 0; i < commits.size(); i++) {
+      final HoodieInstant commit = commits.get(i);
+      final HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(
+              timeline.getInstantDetails(commit).get(),
+              HoodieCommitMetadata.class);
+
+      for (Map.Entry<String, List<HoodieWriteStat>> partitionWriteStat :
+              commitMetadata.getPartitionToWriteStats().entrySet()) {
+        for (HoodieWriteStat hoodieWriteStat : partitionWriteStat.getValue()) {
+          rows.add(new Comparable[]{ commit.getAction(), commit.getTimestamp(), hoodieWriteStat.getPartitionPath(),
+                  hoodieWriteStat.getFileId(), hoodieWriteStat.getPrevCommit(), hoodieWriteStat.getNumWrites(),
+                  hoodieWriteStat.getNumInserts(), hoodieWriteStat.getNumDeletes(),
+                  hoodieWriteStat.getNumUpdateWrites(), hoodieWriteStat.getTotalWriteErrors(),
+                  hoodieWriteStat.getTotalLogBlocks(), hoodieWriteStat.getTotalCorruptLogBlock(),
+                  hoodieWriteStat.getTotalRollbackBlocks(), hoodieWriteStat.getTotalLogRecords(),
+                  hoodieWriteStat.getTotalUpdatedRecordsCompacted(), hoodieWriteStat.getTotalWriteBytes()
+          });
+        }
+      }
+    }
+
+    final Map<String, Function<Object, String>> fieldNameToConverterMap = new
HashMap<>();
+    fieldNameToConverterMap.put("Total Bytes Written", entry -> {
+      return NumericUtils.humanReadableByteCount((Double.valueOf(entry.toString())));
+    });
+
+    TableHeader header = new TableHeader().addTableHeaderField("action").addTableHeaderField("instant")
+            .addTableHeaderField("partition").addTableHeaderField("file_id").addTableHeaderField("prev_instant")
+            .addTableHeaderField("num_writes").addTableHeaderField("num_inserts").addTableHeaderField("num_deletes")
 
 Review comment:
   Do we add names in snake_case in other header commands ?

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message