gobblin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject incubator-gobblin git commit: [GOBBLIN-247] Fixed ArrayIndexOutOfBoundsException for avro-to-orc Validation Job
Date Tue, 12 Sep 2017 10:06:03 GMT
Repository: incubator-gobblin
Updated Branches:
  refs/heads/master 9402a9037 -> 1507b6055


[GOBBLIN-247] Fixed ArrayIndexOutOfBoundsException for avro-to-orc Validation Job

Closes #2100 from aditya1105/avro-to-orc


Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/1507b605
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/1507b605
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/1507b605

Branch: refs/heads/master
Commit: 1507b6055d3f058cc17b1accb01c897a21b9513c
Parents: 9402a90
Author: aditya1105 <adsharma@linkedin.com>
Authored: Tue Sep 12 03:05:57 2017 -0700
Committer: Abhishek Tiwari <abhishektiwari.btech@gmail.com>
Committed: Tue Sep 12 03:05:57 2017 -0700

----------------------------------------------------------------------
 .../conversion/hive/validation/ValidationJob.java      | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/1507b605/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/conversion/hive/validation/ValidationJob.java
----------------------------------------------------------------------
diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/conversion/hive/validation/ValidationJob.java
b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/conversion/hive/validation/ValidationJob.java
index 53805c8..1a61803 100644
--- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/conversion/hive/validation/ValidationJob.java
+++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/data/management/conversion/hive/validation/ValidationJob.java
@@ -391,7 +391,7 @@ public class ValidationJob extends AbstractJob {
               log.debug(String.format("Going to execute queries: %s for format: %s", dataValidationQueries,
format));
               List<Long> rowDataValidatedCount = ValidationJob.this.getValidationOutputFromHive(dataValidationQueries);
               // Validate and populate report
-              validateAndPopulateReport(hiveDataset.getTable().getCompleteName(), updateTime,
rowCounts, rowDataValidatedCount.get(0));
+              validateAndPopulateReport(hiveDataset.getTable().getCompleteName(), updateTime,
rowCounts, rowDataValidatedCount);
 
               return null;
             }
@@ -460,7 +460,7 @@ public class ValidationJob extends AbstractJob {
                   List<Long> rowDataValidatedCount = ValidationJob.this.getValidationOutputFromHive(dataValidationQueries);
 
                   // Validate and populate report
-                  validateAndPopulateReport(sourcePartition.getCompleteName(), updateTime,
rowCounts, rowDataValidatedCount.get(0));
+                  validateAndPopulateReport(sourcePartition.getCompleteName(), updateTime,
rowCounts, rowDataValidatedCount);
 
                   return null;
                 }
@@ -606,12 +606,17 @@ public class ValidationJob extends AbstractJob {
     return rowCounts;
   }
 
-  private void validateAndPopulateReport(String datasetIdentifier, long conversionInstance,
List<Long> rowCounts, Long rowDataValidatedCount) {
+  private void validateAndPopulateReport(String datasetIdentifier, long conversionInstance,
List<Long> rowCounts, List<Long> rowDataValidatedCount) {
     if (null == rowCounts || rowCounts.size() == 0) {
       this.warnConversions.put(String.format("Dataset: %s Instance: %s", datasetIdentifier,
conversionInstance), "No conversion details found");
       this.eventSubmitter.submit(EventConstants.VALIDATION_NOOP_EVENT, ImmutableMap.of("datasetUrn",
datasetIdentifier));
       return;
     }
+    if (null == rowDataValidatedCount || rowDataValidatedCount.size() == 0) {
+      this.warnConversions.put(String.format("Dataset: %s Instance: %s", datasetIdentifier,
conversionInstance), "No conversion details found");
+      this.eventSubmitter.submit(EventConstants.VALIDATION_NOOP_EVENT, ImmutableMap.of("datasetUrn",
datasetIdentifier));
+      return;
+    }
     long rowCountCached = -1;
     boolean isFirst = true;
     for (Long rowCount : rowCounts) {
@@ -642,7 +647,7 @@ public class ValidationJob extends AbstractJob {
     }
 
     // Data count validation
-    if (rowCountCached == rowDataValidatedCount) {
+    if (rowCountCached == rowDataValidatedCount.get(0)) {
       this.dataValidationSuccessful.put(String.format("Dataset: %s Instance: %s", datasetIdentifier,
conversionInstance),
           "Common rows matched expected value. Expected: " + rowCountCached + " Found: "
+ rowDataValidatedCount);
     } else {


Mime
View raw message