kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mpe...@apache.org
Subject [1/3] kudu git commit: java build: Add script to verify output of generated JARs
Date Wed, 29 Nov 2017 07:21:21 GMT
Repository: kudu
Updated Branches:
  refs/heads/master d74b98fbb -> 9120cdd1d


java build: Add script to verify output of generated JARs

This script reports any files included in packaging JARs that do not
match a set of patterns in the script. This allows us to define certain
dependencies that we will ship (shaded or otherwise) and, if this is
incorporated into the build process, will immediately alert us as things
change over time.

The script runs automatically as part of the regular CI build.

Change-Id: Ie16bca12dfe2d0267e559c91abe1178d906e3122
Reviewed-on: http://gerrit.cloudera.org:8080/8662
Tested-by: Kudu Jenkins
Reviewed-by: Dan Burkert <danburkert@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3f658629
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3f658629
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3f658629

Branch: refs/heads/master
Commit: 3f658629b49d9ccc04fa8bd7a2e4acc0542994d8
Parents: d74b98f
Author: Mike Percy <mpercy@apache.org>
Authored: Mon Nov 27 19:10:30 2017 -0800
Committer: Mike Percy <mpercy@apache.org>
Committed: Wed Nov 29 07:19:59 2017 +0000

----------------------------------------------------------------------
 build-support/jenkins/build-and-test.sh |   4 +
 build-support/verify_jars.pl            | 105 +++++++++++++++++++++++++++
 2 files changed, 109 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/3f658629/build-support/jenkins/build-and-test.sh
----------------------------------------------------------------------
diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh
index c2b0717..e642657 100755
--- a/build-support/jenkins/build-and-test.sh
+++ b/build-support/jenkins/build-and-test.sh
@@ -395,6 +395,10 @@ if [ "$BUILD_JAVA" == "1" ]; then
      fi
   fi
 
+  # Run a script to verify the contents of the JARs to ensure the shading and
+  # packaging is correct.
+  $SOURCE_ROOT/build-support/verify_jars.pl .
+
   set +x
   popd
 fi

http://git-wip-us.apache.org/repos/asf/kudu/blob/3f658629/build-support/verify_jars.pl
----------------------------------------------------------------------
diff --git a/build-support/verify_jars.pl b/build-support/verify_jars.pl
new file mode 100755
index 0000000..803db85
--- /dev/null
+++ b/build-support/verify_jars.pl
@@ -0,0 +1,105 @@
+#!/usr/bin/perl
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+################################################################################
+# This script verifies that the dependencies shipped in the Kudu jars do not
+# change over time without us noticing.
+################################################################################
+use strict;
+use warnings;
+
+# Prefix for shaded classes.
+my $pat_kudu_shaded_prefix = qr{^org/apache/kudu/shaded/};
+
+# Allowed filenames of non-Java files in JARs.
+my $pat_allow_non_java =
+    qr{(?:\.(?:txt|xml|properties|proto|MF)|
+          LICENSE|NOTICE|DEPENDENCIES)$}x;
+
+# Allowed filenames of shaded dependencies in JARs.
+my $pat_allow_kudu_shaded =
+    qr{^org/apache/kudu/shaded/
+        (?:org/jboss/netty|com/google/(?:common|protobuf|thirdparty/publicsuffix)|
+           com/sangupta/murmur)
+      }x;
+
+# Allowed paths of unshaded Kudu dependencies in JARs.
+# Currently, there is no restriction imposed for org.apache.kudu classes.
+my $pat_allow_kudu_unshaded = qr{^org/apache/kudu/.*};
+
+# Allowed paths of unshaded non-Kudu dependencies in JARs.
+my $pat_allow_nonkudu_unshaded = qr{^(?:com/databricks/spark/avro|
+                                        org/apache/parquet/|
+                                        org/apache/yetus/|
+                                        com/stumbleupon/async/)}x;
+
+if (scalar @ARGV != 1) {
+  print STDERR "Usage: $0 <dest_dir>\n";
+  exit 1;
+}
+
+my $dest_dir = $ARGV[0];
+chdir($dest_dir) or die "cannot chdir to destination directory $dest_dir: $!";
+print "Checking jars in directory: " . `pwd`;
+
+chomp(my @jars = `find . -type f -name \*.jar | grep -v build/ | grep -v "tests\.jar" |
+                         grep -v original | grep -v sources\.jar | grep -v javadoc\.jar`);
+
+my $num_errors = 0;
+
+foreach my $jar (@jars) {
+  print "> $jar\n";
+  chomp(my @files = `jar tf $jar`);
+  foreach my $file (@files) {
+    # In each case, ensure the files match the expected patterns.
+    if ($file =~ qr{/$}) {
+      # A directory. Skip.
+    } elsif ($file !~ qr{\.class$}) {
+      # Non-Java stuff.
+      if ($file !~ $pat_allow_non_java) {
+        $num_errors++;
+        print "NON_JAVA $file\n";
+      }
+    } elsif ($file =~ $pat_kudu_shaded_prefix) {
+      # Shaded Kudu classes.
+      if ($file !~ $pat_allow_kudu_shaded) {
+        $num_errors++;
+        print "KUDU_SHADED $file\n";
+      }
+    } elsif ($file =~ qr{^org/apache/kudu}) {
+      # Unshaded Kudu classes.
+      if ($file !~ $pat_allow_kudu_unshaded) {
+        $num_errors++;
+        print "KUDU_UNSHADED $file\n";
+      }
+    } else {
+      # Non-Kudu classes.
+      if ($file !~ $pat_allow_nonkudu_unshaded) {
+        $num_errors++;
+        print "NON_KUDU $file\n";
+      }
+    }
+  }
+}
+
+if ($num_errors != 0) {
+  print "Found $num_errors errors.\n";
+  exit 1;
+}
+print "OK.\n";
+exit 0;


Mime
View raw message