lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject [lucene-solr] branch jira/SOLR-13452_gradle_3 updated: SOLR-13452: Some more work towards a solid transitive dependency solution.
Date Mon, 10 Jun 2019 05:58:16 GMT
This is an automated email from the ASF dual-hosted git repository.

markrmiller pushed a commit to branch jira/SOLR-13452_gradle_3
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/jira/SOLR-13452_gradle_3 by this push:
     new 4780c1a  SOLR-13452: Some more work towards a solid transitive dependency solution.
4780c1a is described below

commit 4780c1ad56c9ce856f96d3ae7d2a54fac71ba68c
Author: markrmiller <markrmiller@apache.org>
AuthorDate: Mon Jun 10 00:57:59 2019 -0500

    SOLR-13452: Some more work towards a solid transitive dependency solution.
---
 build.gradle                                       | 24 ++++++-----
 .../org/apache/lucene/gradle/FindInSrc.groovy      | 12 ++----
 .../gradle/LuceneSolrForbiddenApisPlugin.groovy    |  1 +
 .../org/apache/lucene/gradle/MissingDeps.groovy    |  7 ----
 .../org/apache/lucene/gradle/UnusedDeps.groovy     | 47 +++++++++++-----------
 solr/contrib/dataimporthandler/build.gradle        |  3 ++
 solr/core/build.gradle                             | 29 ++++++++-----
 versions.lock                                      |  3 +-
 8 files changed, 67 insertions(+), 59 deletions(-)

diff --git a/build.gradle b/build.gradle
index 17e5fdc..9414fa4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -198,24 +198,30 @@ configure(allprojects) {
     description = "Lists dependencies that may be unused for a module."
 
     project.tasks.create("jdepsReport", org.apache.lucene.gradle.JdepsReport, jdepsReportDir,
true)
-    project.tasks.create("iunusedDeps", org.apache.lucene.gradle.UnusedDeps, jdepsReportDir)
-    project.iunusedDeps.dependsOn project.jdepsReport
-    project.iunusedDeps.doLast {
-      //delete(jdepsReportDir)
+    task unusedDeps(type: org.apache.lucene.gradle.UnusedDeps) {
+      inputDirectory jdepsReportDir
     }
-    dependsOn iunusedDeps
+    project.unusedDeps.dependsOn project.jdepsReport
+    project.unusedDeps.doLast {
+      delete(jdepsReportDir)
+    }
+    dependsOn unusedDeps
   }
+  
   task missingDependencies {
     group = 'Help'
     description = "Lists classes from this module with missing runtime dependencies (we ignore
scanning some root deps (ie hadoop) and some violations (ie annotations)."
     project.tasks.create("jdepsReportR", org.apache.lucene.gradle.JdepsReport, jdepsReportDir,
true)
     // we exclude hadoop and annotations
-    project.tasks.create("imissingDeps", org.apache.lucene.gradle.MissingDeps, jdepsReportDir,
['**/*hadoop*.dot'], ['org\\.apache\\.yetus\\.audience\\.InterfaceAudience', 'javax\\.annotation\\..*',
'org\\.apache\\.yetus\\.audience', 'com\\.google\\.errorprone\\.annotations\\..*'])
-    project.imissingDeps.dependsOn project.jdepsReportR
-    project.imissingDeps.doLast {
+    task missingDeps(type: org.apache.lucene.gradle.MissingDeps) {
+      inputDirectory jdepsReportDir
+    }
+    
+    project.missingDeps.dependsOn project.jdepsReportR
+    project.missingDeps.doLast {
       delete(jdepsReportDir)
     }
-    dependsOn imissingDeps
+    dependsOn missingDeps
   }
 }
 
diff --git a/buildSrc/src/main/groovy/org/apache/lucene/gradle/FindInSrc.groovy b/buildSrc/src/main/groovy/org/apache/lucene/gradle/FindInSrc.groovy
index 807193d..0137d89 100644
--- a/buildSrc/src/main/groovy/org/apache/lucene/gradle/FindInSrc.groovy
+++ b/buildSrc/src/main/groovy/org/apache/lucene/gradle/FindInSrc.groovy
@@ -46,9 +46,7 @@ class FindInSrc {
   protected static Pattern pattern = Pattern.compile("->\\s\\\"([^\\s]*?)\\s")
   protected static Pattern srcJar = Pattern.compile("(.*?)-sources.jar")
   protected static Pattern dotFilePattern = Pattern.compile("(.*?).jar.dot")
-  
-  
-  
+
   public FindInSrc() {
   }
   
@@ -71,7 +69,8 @@ class FindInSrc {
     Stream.of(listSrcFiles.toArray())
         .parallel()
         .forEach( { file ->
-          if (!file.name.endsWith('-sources.jar')) return
+          // if (!file.name.endsWith('-sources.jar')) return
+         // if (foundInsrc.get()) return // stop after we find first occurence(s)
           Matcher nameMatcher = srcJar.matcher(file.name)
           if (nameMatcher.matches()) {
             String artifactName = nameMatcher.group(1)
@@ -123,8 +122,3 @@ class FindInSrc {
   }
 }
 
-
-
-
-
-
diff --git a/buildSrc/src/main/groovy/org/apache/lucene/gradle/LuceneSolrForbiddenApisPlugin.groovy
b/buildSrc/src/main/groovy/org/apache/lucene/gradle/LuceneSolrForbiddenApisPlugin.groovy
index 1a9bab3..4aae479 100644
--- a/buildSrc/src/main/groovy/org/apache/lucene/gradle/LuceneSolrForbiddenApisPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/lucene/gradle/LuceneSolrForbiddenApisPlugin.groovy
@@ -32,6 +32,7 @@ class LuceneSolrForbiddenApisPlugin implements Plugin<Project> {
     }
 
     project.tasks.withType(CheckForbiddenApis) { task ->
+      task.failOnUnresolvableSignatures = false
       task.bundledSignatures = ['jdk-unsafe', 'jdk-deprecated', 'jdk-non-portable', 'jdk-reflection']
       task.signaturesURLs = [ getClass().getResource('/forbidden/base.txt') ]
       
diff --git a/buildSrc/src/main/groovy/org/apache/lucene/gradle/MissingDeps.groovy b/buildSrc/src/main/groovy/org/apache/lucene/gradle/MissingDeps.groovy
index 824773b..36039fc 100644
--- a/buildSrc/src/main/groovy/org/apache/lucene/gradle/MissingDeps.groovy
+++ b/buildSrc/src/main/groovy/org/apache/lucene/gradle/MissingDeps.groovy
@@ -61,13 +61,6 @@ class MissingDeps extends DefaultTask {
   
   protected configuration = "runtimeClasspath"
   
-  @Inject
-  public MissingDeps(File inputDirectory, List<String> depExcludes, List<String>
classExcludes) {
-    this.inputDirectory = inputDirectory
-    this.depExcludes = depExcludes
-    this.classExcludes = classExcludes
-  }
-  
   @TaskAction
   void execute() {
     
diff --git a/buildSrc/src/main/groovy/org/apache/lucene/gradle/UnusedDeps.groovy b/buildSrc/src/main/groovy/org/apache/lucene/gradle/UnusedDeps.groovy
index 441326a..45c1aba 100644
--- a/buildSrc/src/main/groovy/org/apache/lucene/gradle/UnusedDeps.groovy
+++ b/buildSrc/src/main/groovy/org/apache/lucene/gradle/UnusedDeps.groovy
@@ -20,6 +20,7 @@ import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.ResolvedDependency
 import javax.inject.Inject
 import org.gradle.api.DefaultTask
+import org.gradle.api.GradleException
 import org.gradle.api.Project
 import org.gradle.api.artifacts.Configuration
 import org.gradle.api.artifacts.Dependency
@@ -48,18 +49,15 @@ class UnusedDeps extends DefaultTask {
   protected static Pattern pattern = Pattern.compile("\\(([^\\s]*?\\.jar)\\)")
   
   protected configuration = "runtimeClasspath"
-  protected File distDir
-  protected File jdepsDir
   
   @InputDirectory
   File inputDirectory
   
-  @Inject
-  public UnusedDeps(File inputDirectory) {
-    this.inputDirectory = inputDirectory
-    
-    distDir = new File(inputDirectory, 'distDir')
-    jdepsDir = new File(inputDirectory, 'jdepsDir')
+  @Input
+  @Optional
+  List<String> jarExcludes
+  
+  public UnusedDeps() {
     
     if (project.hasProperty('useConfiguration')) {
       configuration = project.useConfiguration
@@ -68,6 +66,13 @@ class UnusedDeps extends DefaultTask {
     if (!project.configurations.hasProperty(configuration)) {
       return
     }
+  }
+  
+  @TaskAction
+  void execute() {
+    
+    // make sure ant task logging shows up by default
+    ant.lifecycleLogLevel = "INFO"
     
     Configuration config = project.configurations[this.configuration]
     
@@ -79,18 +84,12 @@ class UnusedDeps extends DefaultTask {
         buildProjects.add(dProject)
       }
     })
-  }
-  
-  @TaskAction
-  void execute() {
     
-    // make sure ant task logging shows up by default
-    ant.lifecycleLogLevel = "INFO"
+    File distDir = new File(inputDirectory, 'distDir')
+    File jdepsDir = new File(inputDirectory, 'jdepsDir')
     
     def topLvlProject = getTopLvlProject(project)
     
-    Configuration config = project.configurations[this.configuration]
-    
     Set<String> usedStaticallyJarNames = getStaticallyReferencedDeps(topLvlProject,
project, distDir, jdepsDir)
     
     Set<File> ourDeps = getAllDefinedDeps(project, config)
@@ -142,6 +141,8 @@ class UnusedDeps extends DefaultTask {
     
     println 'Direct deps that may be unused:'
     
+    boolean failTask = false;
+    
     unusedJarNames.forEach({
       if (!depsInDirectUse.contains(it) && ourImmediatelyDefinedDeps.contains(it))
{
         
@@ -149,6 +150,7 @@ class UnusedDeps extends DefaultTask {
         if (findInSrc(it)) {
           println ' - ' + it + ' *'
         } else {
+          failTask = true
           println ' - ' + it
         }
 
@@ -159,13 +161,13 @@ class UnusedDeps extends DefaultTask {
     println 'Deps brought in by other modules that may be unused in this module:'
     unusedJarNames.forEach({
       if (!depsInDirectUse.contains(it) && !ourImmediatelyDefinedDeps.contains(it))
{
-        if (findInSrc(it)) {
-          println ' - ' + it + ' *'
-        } else {
-          println ' - ' + it
-        }
+        println ' - ' + it
       }
     })
+    
+    if (failTask) {
+      throw new GradleException("Unused dependencies found! Remove them or add an exclusion
if they are actually necessary.")
+    }
   }
   
   static class NonProjectSpec implements Spec<Dependency> {
@@ -228,8 +230,7 @@ class UnusedDeps extends DefaultTask {
       depsInDirectUse.addAll(getDirectlyUsedJars(project, it))
     }
   }
-  
-  
+
   protected boolean findInSrc(String jarName) {
     AtomicBoolean foundInsrc = new AtomicBoolean(false)
     def files = project.configurations[configuration].resolvedConfiguration.getFiles()
diff --git a/solr/contrib/dataimporthandler/build.gradle b/solr/contrib/dataimporthandler/build.gradle
index a673846..2ad7155 100644
--- a/solr/contrib/dataimporthandler/build.gradle
+++ b/solr/contrib/dataimporthandler/build.gradle
@@ -42,6 +42,9 @@ dependencies {
   
   implementation ('org.slf4j:slf4j-api') { transitive = false }
   implementation ('commons-io:commons-io') { transitive = false }
+  implementation ('com.google.guava:guava') {
+    exclude group: '*', module: '*' // many annotation deps we don't need
+  }
   implementation ('org.apache.httpcomponents:httpclient') { transitive = false }
   implementation ('org.apache.zookeeper:zookeeper') { transitive = false }
   implementation ('io.dropwizard.metrics:metrics-core') { transitive = false }
diff --git a/solr/core/build.gradle b/solr/core/build.gradle
index c41b409..b0adda1 100644
--- a/solr/core/build.gradle
+++ b/solr/core/build.gradle
@@ -44,6 +44,9 @@ dependencies {
   implementation project(':lucene:lucene-join')
   implementation project(':solr:solr-solrj')
   
+  // used for log4j2 async support
+  runtimeOnly ('com.lmax:disruptor')
+  
   // kerb, kerby, and curator deps used for security and auth with zk/hdfs/kerberos
   runtimeOnly ('org.apache.curator:curator-recipes') {
     exclude group: '*', module: '*' // brings in a lot we don't use
@@ -53,16 +56,15 @@ dependencies {
   runtimeOnly ('org.apache.kerby:kerby-asn1')
   runtimeOnly ('org.apache.kerby:kerby-pkix')
   
-  // used by calicte.avatica
-  runtimeOnly ('com.google.protobuf:protobuf-java')
+  runtimeOnly ('com.google.protobuf:protobuf-java') // used by calicte.avatica
+  runtimeOnly ('org.slf4j:jcl-over-slf4j') // bridge java common logging to slf4j
   
   compileOnly ('javax.servlet:javax.servlet-api')
   
-  // TODO: this is not great
+  // TODO: this is not great, still have to work out logging dist strat
   implementation ('org.apache.logging.log4j:log4j-slf4j-impl')
 
   implementation ('org.slf4j:slf4j-api')
-  implementation ('org.slf4j:jcl-over-slf4j')
   implementation ('org.apache.zookeeper:zookeeper') {
     exclude group: '*', module: '*' // zk has many deps we don't need
   }
@@ -78,13 +80,13 @@ dependencies {
   }
   implementation ('io.dropwizard.metrics:metrics-jvm')
   implementation ('io.dropwizard.metrics:metrics-jmx')
-  api ('commons-codec:commons-codec') // api because of forbidden apis :(
+  implementation ('commons-codec:commons-codec')
   implementation ('commons-io:commons-io')
   implementation ('org.apache.commons:commons-exec')
   implementation ('commons-fileupload:commons-fileupload')
   implementation ('commons-cli:commons-cli')
   implementation ('org.apache.commons:commons-text')
-  api ('com.google.guava:guava') {  // api because of forbidden apis :(
+  implementation ('com.google.guava:guava') {
     exclude group: '*', module: '*' // many annotation deps we don't need
   }
   implementation ('org.locationtech.spatial4j:spatial4j')
@@ -92,13 +94,11 @@ dependencies {
   implementation ('org.ow2.asm:asm')
   
   implementation ('org.restlet.jee:org.restlet')
-  implementation ('org.restlet.jee:org.restlet.ext.servlet')
   implementation ('com.carrotsearch:hppc')
   
   implementation ('org.apache.logging.log4j:log4j-api')
   implementation ('org.apache.logging.log4j:log4j-core')
   implementation ('org.apache.logging.log4j:log4j-1.2-api')
-  implementation ('org.slf4j:jcl-over-slf4j')
   
   implementation ('com.fasterxml.jackson.core:jackson-core')
   implementation ('com.fasterxml.jackson.core:jackson-databind')
@@ -153,7 +153,7 @@ dependencies {
   implementation ('org.eclipse.jetty:jetty-servlet')
   implementation ('org.eclipse.jetty:jetty-server')
   
-  testRuntimeOnly ('com.lmax:disruptor')
+  testImplementation ('org.restlet.jee:org.restlet.ext.servlet')
   
   // kerby
   testRuntimeOnly ('org.apache.kerby:kerb-core')
@@ -211,6 +211,17 @@ forbiddenApisTest {
   exclude 'org/apache/hadoop/**'
 }
 
+missingDeps() {
+  depExcludes = ['**/*hadoop*.dot'] // we exclude examining hadoop jars because we don't
need lots of these deps
+  
+  classExcludes = [
+        'org\\.apache\\.yetus\\.audience\\.InterfaceAudience', // annotations jar
+        'javax\\.annotation\\..*', // annotations jar
+        'org\\.apache\\.yetus\\.audience', // annotations jar
+        'com\\.google\\.errorprone\\.annotations\\..*' // annotations jar
+      ]
+}
+
 task javacc(type: org.apache.lucene.gradle.SolrQPJavaCC) {
   inputFile file('src/java/org/apache/solr/parser/QueryParser.jj')
   target file('src/java/org/apache/solr/parser')
diff --git a/versions.lock b/versions.lock
index 85b338b..56f6ca8 100644
--- a/versions.lock
+++ b/versions.lock
@@ -210,8 +210,7 @@ org.ow2.asm:asm-analysis:6.2 (1 constraints: e309d6a5)
 org.ow2.asm:asm-commons:6.2 (1 constraints: aa04272c)
 org.ow2.asm:asm-tree:6.2 (2 constraints: 2d14228c)
 org.quartz-scheduler:quartz:2.2.0 (1 constraints: 2106eb4d)
-org.restlet.jee:org.restlet:2.3.0 (2 constraints: e3159ee6)
-org.restlet.jee:org.restlet.ext.servlet:2.3.0 (1 constraints: 0705fe35)
+org.restlet.jee:org.restlet:2.3.0 (1 constraints: 0705fe35)
 org.rrd4j:rrd4j:3.5 (1 constraints: ac04252c)
 org.simpleframework:simple-xml:2.7.1 (3 constraints: bd20563c)
 org.slf4j:jcl-over-slf4j:1.7.25 (4 constraints: f629cf87)


Mime
View raw message