lucene-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markrmil...@apache.org
Subject [lucene-solr] branch jira/SOLR-13452_gradle updated: SOLR-13452: Add some of the download and extract tasks for lucene/benchmark.
Date Sun, 12 May 2019 03:19:41 GMT
This is an automated email from the ASF dual-hosted git repository.

markrmiller pushed a commit to branch jira/SOLR-13452_gradle
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/jira/SOLR-13452_gradle by this push:
     new a068954  SOLR-13452: Add some of the download and extract tasks for lucene/benchmark.
a068954 is described below

commit a06895402dc35fca8f019f3eea5e4376d0d7e790
Author: markrmiller <markrmiller@apache.org>
AuthorDate: Sat May 11 22:19:37 2019 -0500

    SOLR-13452: Add some of the download and extract tasks for lucene/benchmark.
---
 .../groovy/org/apache/lucene/gradle/Get.groovy     |  8 ++-
 lucene/benchmark/build.gradle                      | 62 ++++++++++++++++++++++
 lucene/core/build.gradle                           |  1 -
 3 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/buildSrc/src/main/groovy/org/apache/lucene/gradle/Get.groovy b/buildSrc/src/main/groovy/org/apache/lucene/gradle/Get.groovy
index 0b8f887..d5d5098 100644
--- a/buildSrc/src/main/groovy/org/apache/lucene/gradle/Get.groovy
+++ b/buildSrc/src/main/groovy/org/apache/lucene/gradle/Get.groovy
@@ -38,12 +38,18 @@ class Get extends DefaultTask {
   @Input
   String sourceUrl
   
+  @Input 
+  boolean overwrite = false;
+  
   @OutputFile
   File target
   
   @TaskAction
   void download() {
-    ant.get(src: sourceUrl, dest: target)
+    if (target.isDirectory()) throw IllegalArgumentException("Must specify target output
file.");
+    if (overwrite || !target.exists()) {
+      ant.get(src: sourceUrl, dest: target)
+    }
   }
 }
 
diff --git a/lucene/benchmark/build.gradle b/lucene/benchmark/build.gradle
index 7c4f9b5..2b612c6 100644
--- a/lucene/benchmark/build.gradle
+++ b/lucene/benchmark/build.gradle
@@ -1,5 +1,7 @@
 apply plugin: 'java-library'
 
+def workingDir = "${projectDir}/work"
+
 dependencies {
   
   implementationTran project(':lucene:lucene-core')
@@ -26,4 +28,64 @@ task copyConfResources(type: Copy) {
 }
 processTestResources.dependsOn copyConfResources
 
+// Enwiki
+
+task getEnwiki(type: org.apache.lucene.gradle.Get) {
+  sourceUrl = "http://home.apache.org/~dsmiley/data/enwiki-20070527-pages-articles.xml.bz2"
+  target = new File("${projectDir}/temp/enwiki-20070527-pages-articles.xml.bz2")
+}
+
+task expandEnwiki(){
+  doLast {
+    ant.bunzip2(src: "${projectDir}/temp/enwiki-20070527-pages-articles.xml.bz2", dest:"${projectDir}/temp/")
+  }
+}
+
+expandEnwiki.dependsOn getEnwiki
+
+// GeoNames
+
+task getGeoNames(type: org.apache.lucene.gradle.Get) {
+  sourceUrl = "http://home.apache.org/~dsmiley/data/geonames_20130921_randomOrder_allCountries.txt.bz2"
+  target = new File("${projectDir}/temp/allCountries.txt.bz2")
+}
+
+task expandGeoNames(){
+  doLast {
+    new File("${workingDir}/geonames").mkdirs()
+    ant.bunzip2(src: "${projectDir}/temp/allCountries.txt.bz2", dest:"${workingDir}/geonames")
+  }
+}
+
+expandGeoNames.dependsOn getGeoNames
+
+// Reuters
+
+task getReuters(type: org.apache.lucene.gradle.Get) {
+  sourceUrl = "http://www.daviddlewis.com/resources/testcollections/reuters21578/reuters21578.tar.gz"
+  target = new File("${projectDir}/temp/reuters21578.tar.gz")
+}
+
+task expandReuters(){
+  doLast {
+    new File("${workingDir}/reuters").mkdirs()
+    ant.gunzip(src: "${projectDir}/temp/reuters21578.tar.gz", dest:"${projectDir}/temp/")
+    ant.untar(src: "${projectDir}/temp/reuters21578.tar", dest: "${workingDir}/reuters")
+    def tree = fileTree("${workingDir}/reuters")
+    tree.include '**/*.txt'
+    tree.each { it.delete() }
+  }
+}
+
+expandReuters.dependsOn getReuters
+
+task extractReuters(type: JavaExec) {
+  classpath = sourceSets.main.runtimeClasspath
+
+  main = 'org.apache.lucene.benchmark.utils.ExtractReuters'
+
+  args "${workingDir}/reuters"
+  args "${workingDir}/reuters-out"
+}
+
 
diff --git a/lucene/core/build.gradle b/lucene/core/build.gradle
index 23cd266..a1636ef 100644
--- a/lucene/core/build.gradle
+++ b/lucene/core/build.gradle
@@ -86,7 +86,6 @@ task unzipMoman(){
   }
 }
 
-// TODO: only download if we have not already
 task downloadMoman(type: org.apache.lucene.gradle.Get) {
   sourceUrl = "${momanUrl}"
   target = new File("${buildDir}", 'moman.zip')


Mime
View raw message