Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C1148200C56 for ; Thu, 30 Mar 2017 19:24:36 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id BF837160BA6; Thu, 30 Mar 2017 17:24:36 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 7F5BE160BA3 for ; Thu, 30 Mar 2017 19:24:34 +0200 (CEST) Received: (qmail 10600 invoked by uid 500); 30 Mar 2017 17:24:32 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 9791 invoked by uid 99); 30 Mar 2017 17:24:30 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 30 Mar 2017 17:24:30 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3AA4BE1794; Thu, 30 Mar 2017 17:24:30 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: cpoerschke@apache.org To: commits@lucene.apache.org Date: Thu, 30 Mar 2017 17:24:38 -0000 Message-Id: <036cc038c7bf4aa28f1ad57350058d5d@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [09/73] [abbrv] lucene-solr:jira/solr-6203: SOLR-9221: Remove Solr contribs: map-reduce, morphlines-core and morphlines-cell archived-at: Thu, 30 Mar 2017 17:24:36 -0000 http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53e5f34f/solr/contrib/morphlines-core/src/test-files/test-documents/testXML2.xml ---------------------------------------------------------------------- diff --git a/solr/contrib/morphlines-core/src/test-files/test-documents/testXML2.xml b/solr/contrib/morphlines-core/src/test-files/test-documents/testXML2.xml deleted file mode 100644 index 6611ee1..0000000 --- a/solr/contrib/morphlines-core/src/test-files/test-documents/testXML2.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - 123 - Hello World - Solr rocks - http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53e5f34f/solr/contrib/morphlines-core/src/test-files/test-morphlines/loadSolrBasic.conf ---------------------------------------------------------------------- diff --git a/solr/contrib/morphlines-core/src/test-files/test-morphlines/loadSolrBasic.conf b/solr/contrib/morphlines-core/src/test-files/test-morphlines/loadSolrBasic.conf deleted file mode 100644 index 1c02a9a..0000000 --- a/solr/contrib/morphlines-core/src/test-files/test-morphlines/loadSolrBasic.conf +++ /dev/null @@ -1,68 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# morphline.conf example file -# this is a comment -// this is yet another comment - -# for details see https://github.com/typesafehub/config#optional-system-or-env-variable-overrides -SOLR_COLLECTION : "collection1" -SOLR_COLLECTION : ${?ENV_SOLR_COLLECTION} - -ZK_HOST : "127.0.0.1:2181/solr" -ZK_HOST : ${?ENV_ZK_HOST} - -SOLR_HOME_DIR : "example/solr/collection1" -SOLR_HOME_DIR : ${?ENV_SOLR_HOME_DIR} - -SOLR_LOCATOR : { - collection : ${SOLR_COLLECTION} - zkHost : ${ZK_HOST} - solrHomeDir : ${SOLR_HOME_DIR} - # batchSize : 1000 -} -SOLR_LOCATOR : ${?ENV_SOLR_LOCATOR} - -morphlines : [ - { - id : morphline1 - - # using globs (foo.bar.* or foo.bar.**) will not work in Java9 due to classpath scanning limitations - # so we enumarate every command (builder) we know this config uses below. (see SOLR-8876) - importCommands : ["org.kitesdk.morphline.stdlib.LogDebugBuilder", - "org.apache.solr.morphlines.solr.SanitizeUnknownSolrFieldsBuilder", - "org.apache.solr.morphlines.solr.LoadSolrBuilder"] - - commands : [ - { - sanitizeUnknownSolrFields { - solrLocator : ${SOLR_LOCATOR} - } - } - - { - loadSolr { - solrLocator : ${SOLR_LOCATOR} - boosts : { - id : 1.0 - } - } - } - - { logDebug { format : "output record: {}", args : ["@{}"] } } - ] - } -] http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53e5f34f/solr/contrib/morphlines-core/src/test-files/test-morphlines/solrCellDocumentTypes.conf ---------------------------------------------------------------------- diff --git a/solr/contrib/morphlines-core/src/test-files/test-morphlines/solrCellDocumentTypes.conf b/solr/contrib/morphlines-core/src/test-files/test-morphlines/solrCellDocumentTypes.conf deleted file mode 100644 index 4d38256..0000000 --- a/solr/contrib/morphlines-core/src/test-files/test-morphlines/solrCellDocumentTypes.conf +++ /dev/null @@ -1,304 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Application configuration file in HOCON format (Human-Optimized Config Object Notation). -# HOCON syntax is defined at http://github.com/typesafehub/config/blob/master/HOCON.md -# and also used by Akka (http://www.akka.io) and Play (http://www.playframework.org/). -# For more examples see http://doc.akka.io/docs/akka/2.1.2/general/configuration.html - -# morphline.conf example file -# this is a comment -// this is yet another comment - -morphlines : [ - { - id : morphline1 - - # using globs (foo.bar.* or foo.bar.**) will not work in Java9 due to classpath scanning limitations - # so we enumarate every command (builder) we know this config uses below. (see SOLR-8876) - importCommands : ["org.kitesdk.morphline.stdlib.LogDebugBuilder", - "org.kitesdk.morphline.stdlib.SetValuesBuilder", - "org.kitesdk.morphline.stdlib.CallParentPipeBuilder", - "org.kitesdk.morphline.stdlib.GenerateUUIDBuilder", - "org.kitesdk.morphline.stdlib.JavaBuilder", - "org.kitesdk.morphline.stdlib.TryRulesBuilder", - "org.kitesdk.morphline.stdlib.SeparateAttachmentsBuilder", - "org.kitesdk.morphline.stdio.ReadCSVBuilder", - "org.kitesdk.morphline.avro.ReadAvroContainerBuilder", - "org.kitesdk.morphline.avro.ExtractAvroPathsBuilder", - "org.kitesdk.morphline.avro.ExtractAvroTreeBuilder", - "org.kitesdk.morphline.tika.DetectMimeTypeBuilder", - "org.kitesdk.morphline.tika.decompress.DecompressBuilder", - "org.kitesdk.morphline.tika.decompress.UnpackBuilder", - "org.kitesdk.morphline.twitter.ReadJsonTestTweetsBuilder", - "org.apache.solr.morphlines.cell.SolrCellBuilder", - "org.apache.solr.morphlines.solr.SanitizeUnknownSolrFieldsBuilder", - "org.apache.solr.morphlines.solr.GenerateSolrSequenceKeyBuilder", - "org.apache.solr.morphlines.solr.LoadSolrBuilder"] - - commands : [ - { separateAttachments {} } - - # java command that doesn't do anything except for test compilation - { - java { - imports : "import java.util.*;" - code: """ - List tags = record.get("javaWithImports"); - return child.process(record); - """ - } - } - - # java command that doesn't do anything except for test compilation - { - java { - code: """ - List tags = record.get("javaWithoutImports"); - return child.process(record); - """ - } - } - - { - # used for auto-detection if MIME type isn't explicitly supplied - detectMimeType { - includeDefaultMimeTypes : true - mimeTypesFiles : ["RESOURCES_DIR/custom-mimetypes.xml"] - } - } - - { - tryRules { - throwExceptionIfAllRulesFailed : true - rules : [ - # next top-level rule: - { - commands : [ - { logDebug { format : "hello unpack" } } - { unpack {} } - { generateUUID {} } - { callParentPipe {} } - ] - } - - { - commands : [ - { logDebug { format : "hello decompress" } } - { decompress {} } - { callParentPipe {} } - ] - } - - { - commands : [ - { - readCSV { - supportedMimeTypes : [text/csv] - charset : UTF-8 - ignoreFirstLine : false - columns : [ user_screen_name, text ] - } - } - - { - generateUUID { - field : id - preserveExisting : false - } - } - - { - sanitizeUnknownSolrFields { - solrLocator : ${SOLR_LOCATOR} - } - } - ] - } - - { - commands : [ - { - readAvroContainer { - supportedMimeTypes : [avro/binary] - # readerSchemaString : "" # optional, avro json schema blurb for getSchema() - # readerSchemaFile : /path/to/syslog.avsc - } - } - - { extractAvroTree {} } - - { - setValues { - id : "@{/id}" - user_screen_name : "@{/user_screen_name}" - text : "@{/text}" - } - } - - { - sanitizeUnknownSolrFields { - solrLocator : ${SOLR_LOCATOR} - } - } - ] - } - - { - commands : [ - { - readJsonTestTweets { - supportedMimeTypes : ["mytwittertest/json+delimited+length"] - } - } - - { - sanitizeUnknownSolrFields { - solrLocator : ${SOLR_LOCATOR} - } - } - ] - } - - # next top-level rule: - { - commands : [ - { logDebug { format : "hello solrcell" } } - { - # wrap SolrCell around an HTML Tika parser - solrCell { - solrLocator : ${SOLR_LOCATOR} - # captureAttr : true # default is false - capture : [ - - # twitter feed schema - user_friends_count - user_location - user_description - user_statuses_count - user_followers_count - user_name - user_screen_name - created_at - text - retweet_count - retweeted - in_reply_to_user_id - source - in_reply_to_status_id - media_url_https - expanded_url - - # file metadata - file_download_url - file_upload_url - file_scheme - file_host - file_port - file_path - file_name - file_length - file_last_modified - file_owner - file_group - file_permissions_user - file_permissions_group - file_permissions_other - file_permissions_stickybit - ] - - fmap : { content : text, content-type : content_type } # rename "content" field to "text" fields - dateFormats : [ "yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd"] # various java.text.SimpleDateFormat - # xpath : "/xhtml:html/xhtml:body/xhtml:div/descendant:node()" - uprefix : "ignored_" - lowernames : true - # solrContentHandlerFactory : org.apache.solr.tika.TrimSolrContentHandlerFactory - - # Tika parsers to be registered. If multiple parsers support the same MIME type, - # the parser is chosen that is closest to the bottom in this list: - parsers : [ - { parser : org.apache.tika.parser.asm.ClassParser } - # { parser : org.apache.tika.parser.AutoDetectParser } - # { parser : org.gagravarr.tika.OggParser, additionalSupportedMimeTypes : [audio/ogg] } - { parser : org.gagravarr.tika.FlacParser } - { parser : org.apache.tika.parser.audio.AudioParser } - { parser : org.apache.tika.parser.audio.MidiParser } - { parser : org.apache.tika.parser.crypto.Pkcs7Parser } - { parser : org.apache.tika.parser.dwg.DWGParser } - { parser : org.apache.tika.parser.epub.EpubParser } - { parser : org.apache.tika.parser.executable.ExecutableParser } - { parser : org.apache.tika.parser.feed.FeedParser } - { parser : org.apache.tika.parser.font.AdobeFontMetricParser } - { parser : org.apache.tika.parser.font.TrueTypeParser } - { parser : org.apache.tika.parser.xml.XMLParser } - { parser : org.apache.tika.parser.html.HtmlParser } - { parser : org.apache.tika.parser.image.ImageParser } - { parser : org.apache.tika.parser.image.PSDParser } - { parser : org.apache.tika.parser.image.TiffParser } - { parser : org.apache.tika.parser.iptc.IptcAnpaParser } - { parser : org.apache.tika.parser.iwork.IWorkPackageParser } - { parser : org.apache.tika.parser.jpeg.JpegParser } - { parser : org.apache.tika.parser.mail.RFC822Parser } - { parser : org.apache.tika.parser.mbox.MboxParser, additionalSupportedMimeTypes : [message/x-emlx] } - { parser : org.apache.tika.parser.microsoft.OfficeParser } - { parser : org.apache.tika.parser.microsoft.TNEFParser } - { parser : org.apache.tika.parser.microsoft.ooxml.OOXMLParser } - { parser : org.apache.tika.parser.mp3.Mp3Parser } - { parser : org.apache.tika.parser.mp4.MP4Parser } - { parser : org.apache.tika.parser.hdf.HDFParser } - { parser : org.apache.tika.parser.netcdf.NetCDFParser } - { parser : org.apache.tika.parser.odf.OpenDocumentParser } - { parser : org.apache.tika.parser.pdf.PDFParser } - { parser : org.apache.tika.parser.pkg.CompressorParser } - { parser : org.apache.tika.parser.pkg.PackageParser } - { parser : org.apache.tika.parser.rtf.RTFParser } - { parser : org.apache.tika.parser.txt.TXTParser } - { parser : org.apache.tika.parser.video.FLVParser } - { parser : org.apache.tika.parser.xml.DcXMLParser } - { parser : org.apache.tika.parser.xml.FictionBookParser } - { parser : org.apache.tika.parser.chm.ChmParser } - #{ parser : org.apache.tika.parser.AutoDetectParser } - ] - } - } - - { generateUUID { field : ignored_base_id } } - - { - generateSolrSequenceKey { - baseIdField: ignored_base_id - solrLocator : ${SOLR_LOCATOR} - } - } - - ] - } - ] - } - } - - { - loadSolr { - solrLocator : ${SOLR_LOCATOR} - } - } - - { - logDebug { - format : "My output record: {}" - args : ["@{}"] - } - } - - ] - } -] http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53e5f34f/solr/contrib/morphlines-core/src/test-files/test-morphlines/solrCellJPGCompressed.conf ---------------------------------------------------------------------- diff --git a/solr/contrib/morphlines-core/src/test-files/test-morphlines/solrCellJPGCompressed.conf b/solr/contrib/morphlines-core/src/test-files/test-morphlines/solrCellJPGCompressed.conf deleted file mode 100644 index 85cb2a7..0000000 --- a/solr/contrib/morphlines-core/src/test-files/test-morphlines/solrCellJPGCompressed.conf +++ /dev/null @@ -1,150 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Application configuration file in HOCON format (Human-Optimized Config Object Notation). -# HOCON syntax is defined at http://github.com/typesafehub/config/blob/master/HOCON.md -# and also used by Akka (http://www.akka.io) and Play (http://www.playframework.org/). -# For more examples see http://doc.akka.io/docs/akka/2.1.2/general/configuration.html - -# morphline.conf example file -# this is a comment -// this is yet another comment - -morphlines : [ - { - id : morphline1 - - # using globs (foo.bar.* or foo.bar.**) will not work in Java9 due to classpath scanning limitations - # so we enumarate every command (builder) we know this config uses below. (see SOLR-8876) - importCommands : ["org.kitesdk.morphline.stdlib.LogDebugBuilder", - "org.kitesdk.morphline.stdlib.CallParentPipeBuilder", - "org.kitesdk.morphline.stdlib.GenerateUUIDBuilder", - "org.kitesdk.morphline.stdlib.JavaBuilder", - "org.kitesdk.morphline.stdlib.TryRulesBuilder", - "org.kitesdk.morphline.stdlib.SeparateAttachmentsBuilder", - "org.kitesdk.morphline.tika.DetectMimeTypeBuilder", - "org.kitesdk.morphline.tika.decompress.DecompressBuilder", - "org.kitesdk.morphline.tika.decompress.UnpackBuilder", - "org.apache.solr.morphlines.cell.SolrCellBuilder", - "org.apache.solr.morphlines.solr.GenerateSolrSequenceKeyBuilder", - "org.apache.solr.morphlines.solr.LoadSolrBuilder"] - - - commands : [ - { separateAttachments {} } - - # java command that doesn't do anything except for test compilation - { - java { - imports : "import java.util.*;" - code: """ - List tags = record.get("javaWithImports"); - return child.process(record); - """ - } - } - - # java command that doesn't do anything except for test compilation - { - java { - code: """ - List tags = record.get("javaWithoutImports"); - return child.process(record); - """ - } - } - - { - # auto-detect MIME type if it isn't explicitly supplied - detectMimeType { - includeDefaultMimeTypes : true - } - } - - { - tryRules { - throwExceptionIfAllRulesFailed : true - rules : [ - # next top-level rule: - { - commands : [ - { logDebug { format : "hello unpack" } } - { unpack {} } - { callParentPipe {} } - ] - } - - { - commands : [ - { logDebug { format : "hello decompress" } } - { decompress {} } - { callParentPipe {} } - ] - } - - # next top-level rule: - { - commands : [ - { logDebug { format : "hello solrcell" } } - { - # wrap SolrCell around a JPG Tika parser - solrCell { - solrLocator : ${SOLR_LOCATOR} - captureAttr : true # default is false - capture : [content, a, h1, h2] # extract some fields - fmap : { exif_image_height : text, a : anchor, h1 : heading1 } # rename some fields - dateFormats : [ "yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd"] # various java.text.SimpleDateFormat - xpath : "/xhtml:html/xhtml:body/xhtml:div/descendant:node()" - uprefix : "ignored_" - lowernames : true - solrContentHandlerFactory : org.apache.solr.morphlines.cell.TrimSolrContentHandlerFactory - parsers : [ # nested Tika parsers - { parser : org.apache.tika.parser.jpeg.JpegParser } - ] - } - } - - { logDebug { format : "solrcell output: {}", args : ["@{}"] } } - ] - } - ] - } - } - - { generateUUID { field : ignored_base_id } } - - { - generateSolrSequenceKey { - baseIdField: ignored_base_id - solrLocator : ${SOLR_LOCATOR} - } - } - - { - loadSolr { - solrLocator : ${SOLR_LOCATOR} - } - } - - { - logDebug { - format : "My output record: {}" - args : ["@{}"] - } - } - - ] - } -] http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53e5f34f/solr/contrib/morphlines-core/src/test-files/test-morphlines/solrCellXML.conf ---------------------------------------------------------------------- diff --git a/solr/contrib/morphlines-core/src/test-files/test-morphlines/solrCellXML.conf b/solr/contrib/morphlines-core/src/test-files/test-morphlines/solrCellXML.conf deleted file mode 100644 index 9e840a9..0000000 --- a/solr/contrib/morphlines-core/src/test-files/test-morphlines/solrCellXML.conf +++ /dev/null @@ -1,78 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Application configuration file in HOCON format (Human-Optimized Config Object Notation). -# HOCON syntax is defined at http://github.com/typesafehub/config/blob/master/HOCON.md -# and also used by Akka (http://www.akka.io) and Play (http://www.playframework.org/). -# For more examples see http://doc.akka.io/docs/akka/2.1.2/general/configuration.html - -# morphline.conf example file -# this is a comment -// this is yet another comment - -morphlines : [ - { - id : morphline1 - - # using globs (foo.bar.* or foo.bar.**) will not work in Java9 due to classpath scanning limitations - # so we enumarate every command (builder) we know this config uses below. (see SOLR-8876) - importCommands : ["org.kitesdk.morphline.stdlib.LogDebugBuilder", - "org.kitesdk.morphline.stdlib.AddValuesBuilder", - "org.apache.solr.morphlines.cell.SolrCellBuilder", - "org.apache.solr.morphlines.solr.GenerateSolrSequenceKeyBuilder", - "org.apache.solr.morphlines.solr.SanitizeUnknownSolrFieldsBuilder", - "org.apache.solr.morphlines.solr.LoadSolrBuilder"] - - - commands : [ - { - addValues { _attachment_mimetype : application/xml } - # alternatively, consider using detectMimeTypes command - } - - { - # wrap SolrCell around a JPG Tika parser - solrCell { - solrLocator : ${SOLR_LOCATOR} - parsers : [ # nested Tika parsers - { parser : org.apache.tika.parser.xml.XMLParser } - ] - } - } - - { - generateSolrSequenceKey { - baseIdField: base_id - solrLocator : ${SOLR_LOCATOR} - } - } - - { - sanitizeUnknownSolrFields { - solrLocator : ${SOLR_LOCATOR} - } - } - - { logDebug { format : "solrcell output: {}", args : ["@{}"] } } - - { - loadSolr { - solrLocator : ${SOLR_LOCATOR} - } - } - - ] - } -] http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53e5f34f/solr/contrib/morphlines-core/src/test-files/test-morphlines/tokenizeText.conf ---------------------------------------------------------------------- diff --git a/solr/contrib/morphlines-core/src/test-files/test-morphlines/tokenizeText.conf b/solr/contrib/morphlines-core/src/test-files/test-morphlines/tokenizeText.conf deleted file mode 100644 index d9354c4..0000000 --- a/solr/contrib/morphlines-core/src/test-files/test-morphlines/tokenizeText.conf +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -morphlines : [ - { - id : morphline1 - - # using globs (foo.bar.* or foo.bar.**) will not work in Java9 due to classpath scanning limitations - # so we enumarate every command (builder) we know this config uses below. (see SOLR-8876) - importCommands : ["org.kitesdk.morphline.stdlib.LogDebugBuilder", - "org.apache.solr.morphlines.solr.TokenizeTextBuilder"] - - commands : [ - { - tokenizeText { - inputField : message - outputField : tokens - solrFieldType : text_en - solrLocator : ${SOLR_LOCATOR} - } - } - - { logDebug { format : "output record {}", args : ["@{}"] } } - ] - } -] http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53e5f34f/solr/contrib/morphlines-core/src/test-files/test-morphlines/tutorialReadAvroContainer.conf ---------------------------------------------------------------------- diff --git a/solr/contrib/morphlines-core/src/test-files/test-morphlines/tutorialReadAvroContainer.conf b/solr/contrib/morphlines-core/src/test-files/test-morphlines/tutorialReadAvroContainer.conf deleted file mode 100644 index eee4ba5..0000000 --- a/solr/contrib/morphlines-core/src/test-files/test-morphlines/tutorialReadAvroContainer.conf +++ /dev/null @@ -1,145 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Application configuration file in HOCON format (Human-Optimized Config Object Notation). -# HOCON syntax is defined at http://github.com/typesafehub/config/blob/master/HOCON.md -# and also used by Akka (http://www.akka.io) and Play (http://www.playframework.org/). -# For more examples see http://doc.akka.io/docs/akka/2.1.2/general/configuration.html - -# morphline.conf example file -# this is a comment - -# Specify server locations in a SOLR_LOCATOR variable; used later in variable substitutions: -SOLR_LOCATOR : { - # Name of solr collection - collection : collection1 - - # ZooKeeper ensemble - zkHost : "127.0.0.1:2181/solr" - - # The maximum number of documents to send to Solr per network batch (throughput knob) - # batchSize : 1000 -} - -# Specify an array of one or more morphlines, each of which defines an ETL -# transformation chain. A morphline consists of one or more (potentially -# nested) commands. A morphline is a way to consume records (e.g. Flume events, -# HDFS files or blocks), turn them into a stream of records, and pipe the stream -# of records through a set of easily configurable transformations on its way to -# Solr. -morphlines : [ - { - # Name used to identify a morphline. E.g. used if there are multiple morphlines in a - # morphline config file - id : morphline1 - - # using globs (foo.bar.* or foo.bar.**) will not work in Java9 due to classpath scanning limitations - # so we enumarate every command (builder) we know this config uses below. (see SOLR-8876) - importCommands : ["org.kitesdk.morphline.stdlib.LogDebugBuilder", - "org.kitesdk.morphline.avro.ReadAvroContainerBuilder", - "org.kitesdk.morphline.avro.ExtractAvroPathsBuilder", - "org.kitesdk.morphline.stdlib.ConvertTimestampBuilder", - "org.apache.solr.morphlines.solr.SanitizeUnknownSolrFieldsBuilder", - "org.apache.solr.morphlines.solr.LoadSolrBuilder"] - - commands : [ - { - # Parse Avro container file and emit a record for each avro object - readAvroContainer { - # Optionally, require the input record to match one of these MIME types: - # supportedMimeTypes : [avro/binary] - - # Optionally, use a custom Avro schema in JSON format inline: - # readerSchemaString : """""" - - # Optionally, use a custom Avro schema file in JSON format: - # readerSchemaFile : /path/to/syslog.avsc - } - } - - { - # Consume the output record of the previous command and pipe another record downstream. - # - # extractAvroPaths is a command that uses zero or more avro path expressions to extract - # values from an Avro object. Each expression consists of a record output field name (on - # the left side of the colon ':') as well as zero or more path steps (on the right hand - # side), each path step separated by a '/' slash. Avro arrays are traversed with the '[]' - # notation. - # - # The result of a path expression is a list of objects, each of which is added to the - # given record output field. - # - # The path language supports all Avro concepts, including nested structures, records, - # arrays, maps, unions, etc, as well as a flatten option that collects the primitives in - # a subtree into a flat list. - extractAvroPaths { - flatten : false - paths : { - id : /id - text : /text - user_friends_count : /user_friends_count - user_location : /user_location - user_description : /user_description - user_statuses_count : /user_statuses_count - user_followers_count : /user_followers_count - user_name : /user_name - user_screen_name : /user_screen_name - created_at : /created_at - retweet_count : /retweet_count - retweeted : /retweeted - in_reply_to_user_id : /in_reply_to_user_id - source : /source - in_reply_to_status_id : /in_reply_to_status_id - media_url_https : /media_url_https - expanded_url : /expanded_url - } - } - } - - # Consume the output record of the previous command and pipe another record downstream. - # - # convert timestamp field to native Solr timestamp format - # e.g. 2012-09-06T07:14:34Z to 2012-09-06T07:14:34.000Z - { - convertTimestamp { - field : created_at - inputFormats : ["yyyy-MM-dd'T'HH:mm:ss'Z'", "yyyy-MM-dd"] - inputTimezone : UTC -# outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSSZ" - outputTimezone : America/Los_Angeles - } - } - - # Consume the output record of the previous command and pipe another record downstream. - # - # This command sanitizes record fields that are unknown to Solr schema.xml by deleting - # them. Recall that Solr throws an exception on any attempt to load a document that - # contains a field that isn't specified in schema.xml. - { - sanitizeUnknownSolrFields { - # Location from which to fetch Solr schema - solrLocator : ${SOLR_LOCATOR} - } - } - - # log the record at DEBUG level to SLF4J - { logDebug { format : "output record: {}", args : ["@{}"] } } - - # load the record into a Solr server or MapReduce Reducer. - { - loadSolr { - solrLocator : ${SOLR_LOCATOR} - } - } - ] - } -] http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/53e5f34f/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java ---------------------------------------------------------------------- diff --git a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java b/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java deleted file mode 100644 index c91f31b..0000000 --- a/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java +++ /dev/null @@ -1,318 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.morphlines.solr; - -import com.codahale.metrics.MetricRegistry; -import com.google.common.io.Files; -import com.typesafe.config.Config; -import org.apache.commons.io.FileUtils; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; -import org.apache.solr.client.solrj.impl.XMLResponseParser; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrDocument; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.kitesdk.morphline.api.Collector; -import org.kitesdk.morphline.api.Command; -import org.kitesdk.morphline.api.MorphlineContext; -import org.kitesdk.morphline.api.Record; -import org.kitesdk.morphline.base.Compiler; -import org.kitesdk.morphline.base.FaultTolerance; -import org.kitesdk.morphline.base.Fields; -import org.kitesdk.morphline.base.Notifications; -import org.kitesdk.morphline.stdlib.PipeBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Locale; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TimeZone; -import java.util.concurrent.atomic.AtomicInteger; - -public class AbstractSolrMorphlineTestBase extends SolrTestCaseJ4 { - private static Locale savedLocale; - protected Collector collector; - protected Command morphline; - protected SolrClient solrClient; - protected DocumentLoader testServer; - - protected static final boolean TEST_WITH_EMBEDDED_SOLR_SERVER = true; - protected static final String EXTERNAL_SOLR_SERVER_URL = System.getProperty("externalSolrServer"); -// protected static final String EXTERNAL_SOLR_SERVER_URL = "http://127.0.0.1:8983/solr"; - - protected static final String RESOURCES_DIR = getFile("morphlines-core.marker").getParent(); - protected static final String DEFAULT_BASE_DIR = "solr"; - protected static final AtomicInteger SEQ_NUM = new AtomicInteger(); - protected static final AtomicInteger SEQ_NUM2 = new AtomicInteger(); - - protected static final Object NON_EMPTY_FIELD = new Object(); - - private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - protected String tempDir; - - @BeforeClass - public static void beforeClass() throws Exception { - - // TODO: test doesn't work with some Locales, see SOLR-6458 - savedLocale = Locale.getDefault(); - Locale.setDefault(Locale.ENGLISH); - - // we leave this in case the above is addressed - assumeFalse("This test fails on UNIX with Turkish default locale (https://issues.apache.org/jira/browse/SOLR-6387)", - new Locale("tr").getLanguage().equals(Locale.getDefault().getLanguage())); - - myInitCore(DEFAULT_BASE_DIR); - } - - @AfterClass - public static void afterClass() throws Exception { - if (savedLocale != null) { - Locale.setDefault(savedLocale); - } - savedLocale = null; - } - - protected static void myInitCore(String baseDirName) throws Exception { - String solrHome = RESOURCES_DIR + File.separator + baseDirName; - initCore( - "solrconfig.xml", "schema.xml", solrHome - ); - } - - @Before - public void setUp() throws Exception { - super.setUp(); - collector = new Collector(); - - if (EXTERNAL_SOLR_SERVER_URL != null) { - //solrServer = new ConcurrentUpdateSolrServer(EXTERNAL_SOLR_SERVER_URL, 2, 2); - //solrServer = new SafeConcurrentUpdateSolrServer(EXTERNAL_SOLR_SERVER_URL, 2, 2); - solrClient = getHttpSolrClient(EXTERNAL_SOLR_SERVER_URL); - ((HttpSolrClient) solrClient).setParser(new XMLResponseParser()); - } else { - if (TEST_WITH_EMBEDDED_SOLR_SERVER) { - solrClient = new EmbeddedTestSolrServer(h.getCoreContainer(), DEFAULT_TEST_CORENAME); - } else { - throw new RuntimeException("Not yet implemented"); - //solrServer = new TestSolrServer(getSolrClient()); - } - } - - int batchSize = SEQ_NUM2.incrementAndGet() % 2 == 0 ? 100 : 1; //SolrInspector.DEFAULT_SOLR_SERVER_BATCH_SIZE : 1; - testServer = new SolrClientDocumentLoader(solrClient, batchSize); - deleteAllDocuments(); - - tempDir = createTempDir().toFile().getAbsolutePath(); - } - - @After - public void tearDown() throws Exception { - collector = null; - solrClient.close(); - solrClient = null; - super.tearDown(); - } - - protected void testDocumentTypesInternal( - String[] files, - Map expectedRecords, - Map> expectedRecordContents) throws Exception { - - assumeTrue("This test has issues with this locale: https://issues.apache.org/jira/browse/SOLR-5778", - "GregorianCalendar".equals(Calendar.getInstance(TimeZone.getDefault(), Locale.getDefault()).getClass().getSimpleName())); - deleteAllDocuments(); - int numDocs = 0; - for (int i = 0; i < 1; i++) { - - for (String file : files) { - File f = new File(file); - byte[] body = Files.toByteArray(f); - Record event = new Record(); - //event.put(Fields.ID, docId++); - event.getFields().put(Fields.ATTACHMENT_BODY, new ByteArrayInputStream(body)); - event.getFields().put(Fields.ATTACHMENT_NAME, f.getName()); - event.getFields().put(Fields.BASE_ID, f.getName()); - collector.reset(); - load(event); - Integer count = expectedRecords.get(file); - if (count != null) { - numDocs += count; - } else { - numDocs++; - } - assertEquals("unexpected results in " + file, numDocs, queryResultSetSize("*:*")); - Map expectedContents = expectedRecordContents.get(file); - if (expectedContents != null) { - Record actual = collector.getFirstRecord(); - for (Map.Entry entry : expectedContents.entrySet()) { - if (entry.getValue() == NON_EMPTY_FIELD) { - assertNotNull(entry.getKey()); - assertTrue(actual.getFirstValue(entry.getKey()).toString().length() > 0); - } else if (entry.getValue() == null) { - assertEquals("key:" + entry.getKey(), 0, actual.get(entry.getKey()).size()); - } else { - assertEquals("key:" + entry.getKey(), Arrays.asList(entry.getValue()), actual.get(entry.getKey())); - } - } - } - } - } - assertEquals(numDocs, queryResultSetSize("*:*")); - } - - private boolean load(Record record) { - Notifications.notifyStartSession(morphline); - return morphline.process(record); - } - - protected int queryResultSetSize(String query) { -// return collector.getRecords().size(); - try { - testServer.commitTransaction(); - solrClient.commit(false, true, true); - QueryResponse rsp = solrClient.query(new SolrQuery(query).setRows(Integer.MAX_VALUE)); - LOGGER.debug("rsp: {}", rsp); - int i = 0; - for (SolrDocument doc : rsp.getResults()) { - LOGGER.debug("rspDoc #{}: {}", i++, doc); - } - int size = rsp.getResults().size(); - return size; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private void deleteAllDocuments() throws SolrServerException, IOException { - collector.reset(); - SolrClient s = solrClient; - s.deleteByQuery("*:*"); // delete everything! - s.commit(); - } - - protected Command createMorphline(String file) throws IOException { - return new PipeBuilder().build(parse(file), null, collector, createMorphlineContext()); - } - - private MorphlineContext createMorphlineContext() { - return new SolrMorphlineContext.Builder() - .setDocumentLoader(testServer) -// .setDocumentLoader(new CollectingDocumentLoader(100)) - .setExceptionHandler(new FaultTolerance(false, false, SolrServerException.class.getName())) - .setMetricRegistry(new MetricRegistry()) - .build(); - } - - private Config parse(String file) throws IOException { - SolrLocator locator = new SolrLocator(createMorphlineContext()); - locator.setSolrHomeDir(testSolrHome + "/collection1"); - File morphlineFile; - if (new File(file).isAbsolute()) { - morphlineFile = new File(file + ".conf"); - } else { - morphlineFile = new File(RESOURCES_DIR + "/" + file + ".conf"); - } - Config config = new Compiler().parse(morphlineFile, locator.toConfig("SOLR_LOCATOR")); - config = config.getConfigList("morphlines").get(0); - return config; - } - - protected void startSession() { - Notifications.notifyStartSession(morphline); - } - - protected void testDocumentContent(HashMap expectedResultMap) - throws Exception { - QueryResponse rsp = solrClient.query(new SolrQuery("*:*").setRows(Integer.MAX_VALUE)); - // Check that every expected field/values shows up in the actual query - for (Entry current : expectedResultMap.entrySet()) { - String field = current.getKey(); - for (String expectedFieldValue : current.getValue().getFieldValues()) { - ExpectedResult.CompareType compareType = current.getValue().getCompareType(); - boolean foundField = false; - - for (SolrDocument doc : rsp.getResults()) { - Collection