From commits-return-41828-archive-asf-public=cust-asf.ponee.io@nifi.apache.org Tue Oct 1 19:51:44 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id B1C42180608 for ; Tue, 1 Oct 2019 21:51:43 +0200 (CEST) Received: (qmail 79181 invoked by uid 500); 1 Oct 2019 19:51:43 -0000 Mailing-List: contact commits-help@nifi.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@nifi.apache.org Delivered-To: mailing list commits@nifi.apache.org Received: (qmail 79166 invoked by uid 99); 1 Oct 2019 19:51:43 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 Oct 2019 19:51:43 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id D5697890A3; Tue, 1 Oct 2019 19:51:42 +0000 (UTC) Date: Tue, 01 Oct 2019 19:51:41 +0000 To: "commits@nifi.apache.org" Subject: [nifi] branch master updated: Revert NIFI-6703 as it creates startup issues or logs a confusing message to the user. Details on JIRA MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <156995950031.18020.13059947282590829973@gitbox.apache.org> From: joewitt@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: nifi X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: c2746fac5ff210a56704bf5ffe278359b9c85b54 X-Git-Newrev: 8abf330328953022c6506c173a11e12bde1abd64 X-Git-Rev: 8abf330328953022c6506c173a11e12bde1abd64 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. joewitt pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nifi.git The following commit(s) were added to refs/heads/master by this push: new 8abf330 Revert NIFI-6703 as it creates startup issues or logs a confusing message to the user. Details on JIRA 8abf330 is described below commit 8abf330328953022c6506c173a11e12bde1abd64 Author: Joe Witt AuthorDate: Tue Oct 1 15:33:07 2019 -0400 Revert NIFI-6703 as it creates startup issues or logs a confusing message to the user. Details on JIRA This reverts commit 61baa41e7b5a86618e5b8773569c4805867f211b. --- nifi-assembly/pom.xml | 11 - nifi-docker/dockermaven-stateless/Dockerfile | 26 +- .../dockermaven-stateless/integration-test.sh | 46 +++ nifi-docker/dockermaven-stateless/pom.xml | 13 +- .../nifi-resources/src/main/resources/bin/nifi.sh | 23 +- nifi-stateless/README.md | 33 +-- nifi-stateless/nifi-stateless-assembly/LICENSE | 313 +++++++++++++++++++++ nifi-stateless/nifi-stateless-assembly/NOTICE | 144 ++++++++++ .../pom.xml | 119 ++++---- .../src/main/assembly/dependencies.xml} | 36 ++- .../org/apache/nifi/stateless/NiFiStateless.java | 24 +- nifi-stateless/nifi-stateless-core/pom.xml | 25 -- nifi-stateless/pom.xml | 1 + 13 files changed, 637 insertions(+), 177 deletions(-) diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml index fb79bb5..8ce38b5 100755 --- a/nifi-assembly/pom.xml +++ b/nifi-assembly/pom.xml @@ -135,17 +135,6 @@ language governing permissions and limitations under the License. --> org.apache.nifi - nifi-stateless-bootstrap - 1.10.0-SNAPSHOT - - - org.apache.nifi - nifi-stateless-nar - 1.10.0-SNAPSHOT - nar - - - org.apache.nifi nifi-provenance-repository-nar 1.10.0-SNAPSHOT nar diff --git a/nifi-docker/dockermaven-stateless/Dockerfile b/nifi-docker/dockermaven-stateless/Dockerfile index 536329d..9dece1f 100644 --- a/nifi-docker/dockermaven-stateless/Dockerfile +++ b/nifi-docker/dockermaven-stateless/Dockerfile @@ -23,20 +23,36 @@ LABEL maintainer="Apache NiFi " ARG UID=1000 ARG GID=1000 ARG NIFI_VERSION -ARG LIB_DIR +ARG STATELESS_LIB_DIR ARG WORKING_DIR ENV NIFI_BASE_DIR /opt/nifi ENV NIFI_HOME ${NIFI_BASE_DIR}/nifi-current + +#Use Maven-Ant until Docker squash is stable +#COPY $NIFI_BINARY $NIFI_BASE_DIR +#RUN unzip ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip -d ${NIFI_BASE_DIR} \ +# && rm ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION}-bin.zip \ +# && mv ${NIFI_BASE_DIR}/nifi-${NIFI_VERSION} ${NIFI_HOME} +# +#COPY $NIFI_STATELESS_BINARY $NIFI_BASE_DIR +#RUN unzip ${NIFI_BASE_DIR}/nifi-stateless-${NIFI_VERSION}-bin.zip -d ${NIFI_BASE_DIR} \ +# && rm ${NIFI_BASE_DIR}/nifi-stateless-${NIFI_VERSION}-bin.zip \ +# && mv ${NIFI_BASE_DIR}/nifi-stateless-${NIFI_VERSION}/lib ${NIFI_HOME}/stateless-lib \ +# && rm -r ${NIFI_BASE_DIR}/nifi-stateless-${NIFI_VERSION} +# +#RUN java -cp "${NIFI_HOME}/stateless-lib/*" org.apache.nifi.stateless.NiFiStateless ExtractNars +#RUN rm -r ${NIFI_HOME}/lib + # Setup NiFi user RUN addgroup -g ${GID} nifi && adduser -s /bin/sh -u ${UID} -G nifi -D nifi RUN mkdir -p $NIFI_HOME && chown nifi:nifi $NIFI_HOME -RUN mkdir -p ${NIFI_HOME}/work/stateless-nars && chown nifi:nifi ${NIFI_HOME}/work/stateless-nars && chmod 777 ${NIFI_HOME}/work/stateless-nars +RUN mkdir -p ${NIFI_HOME}/work/ && chown nifi:nifi ${NIFI_HOME}/work/ && chmod 777 ${NIFI_HOME}/work/ -COPY --chown=nifi:nifi $WORKING_DIR ${NIFI_HOME}/work -COPY --chown=nifi:nifi $LIB_DIR ${NIFI_HOME}/lib/ +COPY --chown=nifi:nifi $WORKING_DIR ${NIFI_HOME}/work/ +COPY --chown=nifi:nifi $STATELESS_LIB_DIR ${NIFI_HOME}/stateless-lib/ #NiFi's HDFS processors require core-site.xml or hdfs-site.xml to exist on disk before they can be started... @@ -59,5 +75,5 @@ EXPOSE 8080 WORKDIR ${NIFI_HOME} -ENTRYPOINT ["/usr/bin/java", "-cp", "lib/*", "org.apache.nifi.stateless.NiFiStateless"] +ENTRYPOINT ["/usr/bin/java", "-cp", "stateless-lib/*", "org.apache.nifi.stateless.NiFiStateless"] CMD ["RunOpenwhiskActionServer", "8080"] \ No newline at end of file diff --git a/nifi-docker/dockermaven-stateless/integration-test.sh b/nifi-docker/dockermaven-stateless/integration-test.sh new file mode 100755 index 0000000..c5c0f7c --- /dev/null +++ b/nifi-docker/dockermaven-stateless/integration-test.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -exuo pipefail + +TAG=$1 +VERSION=$2 + +trap "{ docker ps -qaf Name=nifi-${TAG}-integration-test | xargs docker rm -f; }" EXIT + +echo "Checking that all files are owned by NiFi" +test -z $(docker run --rm --entrypoint /bin/bash apache/nifi-stateless:${TAG} -c "find /opt/nifi ! -user nifi") + +echo "Checking environment variables" +test "/opt/nifi/nifi-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi-stateless:${TAG} -c 'echo -n $NIFI_HOME')" +test "/opt/nifi/nifi-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi-stateless:${TAG} -c "readlink \${NIFI_BASE_DIR}/nifi-${VERSION}")" +test "/opt/nifi" = "$(docker run --rm --entrypoint /bin/bash apache/nifi-stateless:${TAG} -c 'echo -n $NIFI_BASE_DIR')" + +echo "Starting NiFi container..." +docker run -d --name nifi-${TAG}-integration-test apache/nifi-stateless:${TAG} + +IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' nifi-${TAG}-integration-test) + +for i in $(seq 1 10) :; do + if docker exec nifi-${TAG}-integration-test bash -c "ss -ntl | grep 8080"; then + break + fi + sleep 10 +done + +echo "Stopping NiFi container" +time docker stop nifi-${TAG}-integration-test \ No newline at end of file diff --git a/nifi-docker/dockermaven-stateless/pom.xml b/nifi-docker/dockermaven-stateless/pom.xml index eea6d15..5be23b0 100644 --- a/nifi-docker/dockermaven-stateless/pom.xml +++ b/nifi-docker/dockermaven-stateless/pom.xml @@ -36,9 +36,13 @@ + + + + @@ -52,7 +56,7 @@ - + @@ -68,7 +72,10 @@ - + + + + @@ -92,7 +99,7 @@ 1000 1000 ${project.version} - target/lib + target/stateless-lib target/work apache/nifi-stateless diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh index d2c8f34..b1d30af 100755 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh @@ -334,22 +334,6 @@ run() { echo } -stateless(){ - STATELESS_JAVA_OPTS="${STATELESS_JAVA_OPTS:=-Xms1024m -Xmx1024m}" - - init - shift - - echo - echo "Note: Use of this command is considered experimental. The commands and approach used may change from time to time." - echo - echo "Java home (JAVA_HOME): ${JAVA_HOME}" - echo "NiFi home (NIFI_HOME): ${NIFI_HOME}" - echo "Java options (STATELESS_JAVA_OPTS): ${STATELESS_JAVA_OPTS}" - echo - "${JAVA}" -cp "${NIFI_HOME}/lib/*:${NIFI_HOME}/lib/java11/*" ${STATELESS_JAVA_OPTS} "org.apache.nifi.stateless.NiFiStateless" ExtractNars - "${JAVA}" -cp "${NIFI_HOME}/lib/*:${NIFI_HOME}/lib/java11/*" ${STATELESS_JAVA_OPTS} "org.apache.nifi.stateless.NiFiStateless" "$@" -} main() { init "$1" run "$@" @@ -363,17 +347,12 @@ case "$1" in start|stop|run|status|dump|diagnostics|env) main "$@" ;; - - #Note: Use of this command is considered experimental. The commands and approach used may change from time to time. - stateless) - stateless "$@" - ;; restart) init run "stop" run "start" ;; *) - echo "Usage nifi {start|stop|run|restart|status|dump|diagnostics|install|stateless}" + echo "Usage nifi {start|stop|run|restart|status|dump|diagnostics|install}" ;; esac diff --git a/nifi-stateless/README.md b/nifi-stateless/README.md index 7831971..3b94c7c 100644 --- a/nifi-stateless/README.md +++ b/nifi-stateless/README.md @@ -22,13 +22,10 @@ Note: Provenance, metrics, logs are not extracted at this time. Docker and other Docker image will be tagged apache/nifi-stateless:1.10.0-SNAPSHOT-dockermaven ### Usage: -After building, the image can be used as follows: +After building, the image can be used as follows `docker run apache/nifi-stateless:1.10.0-SNAPSHOT-dockermaven ` -Stateless NiFi flows can also be run using nifi.sh -`./bin/nifi.sh stateless ` - -The dictate the runtime to use: +Where the arguments dictate the runtime to use: ``` 1) RunFromRegistry [Once|Continuous] --json RunFromRegistry [Once|Continuous] --file # Filename of JSON file that matches the examples below. @@ -41,12 +38,11 @@ The dictate the runtime to use: ### Examples: ``` -1) ${NIFI_HOME}/bin/nifi.sh stateless RunFromRegistry Once --file /Users/nifi/nifi-stateless-configs/flow-abc.json -2) docker run --rm -it nifi-stateless:1.10.0-SNAPSHOT-dockermaven \ +1) docker run --rm -it nifi-stateless:1.10.0-SNAPSHOT-dockermaven \ RunFromRegistry Once --file /Users/nifi/nifi-stateless-configs/flow-abc.json -3) docker run --rm -it nifi-stateless:1.10.0-SNAPSHOT-dockermaven \ +2) docker run --rm -it nifi-stateless:1.10.0-SNAPSHOT-dockermaven \ RunYARNServiceFromRegistry http://127.0.0.1:8088 nifi-stateless:latest kafka-to-solr 3 --file kafka-to-solr.json -4) docker run -d nifi-stateless:1.10.0-SNAPSHOT-dockermaven \ +3) docker run -d nifi-stateless:1.10.0-SNAPSHOT-dockermaven \ RunOpenwhiskActionServer 8080 ``` @@ -54,6 +50,7 @@ The dictate the runtime to use: ``` 1) The configuration file must be in JSON format. 2) When providing configurations via JSON, the following attributes must be provided: nifi_registry, nifi_bucket, nifi_flow. + All other attributes will be passed to the flow using the variable registry interface ``` ### JSON Format @@ -72,7 +69,7 @@ input FlowFile will be read as a stream of data and not buffered into heap. Howe Groups. - `flowFiles`: _Optional_ - An array of FlowFiles that should be provided to the flow's Input Port. Each element in the array is a JSON object. That JSON object can have multiple keys. If any of those keys is `nifi_content` then the String value of that element will be the FlowFile's content. Otherwise, the key/value pair is considered an attribute of the FlowFile. -- `parameters`: _Optional_ - Key-value pairs (or objects if sensitive) that will be passed to the NiFi Flow as parameters. +- `variables`: _Optional_ - Key/value pairs that will be passed to the NiFi Flow as variables of the root Process Group. ### Minimal JSON Sample: @@ -101,16 +98,16 @@ keys is `nifi_content` then the String value of that element will be the FlowFil "truststoreType": "JKS" }, "flowFiles":[{ - "absolute.path": "/tmp/nifistateless/input/", - "filename": "test.txt", + "absolute.path": "/tmp/nifistateless/input/", + "filename": "test.txt", - "nifi_content": "hello" + "nifi_content": "hello" }, { - "absolute.path": "/tmp/nifistateless/input/", - "filename": "test2.txt", + "absolute.path": "/tmp/nifistateless/input/", + "filename": "test2.txt", - "nifi_content": "hi" + "nifi_content": "hi" }], "parameters": { "DestinationDirectory" : "/tmp/nifistateless/output2/", @@ -126,6 +123,6 @@ keys is `nifi_content` then the String value of that element will be the FlowFil -StatelessProvenanceReporter.send force option is not appreciated -StatelessProcessSession.adjustCounter immediate is not appreciated * Send logs, metrics, and provenance to kafka/solr (configure a flow ID for each?) -* Counters -* Tests +* counters +* tests * Processor and port IDs from the UI do not match IDs in templates or the registry diff --git a/nifi-stateless/nifi-stateless-assembly/LICENSE b/nifi-stateless/nifi-stateless-assembly/LICENSE new file mode 100644 index 0000000..2734953 --- /dev/null +++ b/nifi-stateless/nifi-stateless-assembly/LICENSE @@ -0,0 +1,313 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +APACHE NIFI SUBCOMPONENTS: + +The Apache NiFi project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +The binary distribution of this product bundles 'Hamcrest' which is available +under a BSD license. More details found here: http://hamcrest.org. + + Copyright (c) 2000-2006, www.hamcrest.org + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. Redistributions in binary form must reproduce + the above copyright notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + + Neither the name of Hamcrest nor the names of its contributors may be used to endorse + or promote products derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT + SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + +The binary distribution of this product bundles 'Antlr 3' which is available +under a "3-clause BSD" license. For details see http://www.antlr3.org/license.html + + Copyright (c) 2010 Terence Parr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + Neither the name of the author nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + + +The binary distribution of this product bundles 'Bouncy Castle JDK 1.5' +under an MIT style license. + + Copyright (c) 2000 - 2015 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + + +This product bundles 'jBCrypt' which is available under an MIT license. +For details see https://github.com/svenkubiak/jBCrypt/blob/0.4.1/LICENSE + + Copyright (c) 2006 Damien Miller + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + + + diff --git a/nifi-stateless/nifi-stateless-assembly/NOTICE b/nifi-stateless/nifi-stateless-assembly/NOTICE new file mode 100644 index 0000000..de46e6f --- /dev/null +++ b/nifi-stateless/nifi-stateless-assembly/NOTICE @@ -0,0 +1,144 @@ +Apache NiFi Stateless +Copyright 2015-2019 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +****************** +Apache Software License v2 +****************** + +The following binary components are provided under the Apache Software License v2 + + (ASLv2) Jetty + The following NOTICE information applies: + Jetty Web Container + Copyright 1995-2017 Mort Bay Consulting Pty Ltd. + + (ASLv2) Google GSON + The following NOTICE information applies: + Copyright 2008 Google Inc. + + (ASLv2) Apache Commons Text + The following NOTICE information applies: + Apache Commons Text + Copyright 2001-2018 The Apache Software Foundation + + (ASLv2) Jackson JSON processor + The following NOTICE information applies: + # Jackson JSON processor + + Jackson is a high-performance, Free/Open Source JSON processing library. + It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has + been in development since 2007. + It is currently developed by a community of developers, as well as supported + commercially by FasterXML.com. + + ## Licensing + + Jackson core and extension components may licensed under different licenses. + To find the details that apply to this artifact see the accompanying LICENSE file. + For more information, including possible other licensing options, contact + FasterXML.com (http://fasterxml.com). + + ## Credits + + A list of contributors may be found from CREDITS file, which is included + in some artifacts (usually source distributions); but is always available + from the source code management (SCM) system project uses. + + (ASLv2) Apache Commons Codec + The following NOTICE information applies: + Apache Commons Codec + Copyright 2002-2014 The Apache Software Foundation + + src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java + contains test data from http://aspell.net/test/orig/batch0.tab. + Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + + =============================================================================== + + The content of package org.apache.commons.codec.language.bm has been translated + from the original php source code available at http://stevemorse.org/phoneticinfo.htm + with permission from the original authors. + Original source copyright: + Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + + (ASLv2) Apache Commons Lang + The following NOTICE information applies: + Apache Commons Lang + Copyright 2001-2017 The Apache Software Foundation + + This product includes software from the Spring Framework, + under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + + (ASLv2) Quartz + The following NOTICE information applies: + Copyright Declaration: + Copyright © 2003-2016 Software AG, Darmstadt, Germany and/or Software AG USA Inc., Reston, VA, USA, and/or its subsidiaries and/or its affiliates and/or their licensors. + + Trademark and Patent declaration + The name Software AG and all Software AG product names are either trademarks or registered trademarks of Software AG and/or Software AG USA Inc. and/or its subsidiaries and/or its affiliates + and/or their licensors. Other company and product names mentioned herein may be trademarks of their respective owners. + + Detailed information on trademarks and patents owned by Software AG and/or its subsidiaries is located at http://softwareag.com/licenses. + + Third Party declaration + This software may include portions of third-party products. For third-party copyright notices, license terms, additional rights or restrictions, please refer to "License Texts, Copyright + Notices and Disclaimers of Third Party Products". For certain specific third-party license restrictions, please refer to section E of the Legal Notices available under "License Terms and + Conditions for Use of Software AG Products / Copyright and Trademark Notices of Software AG Products". These documents are part of the product documentation, located at + http://softwareag.com/licenses and/or in the root installation directory of the licensed product(s). + + Confidentiality Disclaimer: + Use, reproduction, transfer, publication or disclosure is prohibited except as specifically provided for in your License Agreement with Software AG. + Contact GitHub API Training Shop Blog About + + (ASLv2) Spring Framework + The following NOTICE information applies: + Spring Framework 4.x,5.x.RELEASE + Copyright (c) 2002-2015 Pivotal, Inc. + + (ASLv2) Spring Security + The following NOTICE information applies: + Spring Framework 4.0.3.RELEASE + Copyright (c) 2002-2015 Pivotal, Inc. + + (ASLv2) Ehcache 2.x + The following NOTICE information applies: + Copyright 2003-2010 Terracotta, Inc. + + + + + +************************ +Common Development and Distribution License 1.1 +************************ + +The following binary components are provided under the Common Development and Distribution License 1.1. See project link for details. + + (CDDL 1.1) (GPL2 w/ CPE) Java Servlet API (javax.servlet:javax.servlet-api:jar:3.1.0 - http://servlet-spec.java.net) + (CDDL 1.1) (GPL2 w/ CPE) javax.ws.rs-api (javax.ws.rs:javax.ws.rs-api:jar:2.1 - http://jax-rs-spec.java.net) + (CDDL 1.1) (GPL2 w/ CPE) jersey-client (org.glassfish.jersey.core:jersey-client:jar:2.26 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-common (org.glassfish.jersey.core:jersey-common:jar:2.26 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-hk2 (org.glassfish.jersey.inject:jersey-hk2:jar:2.26 - https://jersey.github.io/) + (CDDL 1.1) (GPL2 w/ CPE) jersey-media-json-jackson (org.glassfish.jersey.media:jersey-media-json-jackson:jar:2.26 - https://jersey.github.io/) + + +************************ +Eclipse Public License 1.0 +************************ + +The following binary components are provided under the Eclipse Public License 1.0. See project link for details. + + (EPL 1.0)(LGPL 2.1) Logback Classic (ch.qos.logback:logback-classic:jar:1.2.3 - http://logback.qos.ch/) + (EPL 1.0) AspectJ Weaver (org.aspectj:aspectjweaver:jar:1.8.5 - http://www.eclipse.org/aspectj/) + (EPL 1.0) AspectJ Runtime (org.aspectj:aspectjrt:jar:1.8.0 - http://www.eclipse.org/aspectj/) + +***************** +Public Domain +***************** + +The following binary components are provided to the 'Public Domain'. See project link for details. + + (Public Domain) AOP Alliance 1.0 (http://aopalliance.sourceforge.net/) diff --git a/nifi-stateless/nifi-stateless-core/pom.xml b/nifi-stateless/nifi-stateless-assembly/pom.xml similarity index 57% copy from nifi-stateless/nifi-stateless-core/pom.xml copy to nifi-stateless/nifi-stateless-assembly/pom.xml index 1601c7d..336c112 100644 --- a/nifi-stateless/nifi-stateless-core/pom.xml +++ b/nifi-stateless/nifi-stateless-assembly/pom.xml @@ -24,27 +24,38 @@ 1.10.0-SNAPSHOT - nifi-stateless-core + nifi-stateless-assembly + pom - org.apache.maven.plugins - maven-dependency-plugin + maven-assembly-plugin + + nifi-stateless-${project.version} + false + - copy-dependencies - prepare-package + make shared resource - copy-dependencies + single + package - ${project.build.directory}/nifi-stateless-lib - false - true - true - compile - nar + + 0775 + 0775 + 0664 + + + src/main/assembly/dependencies.xml + + posix + + dir + zip + @@ -53,99 +64,81 @@ + org.apache.nifi - nifi-api - 1.10.0-SNAPSHOT - - - org.apache.nifi - nifi-framework-api - 1.10.0-SNAPSHOT - - - org.apache.nifi - nifi-framework-core-api + nifi-stateless-bootstrap 1.10.0-SNAPSHOT - org.apache.nifi - nifi-framework-nar-loading-utils + nifi-api 1.10.0-SNAPSHOT org.apache.nifi - nifi-nar-utils + nifi-utils 1.10.0-SNAPSHOT org.apache.nifi - nifi-expression-language + nifi-framework-api 1.10.0-SNAPSHOT org.apache.nifi - nifi-data-provenance-utils + nifi-properties 1.10.0-SNAPSHOT - com.google.code.gson - gson - 2.7 + javax.servlet + javax.servlet-api + 3.1.0 + runtime org.slf4j slf4j-api 1.7.25 + runtime - javax.xml.bind - jaxb-api - 2.3.0 - - - com.sun.xml.bind - jaxb-impl - 2.3.0 - - - com.sun.xml.bind - jaxb-core - 2.3.0 + ch.qos.logback + logback-classic + compile + + - javax.annotation - javax.annotation-api - 1.3.2 + org.apache.nifi + nifi-stateless-nar + 1.10.0-SNAPSHOT + nar - javax.activation - javax.activation-api - 1.2.0 + org.apache.nifi + nifi-standard-services-api-nar + 1.10.0-SNAPSHOT + nar - - - - junit - junit - 4.12 - test + org.apache.nifi + nifi-standard-nar + 1.10.0-SNAPSHOT + nar org.apache.nifi - nifi-standard-processors + nifi-update-attribute-nar 1.10.0-SNAPSHOT - test + nar org.apache.nifi - nifi-web-api + nifi-jetty-bundle 1.10.0-SNAPSHOT - war - test + nar - + diff --git a/nifi-stateless/pom.xml b/nifi-stateless/nifi-stateless-assembly/src/main/assembly/dependencies.xml similarity index 52% copy from nifi-stateless/pom.xml copy to nifi-stateless/nifi-stateless-assembly/src/main/assembly/dependencies.xml index 57da2b0..c304675 100644 --- a/nifi-stateless/pom.xml +++ b/nifi-stateless/nifi-stateless-assembly/src/main/assembly/dependencies.xml @@ -1,4 +1,4 @@ - + - - 4.0.0 + + bin + true + nifi-stateless-${project.version} - - org.apache.nifi - nifi - 1.10.0-SNAPSHOT - + + + + false + lib + 0770 + 0664 + false + + - nifi-stateless - pom - - - nifi-stateless-core - nifi-stateless-bootstrap - nifi-stateless-nar - - - + diff --git a/nifi-stateless/nifi-stateless-bootstrap/src/main/java/org/apache/nifi/stateless/NiFiStateless.java b/nifi-stateless/nifi-stateless-bootstrap/src/main/java/org/apache/nifi/stateless/NiFiStateless.java index d72d19d..74b697c 100644 --- a/nifi-stateless/nifi-stateless-bootstrap/src/main/java/org/apache/nifi/stateless/NiFiStateless.java +++ b/nifi-stateless/nifi-stateless-bootstrap/src/main/java/org/apache/nifi/stateless/NiFiStateless.java @@ -40,11 +40,6 @@ public class NiFiStateless { public static final String EXTRACT_NARS = "ExtractNars"; - public static final String framework_nar_prefix = "nifi-framework-nar"; - - public static final String libPath = "lib"; - public static final String narPath = "work/stateless-nars"; - public static void main(final String[] args) throws IOException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, IllegalAccessException { String nifi_home = System.getenv("NIFI_HOME"); @@ -52,8 +47,9 @@ public class NiFiStateless { nifi_home = "."; } - final File libDir = new File(nifi_home+"/"+libPath); - final File narWorkingDirectory = new File(nifi_home+"/"+narPath); + final File libDir = new File(nifi_home+"/lib"); + final File statelesslibDir = new File(nifi_home+"/stateless-lib"); + final File narWorkingDirectory = new File(nifi_home+"/work"); if(args.length >= 1 && args[0].equals(EXTRACT_NARS)){ if (!libDir.exists()) { @@ -61,7 +57,7 @@ public class NiFiStateless { return; } - final File[] narFiles = libDir.listFiles(file -> file.getName().endsWith(".nar") && !file.getName().startsWith(framework_nar_prefix)); + final File[] narFiles = libDir.listFiles(file -> file.getName().endsWith(".nar")); if (narFiles == null) { System.out.println("Could not obtain listing of lib directory <" + libDir + ">"); return; @@ -76,6 +72,14 @@ public class NiFiStateless { for (final File narFile : narFiles) { NarUnpacker.unpackNar(narFile, narWorkingDirectory); } + final File[] statelessNar = statelesslibDir.listFiles(file -> file.getName().endsWith(".nar")); + if (statelessNar == null) { + System.out.println("Could not find stateless nar in stateless-lib dir <" + statelesslibDir + ">"); + return; + } + for (final File narFile : statelessNar) { + NarUnpacker.unpackNar(narFile, narWorkingDirectory); + } final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startUnpack); logger.info("Finished unpacking {} NARs in {} millis", narFiles.length, millis); @@ -98,9 +102,9 @@ public class NiFiStateless { final URL[] statelessCoreUrls = toURLs(statelessCoreFiles); - final File[] jarFiles = libDir.listFiles(file -> file.getName().endsWith(".jar")); + final File[] jarFiles = statelesslibDir.listFiles(file -> file.getName().endsWith(".jar")); if (jarFiles == null) { - System.out.println("Could not obtain listing of lib directory <" + libDir + ">"); + System.out.println("Could not obtain listing of NiFi-Stateless Lib directory <" + libDir + ">"); return; } diff --git a/nifi-stateless/nifi-stateless-core/pom.xml b/nifi-stateless/nifi-stateless-core/pom.xml index 1601c7d..b28773a 100644 --- a/nifi-stateless/nifi-stateless-core/pom.xml +++ b/nifi-stateless/nifi-stateless-core/pom.xml @@ -99,31 +99,6 @@ slf4j-api 1.7.25 - - javax.xml.bind - jaxb-api - 2.3.0 - - - com.sun.xml.bind - jaxb-impl - 2.3.0 - - - com.sun.xml.bind - jaxb-core - 2.3.0 - - - javax.annotation - javax.annotation-api - 1.3.2 - - - javax.activation - javax.activation-api - 1.2.0 - diff --git a/nifi-stateless/pom.xml b/nifi-stateless/pom.xml index 57da2b0..0ba6edf 100644 --- a/nifi-stateless/pom.xml +++ b/nifi-stateless/pom.xml @@ -30,6 +30,7 @@ nifi-stateless-core nifi-stateless-bootstrap + nifi-stateless-assembly nifi-stateless-nar