Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B0DDB200BBB for ; Thu, 10 Nov 2016 21:09:23 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id AFB17160B01; Thu, 10 Nov 2016 20:09:23 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id AA6CF160AF7 for ; Thu, 10 Nov 2016 21:09:22 +0100 (CET) Received: (qmail 78116 invoked by uid 500); 10 Nov 2016 20:09:21 -0000 Mailing-List: contact commits-help@nifi.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@nifi.apache.org Delivered-To: mailing list commits@nifi.apache.org Received: (qmail 78107 invoked by uid 99); 10 Nov 2016 20:09:21 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 Nov 2016 20:09:21 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C4604E01F4; Thu, 10 Nov 2016 20:09:21 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jpercivall@apache.org To: commits@nifi.apache.org Message-Id: <6db322a7e75f49c2a162257b624a1d6c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: nifi-minifi git commit: MINIFI-136 - Fixing ordering issue in ConfigTransformer Date: Thu, 10 Nov 2016 20:09:21 +0000 (UTC) archived-at: Thu, 10 Nov 2016 20:09:23 -0000 Repository: nifi-minifi Updated Branches: refs/heads/master 12a58a869 -> 7954d36e9 MINIFI-136 - Fixing ordering issue in ConfigTransformer This closes #55 Signed-off-by: Joseph Percivall Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi/commit/7954d36e Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi/tree/7954d36e Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi/diff/7954d36e Branch: refs/heads/master Commit: 7954d36e90d6c53828ad042f33c3347fc11fea79 Parents: 12a58a8 Author: Bryan Rosander Authored: Wed Nov 9 21:11:33 2016 -0500 Committer: Joseph Percivall Committed: Thu Nov 10 15:03:47 2016 -0500 ---------------------------------------------------------------------- .../bootstrap/util/ConfigTransformer.java | 16 +- .../bootstrap/util/ConfigTransformerTest.java | 40 ++++- .../test/resources/config-funnel-and-rpg.yml | 170 +++++++++++++++++++ 3 files changed, 217 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7954d36e/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java index 19d3268..09776de 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java @@ -366,14 +366,6 @@ public final class ConfigTransformer { addProcessor(element, processorConfig); } - for (RemoteProcessingGroupSchema remoteProcessingGroupSchema : processGroupSchema.getRemoteProcessingGroups()) { - addRemoteProcessGroup(element, remoteProcessingGroupSchema); - } - - for (FunnelSchema funnelSchema : processGroupSchema.getFunnels()) { - addFunnel(element, funnelSchema); - } - for (PortSchema portSchema : processGroupSchema.getInputPortSchemas()) { addPort(doc, element, portSchema, "inputPort"); } @@ -382,12 +374,20 @@ public final class ConfigTransformer { addPort(doc, element, portSchema, "outputPort"); } + for (FunnelSchema funnelSchema : processGroupSchema.getFunnels()) { + addFunnel(element, funnelSchema); + } + for (ProcessGroupSchema child : processGroupSchema.getProcessGroupSchemas()) { Element processGroups = doc.createElement("processGroup"); element.appendChild(processGroups); addProcessGroup(doc, processGroups, child, parentGroupIdResolver); } + for (RemoteProcessingGroupSchema remoteProcessingGroupSchema : processGroupSchema.getRemoteProcessingGroups()) { + addRemoteProcessGroup(element, remoteProcessingGroupSchema); + } + for (ConnectionSchema connectionConfig : processGroupSchema.getConnections()) { addConnection(element, connectionConfig, parentGroupIdResolver); } http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7954d36e/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java index b41dc90..05f2abf 100644 --- a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java +++ b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java @@ -44,18 +44,23 @@ import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; public class ConfigTransformerTest { - + public static final Map PG_ELEMENT_ORDER_MAP = generateOrderMap( + Arrays.asList("processor", "inputPort", "outputPort", "funnel", "processGroup", "remoteProcessGroup", "connection")); private XPathFactory xPathFactory; private Document document; private Element config; @@ -109,6 +114,11 @@ public class ConfigTransformerTest { testConfigFileTransform("stress-test-framework-funnel.yml"); } + @Test + public void testFunnelAndRpgTransform() throws Exception { + testConfigFileTransform("config-funnel-and-rpg.yml"); + } + public void testConfigFileTransform(String configFile) throws Exception { ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(ConfigTransformerTest.class.getClassLoader().getResourceAsStream(configFile)); @@ -124,6 +134,8 @@ public class ConfigTransformerTest { assertEquals(processGroupSchema.getName(), getText(element, "name")); assertEquals(nullToEmpty(processGroupSchema.getComment()), nullToEmpty(getText(element, "comment"))); + checkOrderOfChildren(element, PG_ELEMENT_ORDER_MAP); + NodeList processorElements = (NodeList) xPathFactory.newXPath().evaluate("processor", element, XPathConstants.NODESET); assertEquals(processGroupSchema.getProcessors().size(), processorElements.getLength()); for (int i = 0; i < processorElements.getLength(); i++) { @@ -256,4 +268,30 @@ public class ConfigTransformerTest { private String nullToEmpty(Object val) { return val == null ? "" : val.toString(); } + + private static Map generateOrderMap(List elements) { + Map map = new HashMap<>(); + for (int i = 0; i < elements.size(); i++) { + map.put(elements.get(i), i); + } + return Collections.unmodifiableMap(map); + } + + private static void checkOrderOfChildren(Element element, Map orderMap) { + int elementOrderList = 0; + NodeList childNodes = element.getChildNodes(); + String lastOrderedElementName = null; + for (int i = 0; i < childNodes.getLength(); i++) { + String nodeName = childNodes.item(i).getNodeName(); + Integer index = orderMap.get(nodeName); + if (index != null) { + if (elementOrderList > index) { + fail("Found " + nodeName + " after " + lastOrderedElementName + "; expected all " + nodeName + " elements to come before the following elements: " + orderMap.entrySet().stream() + .filter(e -> e.getValue() > index ).sorted(Comparator.comparingInt(e -> e.getValue())).map(e -> e.getKey()).collect(Collectors.joining(", "))); + } + lastOrderedElementName = nodeName; + elementOrderList = index; + } + } + } } http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/7954d36e/minifi-bootstrap/src/test/resources/config-funnel-and-rpg.yml ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/resources/config-funnel-and-rpg.yml b/minifi-bootstrap/src/test/resources/config-funnel-and-rpg.yml new file mode 100644 index 0000000..bcc91d6 --- /dev/null +++ b/minifi-bootstrap/src/test/resources/config-funnel-and-rpg.yml @@ -0,0 +1,170 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 2 +Flow Controller: + name: MiNiFi test Funnels multiple dest + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: f028f52b-e4da-44fe-94b0-93eab6918cde + name: GenerateFlowFile + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: CRON_DRIVEN + scheduling period: 0/10 * * * * ? + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '1' + Data Format: Binary + File Size: 10B + Unique FlowFiles: 'false' +- id: dd424151-3e2c-427e-be9e-80b67e574d03 + name: UpdateAttribute + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Delete Attributes Expression: + a: 'true' +- id: 11101824-f926-4250-9dfc-8b52e04212bf + name: UpdateAttribute + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Delete Attributes Expression: + b: 'true' +Process Groups: [] +Input Ports: [] +Output Ports: [] +Funnels: +- id: b34f32fd-e9c3-4907-9e09-484506fc4417 +Connections: +- id: caac3cbe-45cd-4934-a918-f364c38e0bdb + name: GenerateFlowFile/success/b34f32fd-e9c3-4907-9e09-484506fc4417 + source id: f028f52b-e4da-44fe-94b0-93eab6918cde + source relationship names: + - success + destination id: b34f32fd-e9c3-4907-9e09-484506fc4417 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: a238c7ed-8c3f-4574-b5cb-75aa2d642298 + name: UpdateAttribute/success/2c1d6dcc-b11c-4ba7-8c6f-ad9e4f730cbe + source id: 11101824-f926-4250-9dfc-8b52e04212bf + source relationship names: + - success + destination id: 2c1d6dcc-b11c-4ba7-8c6f-ad9e4f730cbe + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: e943a827-d60b-4dd8-ad23-e4f3ab63f906 + name: UpdateAttribute/success/8d2c579e-4ad2-4922-a311-a37b7e551b7a + source id: dd424151-3e2c-427e-be9e-80b67e574d03 + source relationship names: + - success + destination id: 8d2c579e-4ad2-4922-a311-a37b7e551b7a + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: 45d9f462-e03d-4e96-98e8-17ea9844ca96 + name: b34f32fd-e9c3-4907-9e09-484506fc4417//UpdateAttribute + source id: b34f32fd-e9c3-4907-9e09-484506fc4417 + source relationship names: [] + destination id: dd424151-3e2c-427e-be9e-80b67e574d03 + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: 0eacb4c0-3c1d-4858-884b-5b3c8d1b1f6c + name: b34f32fd-e9c3-4907-9e09-484506fc4417//UpdateAttribute + source id: b34f32fd-e9c3-4907-9e09-484506fc4417 + source relationship names: [] + destination id: 11101824-f926-4250-9dfc-8b52e04212bf + max work queue size: 0 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: '' +Remote Processing Groups: +- name: NiFi Flow + url: http://localhost:8080/nifi + comment: '' + timeout: 30 sec + yield period: 10 sec + Input Ports: + - id: 8d2c579e-4ad2-4922-a311-a37b7e551b7a + name: input port + comment: '' + max concurrent tasks: 1 + use compression: false + - id: 2c1d6dcc-b11c-4ba7-8c6f-ad9e4f730cbe + name: provenance + comment: '' + max concurrent tasks: 1 + use compression: false