nifi-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alopre...@apache.org
Subject [nifi] branch main updated: NIFI-7777 Added optional Password property to UnpackContent for decrypting Zip archives
Date Wed, 07 Oct 2020 17:57:11 GMT
This is an automated email from the ASF dual-hosted git repository.

alopresto pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git


The following commit(s) were added to refs/heads/main by this push:
     new ea6b011  NIFI-7777 Added optional Password property to UnpackContent for decrypting
Zip archives
ea6b011 is described below

commit ea6b01159d16a1d14e611f3a211caa42fe5577f2
Author: exceptionfactory <exceptionfactory@gmail.com>
AuthorDate: Mon Oct 5 20:28:41 2020 -0400

    NIFI-7777 Added optional Password property to UnpackContent for decrypting Zip archives
    
    This closes #4572.
    
    Signed-off-by: Andy LoPresto <alopresto@apache.org>
---
 nifi-assembly/NOTICE                               |  5 ++
 .../nifi-standard-processors/pom.xml               |  5 ++
 .../nifi/processors/standard/UnpackContent.java    | 54 ++++++++++++++++++----
 .../processors/standard/TestUnpackContent.java     | 54 ++++++++++++++++++++++
 4 files changed, 108 insertions(+), 10 deletions(-)

diff --git a/nifi-assembly/NOTICE b/nifi-assembly/NOTICE
index b6ee96c..177f0ea 100644
--- a/nifi-assembly/NOTICE
+++ b/nifi-assembly/NOTICE
@@ -1946,6 +1946,11 @@ The following binary components are provided under the Apache Software
License v
       Apache FtpServer Core 1.1.1
       Copyright 2003-2017 The Apache Software Foundation
 
+  (ASLv2) Zip4j
+    The following NOTICE information applies:
+      Zip4j 2.6.3.
+      Copyright 2020 Srikanth Reddy Lingala
+
 ************************
 Common Development and Distribution License 1.1
 ************************
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 231e8eb..31d34aa 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -293,6 +293,11 @@
             <artifactId>snappy-java</artifactId>
         </dependency>
         <dependency>
+            <groupId>net.lingala.zip4j</groupId>
+            <artifactId>zip4j</artifactId>
+            <version>2.6.3</version>
+        </dependency>
+        <dependency>
             <groupId>com.h2database</groupId>
             <artifactId>h2</artifactId>
             <scope>test</scope>
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
index f802a5d..d6bf5a3 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
@@ -35,10 +35,11 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Pattern;
+
+import net.lingala.zip4j.model.LocalFileHeader;
 import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
 import org.apache.nifi.annotation.behavior.EventDriven;
 import org.apache.nifi.annotation.behavior.InputRequirement;
 import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
@@ -53,6 +54,8 @@ import org.apache.nifi.annotation.documentation.Tags;
 import org.apache.nifi.annotation.lifecycle.OnScheduled;
 import org.apache.nifi.annotation.lifecycle.OnStopped;
 import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.PropertyValue;
+import org.apache.nifi.expression.ExpressionLanguageScope;
 import org.apache.nifi.flowfile.FlowFile;
 import org.apache.nifi.flowfile.attributes.CoreAttributes;
 import org.apache.nifi.flowfile.attributes.FragmentAttributes;
@@ -72,6 +75,7 @@ import org.apache.nifi.util.FlowFileUnpackager;
 import org.apache.nifi.util.FlowFileUnpackagerV1;
 import org.apache.nifi.util.FlowFileUnpackagerV2;
 import org.apache.nifi.util.FlowFileUnpackagerV3;
+import net.lingala.zip4j.io.inputstream.ZipInputStream;
 
 @EventDriven
 @SideEffectFree
@@ -100,7 +104,8 @@ import org.apache.nifi.util.FlowFileUnpackagerV3;
     @WritesAttribute(attribute = "file.creationTime", description = "The date and time that
the file was created. This attribute holds always the same value as file.lastModifiedTime
(tar only)."),
     @WritesAttribute(attribute = "file.owner", description = "The owner of the unpacked file
(tar only)"),
     @WritesAttribute(attribute = "file.group", description = "The group owner of the unpacked
file (tar only)"),
-    @WritesAttribute(attribute = "file.permissions", description = "The read/write/execute
permissions of the unpacked file (tar only)")})
+    @WritesAttribute(attribute = "file.permissions", description = "The read/write/execute
permissions of the unpacked file (tar only)"),
+    @WritesAttribute(attribute = "file.encryptionMethod", description = "The encryption method
for entries in Zip archives")})
 @SeeAlso(MergeContent.class)
 public class UnpackContent extends AbstractProcessor {
     // attribute keys
@@ -123,6 +128,7 @@ public class UnpackContent extends AbstractProcessor {
     public static final String FILE_OWNER_ATTRIBUTE = "file.owner";
     public static final String FILE_GROUP_ATTRIBUTE = "file.group";
     public static final String FILE_PERMISSIONS_ATTRIBUTE = "file.permissions";
+    public static final String FILE_ENCRYPTION_METHOD_ATTRIBUTE = "file.encryptionMethod";
 
     public static final String FILE_MODIFIED_DATE_ATTR_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
     public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern(FILE_MODIFIED_DATE_ATTR_FORMAT).withZone(ZoneId.systemDefault());
@@ -145,6 +151,16 @@ public class UnpackContent extends AbstractProcessor {
             .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
             .build();
 
+    public static final PropertyDescriptor PASSWORD = new PropertyDescriptor.Builder()
+            .name("Password")
+            .displayName("Password")
+            .description("Password used for decrypting archive entries. Supports Zip files
encrypted with ZipCrypto or AES")
+            .required(false)
+            .sensitive(true)
+            .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+            .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+            .build();
+
     public static final Relationship REL_SUCCESS = new Relationship.Builder()
             .name("success")
             .description("Unpacked FlowFiles are sent to this relationship")
@@ -177,6 +193,7 @@ public class UnpackContent extends AbstractProcessor {
         final List<PropertyDescriptor> properties = new ArrayList<>();
         properties.add(PACKAGING_FORMAT);
         properties.add(FILE_FILTER);
+        properties.add(PASSWORD);
         this.properties = Collections.unmodifiableList(properties);
     }
 
@@ -200,7 +217,13 @@ public class UnpackContent extends AbstractProcessor {
         if (fileFilter == null) {
             fileFilter = Pattern.compile(context.getProperty(FILE_FILTER).getValue());
             tarUnpacker = new TarUnpacker(fileFilter);
-            zipUnpacker = new ZipUnpacker(fileFilter);
+
+            char[] password = null;
+            final PropertyValue passwordProperty = context.getProperty(PASSWORD);
+            if (passwordProperty.isSet()) {
+                password = passwordProperty.evaluateAttributeExpressions().getValue().toCharArray();
+            }
+            zipUnpacker = new ZipUnpacker(fileFilter, password);
         }
     }
 
@@ -301,8 +324,12 @@ public class UnpackContent extends AbstractProcessor {
 
         abstract void unpack(ProcessSession session, FlowFile source, List<FlowFile>
unpacked);
 
-        protected boolean fileMatches(ArchiveEntry entry) {
-            return fileFilter == null || fileFilter.matcher(entry.getName()).find();
+        protected boolean fileMatches(final ArchiveEntry entry) {
+            return fileMatches(entry.getName());
+        }
+
+        protected boolean fileMatches(final String entryName) {
+            return fileFilter == null || fileFilter.matcher(entryName).find();
         }
     }
 
@@ -370,8 +397,11 @@ public class UnpackContent extends AbstractProcessor {
     }
 
     private static class ZipUnpacker extends Unpacker {
-        public ZipUnpacker(Pattern fileFilter) {
+        private char[] password;
+
+        public ZipUnpacker(final Pattern fileFilter, final char[] password) {
             super(fileFilter);
+            this.password = password;
         }
 
         @Override
@@ -381,13 +411,14 @@ public class UnpackContent extends AbstractProcessor {
                 @Override
                 public void process(final InputStream in) throws IOException {
                     int fragmentCount = 0;
-                    try (final ZipArchiveInputStream zipIn = new ZipArchiveInputStream(new
BufferedInputStream(in))) {
-                        ArchiveEntry zipEntry;
+                    try (final ZipInputStream zipIn = new ZipInputStream(new BufferedInputStream(in),
password)) {
+                        LocalFileHeader zipEntry;
                         while ((zipEntry = zipIn.getNextEntry()) != null) {
-                            if (zipEntry.isDirectory() || !fileMatches(zipEntry)) {
+                            final String zipEntryName = zipEntry.getFileName();
+                            if (zipEntry.isDirectory() || !fileMatches(zipEntryName)) {
                                 continue;
                             }
-                            final File file = new File(zipEntry.getName());
+                            final File file = new File(zipEntryName);
                             final String parentDirectory = (file.getParent() == null) ? "/"
: file.getParent();
                             final Path absPath = file.toPath().toAbsolutePath();
                             final String absPathString = absPath.getParent().toString() +
"/";
@@ -403,6 +434,9 @@ public class UnpackContent extends AbstractProcessor {
                                 attributes.put(FRAGMENT_ID, fragmentId);
                                 attributes.put(FRAGMENT_INDEX, String.valueOf(++fragmentCount));
 
+                                final String encryptionMethod = zipEntry.getEncryptionMethod().toString();
+                                attributes.put(FILE_ENCRYPTION_METHOD_ATTRIBUTE, encryptionMethod);
+
                                 unpackedFile = session.putAllAttributes(unpackedFile, attributes);
                                 unpackedFile = session.write(unpackedFile, new OutputStreamCallback()
{
                                     @Override
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
index 42cc244..73635bd 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -31,7 +32,11 @@ import java.time.format.DateTimeParseException;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
+import net.lingala.zip4j.io.outputstream.ZipOutputStream;
+import net.lingala.zip4j.model.ZipParameters;
+import net.lingala.zip4j.model.enums.EncryptionMethod;
 import org.apache.nifi.flowfile.attributes.CoreAttributes;
 import org.apache.nifi.util.MockFlowFile;
 import org.apache.nifi.util.TestRunner;
@@ -222,6 +227,16 @@ public class TestUnpackContent {
     }
 
     @Test
+    public void testZipEncryptionZipStandard() throws IOException {
+        runZipEncryptionMethod(EncryptionMethod.ZIP_STANDARD);
+    }
+
+    @Test
+    public void testZipEncryptionAes() throws IOException {
+        runZipEncryptionMethod(EncryptionMethod.AES);
+    }
+
+    @Test
     public void testZipWithFilter() throws IOException {
         final TestRunner unpackRunner = TestRunners.newTestRunner(new UnpackContent());
         final TestRunner autoUnpackRunner = TestRunners.newTestRunner(new UnpackContent());
@@ -450,4 +465,43 @@ public class TestUnpackContent {
 
         runner.assertTransferCount(UnpackContent.REL_SUCCESS, numThreads*2);
     }
+
+    private void runZipEncryptionMethod(final EncryptionMethod encryptionMethod) throws IOException
{
+        final TestRunner runner = TestRunners.newTestRunner(new UnpackContent());
+        runner.setProperty(UnpackContent.PACKAGING_FORMAT, UnpackContent.PackageFormat.ZIP_FORMAT.toString());
+
+        final String password = String.class.getSimpleName();
+        runner.setProperty(UnpackContent.PASSWORD, password);
+
+        final char[] streamPassword = password.toCharArray();
+
+        final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        final ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream, streamPassword);
+
+        final String name = UUID.randomUUID().toString();
+        final String contents = TestRunner.class.getCanonicalName();
+
+        final ZipParameters zipParameters = new ZipParameters();
+        zipParameters.setEncryptionMethod(encryptionMethod);
+        zipParameters.setEncryptFiles(true);
+        zipParameters.setFileNameInZip(name);
+        zipOutputStream.putNextEntry(zipParameters);
+        zipOutputStream.write(contents.getBytes());
+        zipOutputStream.closeEntry();
+        zipOutputStream.close();
+
+        final byte[] bytes = outputStream.toByteArray();
+        runner.enqueue(bytes);
+        runner.run();
+
+        runner.assertTransferCount(UnpackContent.REL_SUCCESS, 1);
+        runner.assertTransferCount(UnpackContent.REL_ORIGINAL, 1);
+
+        final MockFlowFile unpacked = runner.getFlowFilesForRelationship(UnpackContent.REL_SUCCESS).iterator().next();
+        unpacked.assertAttributeEquals(UnpackContent.FILE_ENCRYPTION_METHOD_ATTRIBUTE, encryptionMethod.toString());
+
+        final byte[] unpackedBytes = runner.getContentAsByteArray(unpacked);
+        final String unpackedContents = new String(unpackedBytes);
+        assertEquals("Unpacked Contents not matched", contents, unpackedContents);
+    }
 }


Mime
View raw message