nifi-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From exceptionfact...@apache.org
Subject [nifi] branch main updated: NIFI-8523 Added SFTP algorithm and cipher properties
Date Thu, 24 Jun 2021 16:52:02 GMT
This is an automated email from the ASF dual-hosted git repository.

exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git


The following commit(s) were added to refs/heads/main by this push:
     new 115bba9  NIFI-8523 Added SFTP algorithm and cipher properties
115bba9 is described below

commit 115bba9ab01d432da60d1fc29359812690a5b453
Author: Jon Kessler <kesslerj@varentech.com>
AuthorDate: Thu May 6 16:10:36 2021 +0000

    NIFI-8523 Added SFTP algorithm and cipher properties
    
    - Updated secure FTP processors to configure which algorithms, ciphers and message authentication
codes are allowed to be used by the SSH Client
    - Included Expression Language Variable Registry support for properties
    
    This closes #5061
    
    Signed-off-by: David Handermann <exceptionfactory@apache.org>
---
 .../apache/nifi/processors/standard/FetchSFTP.java |   4 +
 .../apache/nifi/processors/standard/GetSFTP.java   |   4 +
 .../apache/nifi/processors/standard/ListSFTP.java  |   4 +
 .../apache/nifi/processors/standard/PutSFTP.java   |   4 +
 .../processors/standard/util/SFTPTransfer.java     | 113 +++++++++++++++++++++
 .../processors/standard/util/TestSFTPTransfer.java |  38 +++++++
 6 files changed, 167 insertions(+)

diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FetchSFTP.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FetchSFTP.java
index 2124b68..69281e5 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FetchSFTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FetchSFTP.java
@@ -87,6 +87,10 @@ public class FetchSFTP extends FetchFileTransfer {
         properties.add(FTPTransfer.HTTP_PROXY_USERNAME);
         properties.add(FTPTransfer.HTTP_PROXY_PASSWORD);
         properties.add(FILE_NOT_FOUND_LOG_LEVEL);
+        properties.add(SFTPTransfer.CIPHERS_ALLOWED);
+        properties.add(SFTPTransfer.KEY_ALGORITHMS_ALLOWED);
+        properties.add(SFTPTransfer.KEY_EXCHANGE_ALGORITHMS_ALLOWED);
+        properties.add(SFTPTransfer.MESSAGE_AUTHENTICATION_CODES_ALLOWED);
         return properties;
     }
 
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
index 5a3d667..0e105a8 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
@@ -89,6 +89,10 @@ public class GetSFTP extends GetFileTransfer {
         properties.add(FTPTransfer.PROXY_PORT);
         properties.add(FTPTransfer.HTTP_PROXY_USERNAME);
         properties.add(FTPTransfer.HTTP_PROXY_PASSWORD);
+        properties.add(SFTPTransfer.CIPHERS_ALLOWED);
+        properties.add(SFTPTransfer.KEY_ALGORITHMS_ALLOWED);
+        properties.add(SFTPTransfer.KEY_EXCHANGE_ALGORITHMS_ALLOWED);
+        properties.add(SFTPTransfer.MESSAGE_AUTHENTICATION_CODES_ALLOWED);
         this.properties = Collections.unmodifiableList(properties);
     }
 
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
index c345b3b..704cebc 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
@@ -116,6 +116,10 @@ public class ListSFTP extends ListFileTransfer {
         properties.add(ListFile.MAX_AGE);
         properties.add(ListFile.MIN_SIZE);
         properties.add(ListFile.MAX_SIZE);
+        properties.add(SFTPTransfer.CIPHERS_ALLOWED);
+        properties.add(SFTPTransfer.KEY_ALGORITHMS_ALLOWED);
+        properties.add(SFTPTransfer.KEY_EXCHANGE_ALGORITHMS_ALLOWED);
+        properties.add(SFTPTransfer.MESSAGE_AUTHENTICATION_CODES_ALLOWED);
         return properties;
     }
 
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSFTP.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSFTP.java
index 35bb174..e431042 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSFTP.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSFTP.java
@@ -80,6 +80,10 @@ public class PutSFTP extends PutFileTransfer<SFTPTransfer> {
         properties.add(FTPTransfer.PROXY_PORT);
         properties.add(FTPTransfer.HTTP_PROXY_USERNAME);
         properties.add(FTPTransfer.HTTP_PROXY_PASSWORD);
+        properties.add(SFTPTransfer.CIPHERS_ALLOWED);
+        properties.add(SFTPTransfer.KEY_ALGORITHMS_ALLOWED);
+        properties.add(SFTPTransfer.KEY_EXCHANGE_ALGORITHMS_ALLOWED);
+        properties.add(SFTPTransfer.MESSAGE_AUTHENTICATION_CODES_ALLOWED);
         this.properties = Collections.unmodifiableList(properties);
     }
 
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
index e7980cd..01cc6cf 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
@@ -18,8 +18,10 @@ package org.apache.nifi.processors.standard.util;
 
 import net.schmizz.keepalive.KeepAlive;
 import net.schmizz.keepalive.KeepAliveProvider;
+import net.schmizz.sshj.Config;
 import net.schmizz.sshj.DefaultConfig;
 import net.schmizz.sshj.SSHClient;
+import net.schmizz.sshj.common.Factory;
 import net.schmizz.sshj.connection.ConnectionException;
 import net.schmizz.sshj.connection.ConnectionImpl;
 import net.schmizz.sshj.sftp.FileAttributes;
@@ -72,17 +74,50 @@ import java.nio.file.Paths;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static org.apache.nifi.processors.standard.util.FTPTransfer.createComponentProxyConfigSupplier;
 
 public class SFTPTransfer implements FileTransfer {
+    private static final Set<String> DEFAULT_KEY_ALGORITHM_NAMES;
+    private static final Set<String> DEFAULT_CIPHER_NAMES;
+    private static final Set<String> DEFAULT_MESSAGE_AUTHENTICATION_CODE_NAMES;
+    private static final Set<String> DEFAULT_KEY_EXCHANGE_ALGORITHM_NAMES;
+
+    static {
+        DefaultConfig defaultConfig = new DefaultConfig();
+
+        DEFAULT_KEY_ALGORITHM_NAMES = Collections.unmodifiableSet(defaultConfig.getKeyAlgorithms().stream()
+                .map(Factory.Named::getName).collect(Collectors.toSet()));
+        DEFAULT_CIPHER_NAMES = Collections.unmodifiableSet(defaultConfig.getCipherFactories().stream()
+                .map(Factory.Named::getName).collect(Collectors.toSet()));
+        DEFAULT_MESSAGE_AUTHENTICATION_CODE_NAMES = Collections.unmodifiableSet(defaultConfig.getMACFactories().stream()
+                .map(Factory.Named::getName).collect(Collectors.toSet()));
+        DEFAULT_KEY_EXCHANGE_ALGORITHM_NAMES = Collections.unmodifiableSet(defaultConfig.getKeyExchangeFactories().stream()
+                .map(Factory.Named::getName).collect(Collectors.toSet()));
+    }
+
+    /**
+     * Converts a set of names into an alphabetically ordered comma separated value list.
+     *
+     * @param factorySetNames The set of names
+     * @return An alphabetically ordered comma separated value list of names
+     */
+    private static String convertFactorySetToString(Set<String> factorySetNames) {
+        return factorySetNames
+                .stream()
+                .sorted()
+                .collect(Collectors.joining(", "));
+    }
 
     public static final PropertyDescriptor PRIVATE_KEY_PATH = new PropertyDescriptor.Builder()
         .name("Private Key Path")
@@ -131,6 +166,44 @@ public class SFTPTransfer implements FileTransfer {
         .required(true)
         .build();
 
+    public static final PropertyDescriptor KEY_ALGORITHMS_ALLOWED = new PropertyDescriptor.Builder()
+            .name("Key Algorithms Allowed")
+            .displayName("Key Algorithms Allowed")
+            .description("A comma-separated list of Key Algorithms allowed for SFTP connections.
Leave unset to allow all. Available options are: "
+                    + convertFactorySetToString(DEFAULT_KEY_ALGORITHM_NAMES))
+            .required(false)
+            .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+            .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+            .build();
+
+    public static final PropertyDescriptor CIPHERS_ALLOWED = new PropertyDescriptor.Builder()
+            .name("Ciphers Allowed")
+            .displayName("Ciphers Allowed")
+            .description("A comma-separated list of Ciphers allowed for SFTP connections.
Leave unset to allow all. Available options are: " + convertFactorySetToString(DEFAULT_CIPHER_NAMES))
+            .required(false)
+            .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+            .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+            .build();
+
+    public static final PropertyDescriptor MESSAGE_AUTHENTICATION_CODES_ALLOWED = new PropertyDescriptor.Builder()
+            .name("Message Authentication Codes Allowed")
+            .displayName("Message Authentication Codes Allowed")
+            .description("A comma-separated list of Message Authentication Codes allowed
for SFTP connections. Leave unset to allow all. Available options are: "
+                    + convertFactorySetToString(DEFAULT_MESSAGE_AUTHENTICATION_CODE_NAMES))
+            .required(false)
+            .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+            .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+            .build();
+
+    public static final PropertyDescriptor KEY_EXCHANGE_ALGORITHMS_ALLOWED = new PropertyDescriptor.Builder()
+            .name("Key Exchange Algorithms Allowed")
+            .displayName("Key Exchange Algorithms Allowed")
+            .description("A comma-separated list of Key Exchange Algorithms allowed for SFTP
connections. Leave unset to allow all. Available options are: "
+                    + convertFactorySetToString(DEFAULT_KEY_EXCHANGE_ALGORITHM_NAMES))
+            .required(false)
+            .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+            .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+            .build();
 
     /**
      * Property which is used to decide if the {@link #ensureDirectoryExists(FlowFile, File)}
method should perform a {@link SFTPClient#ls(String)} before calling
@@ -521,6 +594,8 @@ public class SFTPTransfer implements FileTransfer {
             sshClientConfig.setKeepAliveProvider(NO_OP_KEEP_ALIVE);
         }
 
+        updateConfigAlgorithms(sshClientConfig);
+
         final SSHClient sshClient = new SSHClient(sshClientConfig);
 
         // Create a Proxy if the config was specified, proxy will be null if type was NO_PROXY
@@ -642,6 +717,44 @@ public class SFTPTransfer implements FileTransfer {
         return sftpClient;
     }
 
+    void updateConfigAlgorithms(final Config config) {
+        if (ctx.getProperty(CIPHERS_ALLOWED).isSet()) {
+            Set<String> allowedCiphers = Arrays.stream(ctx.getProperty(CIPHERS_ALLOWED).evaluateAttributeExpressions().getValue().split(","))
+                    .map(String::trim)
+                    .collect(Collectors.toSet());
+            config.setCipherFactories(config.getCipherFactories().stream()
+                    .filter(cipherNamed -> allowedCiphers.contains(cipherNamed.getName()))
+                    .collect(Collectors.toList()));
+        }
+
+        if (ctx.getProperty(KEY_ALGORITHMS_ALLOWED).isSet()) {
+            Set<String> allowedKeyAlgorithms = Arrays.stream(ctx.getProperty(KEY_ALGORITHMS_ALLOWED).evaluateAttributeExpressions().getValue().split(","))
+                    .map(String::trim)
+                    .collect(Collectors.toSet());
+            config.setKeyAlgorithms(config.getKeyAlgorithms().stream()
+                    .filter(keyAlgorithmNamed -> allowedKeyAlgorithms.contains(keyAlgorithmNamed.getName()))
+                    .collect(Collectors.toList()));
+        }
+
+        if (ctx.getProperty(KEY_EXCHANGE_ALGORITHMS_ALLOWED).isSet()) {
+            Set<String> allowedKeyExchangeAlgorithms = Arrays.stream(ctx.getProperty(KEY_EXCHANGE_ALGORITHMS_ALLOWED).evaluateAttributeExpressions().getValue().split(","))
+                    .map(String::trim)
+                    .collect(Collectors.toSet());
+            config.setKeyExchangeFactories(config.getKeyExchangeFactories().stream()
+                    .filter(keyExchangeNamed -> allowedKeyExchangeAlgorithms.contains(keyExchangeNamed.getName()))
+                    .collect(Collectors.toList()));
+        }
+
+        if (ctx.getProperty(MESSAGE_AUTHENTICATION_CODES_ALLOWED).isSet()) {
+            Set<String> allowedMessageAuthenticationCodes = Arrays.stream(ctx.getProperty(MESSAGE_AUTHENTICATION_CODES_ALLOWED).evaluateAttributeExpressions().getValue().split(","))
+                    .map(String::trim)
+                    .collect(Collectors.toSet());
+            config.setMACFactories(config.getMACFactories().stream()
+                    .filter(macNamed -> allowedMessageAuthenticationCodes.contains(macNamed.getName()))
+                    .collect(Collectors.toList()));
+        }
+    }
+
     @Override
     public String getHomeDirectory(final FlowFile flowFile) throws IOException {
         getSFTPClient(flowFile);
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestSFTPTransfer.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestSFTPTransfer.java
index e9d9a0b..e4307aa 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestSFTPTransfer.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestSFTPTransfer.java
@@ -16,13 +16,18 @@
  */
 package org.apache.nifi.processors.standard.util;
 
+import net.schmizz.sshj.DefaultConfig;
+import net.schmizz.sshj.common.Factory;
 import net.schmizz.sshj.sftp.Response;
 import net.schmizz.sshj.sftp.SFTPClient;
 import net.schmizz.sshj.sftp.SFTPException;
+import org.apache.nifi.components.PropertyDescriptor;
 import org.apache.nifi.flowfile.FlowFile;
 import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.mock.MockComponentLogger;
 import org.apache.nifi.processor.ProcessContext;
 import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.MockPropertyContext;
 import org.apache.nifi.util.MockPropertyValue;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -30,7 +35,10 @@ import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
@@ -244,4 +252,34 @@ public class TestSFTPTransfer {
         verify(sftpClient).mkdir(eq("/dir1/dir2/dir3")); // dir3 was created blindly.
     }
 
+    @Test
+    public void testRestrictSSHOptions() {
+        Map<PropertyDescriptor, String> propertyDescriptorValues = new HashMap<>();
+
+        DefaultConfig defaultConfig = new DefaultConfig();
+
+        String allowedMac = defaultConfig.getMACFactories().stream().map(Factory.Named::getName).collect(Collectors.toList()).get(0);
+        String allowedKeyAlgorithm = defaultConfig.getKeyAlgorithms().stream().map(Factory.Named::getName).collect(Collectors.toList()).get(0);
+        String allowedKeyExchangeAlgorithm = defaultConfig.getKeyExchangeFactories().stream().map(Factory.Named::getName).collect(Collectors.toList()).get(0);
+        String allowedCipher = defaultConfig.getCipherFactories().stream().map(Factory.Named::getName).collect(Collectors.toList()).get(0);
+
+        propertyDescriptorValues.put(SFTPTransfer.MESSAGE_AUTHENTICATION_CODES_ALLOWED, allowedMac);
+        propertyDescriptorValues.put(SFTPTransfer.CIPHERS_ALLOWED, allowedCipher);
+        propertyDescriptorValues.put(SFTPTransfer.KEY_ALGORITHMS_ALLOWED, allowedKeyAlgorithm);
+        propertyDescriptorValues.put(SFTPTransfer.KEY_EXCHANGE_ALGORITHMS_ALLOWED, allowedKeyExchangeAlgorithm);
+        MockPropertyContext mockPropertyContext = new MockPropertyContext(propertyDescriptorValues);
+        SFTPTransfer sftpTransfer = new SFTPTransfer(mockPropertyContext, new MockComponentLogger());
+
+        sftpTransfer.updateConfigAlgorithms(defaultConfig);
+
+        assertEquals(1, defaultConfig.getCipherFactories().size());
+        assertEquals(1, defaultConfig.getKeyAlgorithms().size());
+        assertEquals(1, defaultConfig.getKeyExchangeFactories().size());
+        assertEquals(1, defaultConfig.getMACFactories().size());
+
+        assertEquals(allowedCipher, defaultConfig.getCipherFactories().get(0).getName());
+        assertEquals(allowedKeyAlgorithm, defaultConfig.getKeyAlgorithms().get(0).getName());
+        assertEquals(allowedKeyExchangeAlgorithm, defaultConfig.getKeyExchangeFactories().get(0).getName());
+        assertEquals(allowedMac, defaultConfig.getMACFactories().get(0).getName());
+    }
 }

Mime
View raw message