nifi-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apiri <...@git.apache.org>
Subject [GitHub] nifi pull request: NIFI-840: Create ListS3 processor
Date Sat, 23 Apr 2016 03:37:29 GMT
Github user apiri commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/238#discussion_r60822522
  
    --- Diff: nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/ListS3.java
---
    @@ -0,0 +1,232 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi.processors.aws.s3;
    +
    +import com.amazonaws.services.s3.AmazonS3;
    +import com.amazonaws.services.s3.model.ListObjectsRequest;
    +import com.amazonaws.services.s3.model.ObjectListing;
    +import com.amazonaws.services.s3.model.S3ObjectSummary;
    +import org.apache.nifi.annotation.behavior.InputRequirement;
    +import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
    +import org.apache.nifi.annotation.behavior.Stateful;
    +import org.apache.nifi.annotation.behavior.TriggerSerially;
    +import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
    +import org.apache.nifi.annotation.behavior.WritesAttribute;
    +import org.apache.nifi.annotation.behavior.WritesAttributes;
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.components.state.Scope;
    +import org.apache.nifi.components.state.StateMap;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.flowfile.attributes.CoreAttributes;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.util.StandardValidators;
    +
    +import java.io.IOException;
    +import java.util.Arrays;
    +import java.util.Collections;
    +import java.util.HashMap;
    +import java.util.HashSet;
    +import java.util.List;
    +import java.util.Map;
    +import java.util.Set;
    +import java.util.concurrent.TimeUnit;
    +
    +@TriggerSerially
    +@TriggerWhenEmpty
    +@InputRequirement(Requirement.INPUT_FORBIDDEN)
    +@Tags({"Amazon", "S3", "AWS", "list"})
    +@CapabilityDescription("Retrieves a listing of objects from an S3 bucket. For each object
that is listed, creates a FlowFile that represents \"\n" +
    +        "        + \"the object so that it can be fetched in conjunction with FetchS3Object.
This Processor is designed to run on Primary Node only \"\n" +
    +        "        + \"in a cluster. If the primary node changes, the new Primary Node
will pick up where the previous node left off without duplicating \"\n" +
    +        "        + \"all of the data.")
    +@Stateful(scopes = Scope.CLUSTER, description = "After performing a listing of keys,
the timestamp of the newest key is stored, "
    +        + "along with the keys that share that same timestamp. This allows the Processor
to list only keys that have been added or modified after "
    +        + "this date the next time that the Processor is run. State is stored across
the cluster so that this Processor can be run on Primary Node only and if a new Primary "
    +        + "Node is selected, the new node can pick up where the previous node left off,
without duplicating the data.")
    +@WritesAttributes({
    +        @WritesAttribute(attribute = "s3.bucket", description = "The name of the S3 bucket"),
    +        @WritesAttribute(attribute = "filename", description = "The name of the file"),
    +        @WritesAttribute(attribute = "s3.etag", description = "The ETag that can be used
to see if the file has changed"),
    +        @WritesAttribute(attribute = "s3.lastModified", description = "The last modified
time in milliseconds since epoch in UTC time"),
    +        @WritesAttribute(attribute = "s3.length", description = "The size of the object
in bytes"),
    +        @WritesAttribute(attribute = "s3.storeClass", description = "The storage class
of the object"),})
    +@SeeAlso({FetchS3Object.class, PutS3Object.class, DeleteS3Object.class})
    +public class ListS3 extends AbstractS3Processor {
    +
    +    public static final PropertyDescriptor DELIMITER = new PropertyDescriptor.Builder()
    +            .name("delimiter")
    +            .displayName("Delimiter")
    +            .expressionLanguageSupported(false)
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .description("The string used to delimit directories within the bucket. Please
consult the AWS documentation " +
    +                    "for the correct use of this field.")
    +            .build();
    +
    +    public static final PropertyDescriptor PREFIX = new PropertyDescriptor.Builder()
    +            .name("prefix")
    +            .displayName("Prefix")
    +            .expressionLanguageSupported(true)
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .description("The prefix used to filter the object list. In most cases, it
should end with a forward slash ('/').")
    +            .build();
    +
    +    public static final List<PropertyDescriptor> properties = Collections.unmodifiableList(
    +            Arrays.asList(BUCKET, KEY, REGION, ACCESS_KEY, SECRET_KEY, CREDENTIALS_FILE,
    +                    AWS_CREDENTIALS_PROVIDER_SERVICE, TIMEOUT, SSL_CONTEXT_SERVICE, ENDPOINT_OVERRIDE,
    +                    PROXY_HOST, PROXY_HOST_PORT, DELIMITER, PREFIX));
    +
    +    public static final Set<Relationship> relationships = Collections.unmodifiableSet(
    +            new HashSet<>(Collections.singletonList(REL_SUCCESS)));
    +
    +    public static final String CURRENT_TIMESTAMP = "currentTimestamp";
    +    public static final String CURRENT_KEY_PREFIX = "key-";
    +
    +    // State tracking
    +    private long currentTimestamp = 0L;
    +    private Set<String> currentKeys;
    +
    +    @Override
    +    protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
    +        return properties;
    +    }
    +
    +    @Override
    +    public Set<Relationship> getRelationships() {
    +        return relationships;
    +    }
    +
    +    private Set<String> extractKeys(final StateMap stateMap) {
    +        Set<String> keys = new HashSet<>();
    +        for (Map.Entry<String, String>  entry : stateMap.toMap().entrySet()) {
    +            if (entry.getKey().startsWith(CURRENT_KEY_PREFIX)) {
    +                keys.add(entry.getValue());
    +            }
    +        }
    +        return keys;
    +    }
    +
    +    private void restoreState(final ProcessContext context) throws IOException {
    +        final StateMap stateMap = context.getStateManager().getState(Scope.CLUSTER);
    +        if (stateMap.getVersion() == -1L || stateMap.get(CURRENT_TIMESTAMP) == null ||
stateMap.get(CURRENT_KEY_PREFIX) == null) {
    +            currentTimestamp = 0L;
    +            currentKeys = new HashSet<>();
    +        } else {
    +            currentTimestamp = Long.parseLong(stateMap.get(CURRENT_TIMESTAMP));
    +            currentKeys = extractKeys(stateMap);
    +        }
    +    }
    +
    +    private void persistState(final ProcessContext context) {
    +        Map<String, String> state = new HashMap<>();
    +        state.put(CURRENT_TIMESTAMP, String.valueOf(currentTimestamp));
    +        int i = 0;
    +        for (String key : currentKeys) {
    +            state.put(CURRENT_KEY_PREFIX+i, key);
    +        }
    +        try {
    +            context.getStateManager().setState(state, Scope.CLUSTER);
    +        } catch (IOException ioe) {
    +            getLogger().error("Failed to save cluster-wide state. If NiFi is restarted,
data duplication may occur", ioe);
    +        }
    +    }
    +
    +    @Override
    +    public void onTrigger(final ProcessContext context, final ProcessSession session)
{
    +        try {
    +            restoreState(context);
    --- End diff --
    
    Seems a bit heavy handed to have to restore state from the state manager on each trigger.
 What are your thoughts on managing a map within the processor, which has its contents persisted
after each execution and only performing a restore state in an @OnScheduled?


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message