flink-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (FLINK-9642) Reduce the count to deal with state during a CEP process
Date Fri, 10 Aug 2018 13:17:02 GMT

    [ https://issues.apache.org/jira/browse/FLINK-9642?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16576261#comment-16576261
] 

ASF GitHub Bot commented on FLINK-9642:
---------------------------------------

dawidwys commented on a change in pull request #6205: [FLINK-9642]Reduce the count to deal
with state during a CEP process
URL: https://github.com/apache/flink/pull/6205#discussion_r209251047
 
 

 ##########
 File path: flink-libraries/flink-cep/src/main/java/org/apache/flink/cep/nfa/sharedbuffer/SharedBuffer.java
 ##########
 @@ -204,227 +133,78 @@ public NodeId put(
 	 * @throws Exception Thrown if the system cannot access the state.
 	 */
 	public boolean isEmpty() throws Exception {
-		return Iterables.isEmpty(eventsBuffer.keys());
+		return Iterables.isEmpty(eventsBuffer.keys()) && Iterables.isEmpty(eventsBufferCache.keySet());
 	}
 
 	/**
-	 * Returns all elements from the previous relation starting at the given entry.
-	 *
-	 * @param nodeId  id of the starting entry
-	 * @param version Version of the previous relation which shall be extracted
-	 * @return Collection of previous relations starting with the given value
-	 * @throws Exception Thrown if the system cannot access the state.
+	 * Put an event to cache.
+	 * @param eventId id of the event
+	 * @param event event body
 	 */
-	public List<Map<String, List<EventId>>> extractPatterns(
-			final NodeId nodeId,
-			final DeweyNumber version) throws Exception {
-
-		List<Map<String, List<EventId>>> result = new ArrayList<>();
-
-		// stack to remember the current extraction states
-		Stack<ExtractionState> extractionStates = new Stack<>();
-
-		// get the starting shared buffer entry for the previous relation
-		Lockable<SharedBufferNode> entryLock = entries.get(nodeId);
-
-		if (entryLock != null) {
-			SharedBufferNode entry = entryLock.getElement();
-			extractionStates.add(new ExtractionState(Tuple2.of(nodeId, entry), version, new Stack<>()));
-
-			// use a depth first search to reconstruct the previous relations
-			while (!extractionStates.isEmpty()) {
-				final ExtractionState extractionState = extractionStates.pop();
-				// current path of the depth first search
-				final Stack<Tuple2<NodeId, SharedBufferNode>> currentPath = extractionState.getPath();
-				final Tuple2<NodeId, SharedBufferNode> currentEntry = extractionState.getEntry();
-
-				// termination criterion
-				if (currentEntry == null) {
-					final Map<String, List<EventId>> completePath = new LinkedHashMap<>();
-
-					while (!currentPath.isEmpty()) {
-						final NodeId currentPathEntry = currentPath.pop().f0;
-
-						String page = currentPathEntry.getPageName();
-						List<EventId> values = completePath
-							.computeIfAbsent(page, k -> new ArrayList<>());
-						values.add(currentPathEntry.getEventId());
-					}
-					result.add(completePath);
-				} else {
-
-					// append state to the path
-					currentPath.push(currentEntry);
-
-					boolean firstMatch = true;
-					for (SharedBufferEdge edge : currentEntry.f1.getEdges()) {
-						// we can only proceed if the current version is compatible to the version
-						// of this previous relation
-						final DeweyNumber currentVersion = extractionState.getVersion();
-						if (currentVersion.isCompatibleWith(edge.getDeweyNumber())) {
-							final NodeId target = edge.getTarget();
-							Stack<Tuple2<NodeId, SharedBufferNode>> newPath;
-
-							if (firstMatch) {
-								// for the first match we don't have to copy the current path
-								newPath = currentPath;
-								firstMatch = false;
-							} else {
-								newPath = new Stack<>();
-								newPath.addAll(currentPath);
-							}
-
-							extractionStates.push(new ExtractionState(
-								target != null ? Tuple2.of(target, entries.get(target).getElement()) : null,
-								edge.getDeweyNumber(),
-								newPath));
-						}
-					}
-				}
-
-			}
-		}
-		return result;
+	void cacheEvent(EventId eventId, Lockable<V> event) {
+		this.eventsBufferCache.put(eventId, event);
 	}
 
-	public Map<String, List<V>> materializeMatch(Map<String, List<EventId>>
match) {
-		return materializeMatch(match, new HashMap<>());
+	/**
+	 * Put a ShareBufferNode to cache.
+	 * @param nodeId id of the event
+	 * @param entry SharedBufferNode
+	 */
+	void cacheEntry(NodeId nodeId, Lockable<SharedBufferNode> entry) {
+		this.entryCache.put(nodeId, entry);
 	}
 
-	public Map<String, List<V>> materializeMatch(Map<String, List<EventId>>
match, Map<EventId, V> cache) {
-
-		Map<String, List<V>> materializedMatch = new LinkedHashMap<>(match.size());
-
-		for (Map.Entry<String, List<EventId>> pattern : match.entrySet()) {
-			List<V> events = new ArrayList<>(pattern.getValue().size());
-			for (EventId eventId : pattern.getValue()) {
-				V event = cache.computeIfAbsent(eventId, id -> {
-					try {
-						return eventsBuffer.get(id).getElement();
-					} catch (Exception ex) {
-						throw new WrappingRuntimeException(ex);
-					}
-				});
-				events.add(event);
-			}
-			materializedMatch.put(pattern.getKey(), events);
-		}
-
-		return materializedMatch;
+	/**
+	 * Remove an event from cache and state.
+	 * @param eventId id of the event
+	 */
+	void removeEvent(EventId eventId) throws Exception {
+		this.eventsBufferCache.remove(eventId);
+		this.eventsBuffer.remove(eventId);
 	}
 
 	/**
-	 * Increases the reference counter for the given entry so that it is not
-	 * accidentally removed.
-	 *
-	 * @param node id of the entry
-	 * @throws Exception Thrown if the system cannot access the state.
+	 * remove a ShareBufferNode from cache and state.
+	 * @param nodeId id of the event
 	 */
-	public void lockNode(final NodeId node) throws Exception {
-		Lockable<SharedBufferNode> sharedBufferNode = entries.get(node);
-		if (sharedBufferNode != null) {
-			sharedBufferNode.lock();
-			entries.put(node, sharedBufferNode);
-		}
+	void removeEntry(NodeId nodeId) throws Exception {
+		this.entryCache.remove(nodeId);
+		this.entries.remove(nodeId);
 	}
 
 	/**
-	 * Decreases the reference counter for the given entry so that it can be
-	 * removed once the reference counter reaches 0.
-	 *
-	 * @param node id of the entry
+	 * Try to get the sharedBufferNode from state iff the node has not been quered during this
turn process.
+	 * @param nodeId id of the event
+	 * @return SharedBufferNode
 	 * @throws Exception Thrown if the system cannot access the state.
 	 */
-	public void releaseNode(final NodeId node) throws Exception {
-		Lockable<SharedBufferNode> sharedBufferNode = entries.get(node);
-		if (sharedBufferNode != null) {
-			if (sharedBufferNode.release()) {
-				removeNode(node, sharedBufferNode.getElement());
-			} else {
-				entries.put(node, sharedBufferNode);
-			}
-		}
+	Lockable<SharedBufferNode> getEntry(NodeId nodeId) throws Exception {
+		Lockable<SharedBufferNode> entry = entryCache.get(nodeId);
+		return  entry != null ? entry : entries.get(nodeId);
 	}
 
-	private void removeNode(NodeId node, SharedBufferNode sharedBufferNode) throws Exception
{
-		entries.remove(node);
-		EventId eventId = node.getEventId();
-		releaseEvent(eventId);
-
-		for (SharedBufferEdge sharedBufferEdge : sharedBufferNode.getEdges()) {
-			releaseNode(sharedBufferEdge.getTarget());
-		}
-	}
-
-	private void lockEvent(EventId eventId) throws Exception {
-		Lockable<V> eventWrapper = eventsBuffer.get(eventId);
-		checkState(
-			eventWrapper != null,
-			"Referring to non existent event with id %s",
-			eventId);
-		eventWrapper.lock();
-		eventsBuffer.put(eventId, eventWrapper);
+	Lockable<V> getEvent(EventId eventId) throws Exception {
+		Lockable<V> event = eventsBufferCache.get(eventId);
+		return event != null ? event : eventsBuffer.get(eventId);
 	}
 
 	/**
-	 * Decreases the reference counter for the given event so that it can be
-	 * removed once the reference counter reaches 0.
-	 *
-	 * @param eventId id of the event
+	 * Flush the event and node from cache to state.
 
 Review comment:
   Missing empty line

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


> Reduce the count to deal with state during a CEP process 
> ---------------------------------------------------------
>
>                 Key: FLINK-9642
>                 URL: https://issues.apache.org/jira/browse/FLINK-9642
>             Project: Flink
>          Issue Type: Improvement
>          Components: CEP
>    Affects Versions: 1.6.0
>            Reporter: aitozi
>            Assignee: aitozi
>            Priority: Major
>              Labels: pull-request-available
>
> With the rework of sharedBuffer Flink-9418, the lock & release operation is deal
with rocksdb state which is different from the previous version which will read the state
of sharedBuffer all to memory, i think we can add a cache or variable in sharedbuffer to cache
the LockAble Object to mark the ref change in once process in NFA, this will reduce the count
when the events point to the same NodeId.. And flush the result to MapState at the end of
process. 



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message