eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h..@apache.org
Subject incubator-eagle git commit: [EAGLE-591][WIP] Fix conflict streamId between different sites while installing
Date Sat, 08 Oct 2016 03:36:27 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master f467954ac -> 7d43cfe10


[EAGLE-591][WIP] Fix conflict streamId between different sites while installing

Eagle alert engine currently only support global-unique streamId (where a "stream" in fact
means a physicla data source with schema), but it's common use case the different sites may
have same-kind (same stream schema) of streams especially when installing same app into different
sites, so we should refactor the alert engine metadata to support such kinds of use cases.

This ticket will only fix conflict streamId between different sites when installation as a
quick solution by simply using `{streamId}_{siteId}` as the physical installed `streamId`.

Author: Hao Chen <hao@apache.org>

Closes #476 from haoch/FixConflictStreamId.


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/7d43cfe1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/7d43cfe1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/7d43cfe1

Branch: refs/heads/master
Commit: 7d43cfe1057f9f926832d1056f433a6dbe57be96
Parents: f467954
Author: Hao Chen <hao@apache.org>
Authored: Sat Oct 8 11:36:16 2016 +0800
Committer: Hao Chen <hao@apache.org>
Committed: Sat Oct 8 11:36:16 2016 +0800

----------------------------------------------------------------------
 .../engine/coordinator/StreamDefinition.java    | 37 +++++++++++++++-
 .../service/ApplicationOperationContext.java    | 24 ++++++++---
 .../eagle/metadata/model/ApplicationEntity.java |  2 +-
 eagle-server/pom.xml                            | 45 ++++++++++----------
 4 files changed, 78 insertions(+), 30 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7d43cfe1/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinition.java
b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinition.java
index a23c963..e0789f9 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinition.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/StreamDefinition.java
@@ -29,12 +29,27 @@ import java.util.List;
  */
 public class StreamDefinition implements Serializable {
     private static final long serialVersionUID = 2352202882328931825L;
+
+    // Stream unique ID
     private String streamId;
-    private String dataSource;
+
+    // Stream description
     private String description;
+
+    // Is validateable or not
     private boolean validate;
+
+    // Is timeseries-based stream or not
     private boolean timeseries;
 
+    // TODO: Decouple dataSource and siteId from stream definition
+
+    // Stream data source ID
+    private String dataSource;
+
+    // Tenant (Site) ID
+    private String siteId;
+
     private List<StreamColumn> columns = new ArrayList<>();
 
     public String toString() {
@@ -107,4 +122,24 @@ public class StreamDefinition implements Serializable {
         }
         return -1;
     }
+
+    public String getSiteId() {
+        return siteId;
+    }
+
+    public void setSiteId(String siteId) {
+        this.siteId = siteId;
+    }
+
+    public StreamDefinition copy() {
+        StreamDefinition copied = new StreamDefinition();
+        copied.setColumns(this.getColumns());
+        copied.setDataSource(this.getDataSource());
+        copied.setDescription(this.getDescription());
+        copied.setSiteId(this.getSiteId());
+        copied.setStreamId(this.getStreamId());
+        copied.setTimeseries(this.isTimeseries());
+        copied.setValidate(this.isValidate());
+        return copied;
+    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7d43cfe1/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
index b4ae9dd..33a407c 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperationContext.java
@@ -80,22 +80,34 @@ public class ApplicationOperationContext implements Serializable, ApplicationLif
         this.alertMetadataService = alertMetadataService;
     }
 
+    /**
+     * Generate global unique streamId to install.
+     * TODO refactor with streamId and siteId
+     */
+    private static String generateUniqueStreamId(String siteId,String streamTypeId) {
+        return String.format("%s_%s",streamTypeId,siteId).toUpperCase();
+    }   
+
     @Override
     public void onInstall() {
         metadata.setExecutable(application.isExecutable());
         if (metadata.getDescriptor().getStreams() != null) {
-            List<StreamDesc> streamDescCollection = metadata.getDescriptor().getStreams().stream().map((streamDefinition
-> {
-                StreamSinkConfig streamSinkConfig = this.runtime.environment().streamSink().getSinkConfig(streamDefinition.getStreamId(),
this.config);
+            List<StreamDesc> streamDescToInstall = metadata.getDescriptor().getStreams().stream().map((streamDefinition
-> {
+                StreamDefinition copied = streamDefinition.copy();
+                copied.setSiteId(metadata.getSite().getSiteId());
+                copied.setStreamId(generateUniqueStreamId(metadata.getSite().getSiteId(),copied.getStreamId()));
+                StreamSinkConfig streamSinkConfig = this.runtime.environment().streamSink().getSinkConfig(copied.getStreamId(),
this.config);
                 StreamDesc streamDesc = new StreamDesc();
-                streamDesc.setSchema(streamDefinition);
+                streamDesc.setSchema(copied);
                 streamDesc.setSink(streamSinkConfig);
-                streamDesc.setStreamId(streamDefinition.getStreamId());
+                streamDesc.setStreamId(copied.getStreamId());
                 return streamDesc;
             })).collect(Collectors.toList());
-            metadata.setStreams(streamDescCollection);
+            metadata.setStreams(streamDescToInstall);
 
+            // TODO: Decouple converting from StreamSink to Alert DataSource
             // iterate each stream descriptor and create alert datasource for each
-            for (StreamDesc streamDesc : streamDescCollection) {
+            for (StreamDesc streamDesc : streamDescToInstall) {
                 // only take care of Kafka sink
                 if (streamDesc.getSink() instanceof KafkaStreamSinkConfig) {
                     KafkaStreamSinkConfig kafkaCfg = (KafkaStreamSinkConfig) streamDesc.getSink();

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7d43cfe1/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
index 2e3c387..f10ae76 100644
--- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
@@ -99,7 +99,7 @@ public class ApplicationEntity extends PersistenceEntity {
     public void ensureDefault() {
         super.ensureDefault();
         if (this.appId == null) {
-            this.appId = String.format("%s-%s", this.getDescriptor().getType(), this.getSite().getSiteId());
+            this.appId = String.format("%s_%s", this.getDescriptor().getType(), this.getSite().getSiteId()).toUpperCase();
         }
         if (this.status == null) {
             this.status = Status.INITIALIZED;

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7d43cfe1/eagle-server/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-server/pom.xml b/eagle-server/pom.xml
index 13cdff6..3f4c70b 100644
--- a/eagle-server/pom.xml
+++ b/eagle-server/pom.xml
@@ -223,27 +223,28 @@
         </profile>
     </profiles>
     <build>
-        <plugins>
-            <plugin>
-                <groupId>org.codehaus.mojo</groupId>
-                <artifactId>exec-maven-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>exec-ui-install</id>
-                        <phase>generate-sources</phase>
-                        <goals>
-                            <goal>exec</goal>
-                        </goals>
-                        <configuration>
-                            <executable>bash</executable>
-                            <arguments>
-                                <argument>${basedir}/ui-build.sh</argument>
-                            </arguments>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
+        <!-- TODO: Temporarily disable ui-build.sh until INFRA-12669 was resolved -->
+        <!--<plugins>-->
+            <!--<plugin>-->
+                <!--<groupId>org.codehaus.mojo</groupId>-->
+                <!--<artifactId>exec-maven-plugin</artifactId>-->
+                <!--<executions>-->
+                    <!--<execution>-->
+                        <!--<id>exec-ui-install</id>-->
+                        <!--<phase>generate-sources</phase>-->
+                        <!--<goals>-->
+                            <!--<goal>exec</goal>-->
+                        <!--</goals>-->
+                        <!--<configuration>-->
+                            <!--<executable>bash</executable>-->
+                            <!--<arguments>-->
+                                <!--<argument>${basedir}/ui-build.sh</argument>-->
+                            <!--</arguments>-->
+                        <!--</configuration>-->
+                    <!--</execution>-->
+                <!--</executions>-->
+            <!--</plugin>-->
+        <!--</plugins>-->
         <resources>
             <resource>
                 <directory>src/main/webapp/app/ui</directory>
@@ -254,4 +255,4 @@
             </resource>
         </resources>
     </build>
-</project>
+</project>
\ No newline at end of file


Mime
View raw message