eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qingwz...@apache.org
Subject eagle git commit: [EAGLE-999] JobConfigSerDeser fails to serialize/deserialize data with long string
Date Tue, 11 Apr 2017 09:37:00 GMT
Repository: eagle
Updated Branches:
  refs/heads/master 20b4bea3e -> 7c0d83f3c


[EAGLE-999] JobConfigSerDeser fails to serialize/deserialize data with long string

https://issues.apache.org/jira/browse/EAGLE-999

Author: Zhao, Qingwen <qingwzhao@apache.org>

Closes #909 from qingwen220/EAGLE-999.


Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/7c0d83f3
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/7c0d83f3
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/7c0d83f3

Branch: refs/heads/master
Commit: 7c0d83f3c4847dac450ba505153a5bf17d30fd73
Parents: 20b4bea
Author: Zhao, Qingwen <qingwzhao@apache.org>
Authored: Tue Apr 11 17:36:50 2017 +0800
Committer: Zhao, Qingwen <qingwzhao@apache.org>
Committed: Tue Apr 11 17:36:50 2017 +0800

----------------------------------------------------------------------
 ...doop.queue.HadoopQueueRunningAppProvider.xml |  2 +-
 eagle-jpm/eagle-jpm-entity/pom.xml              |  3 +-
 .../jpm/mr/historyentity/JobConfigSerDeser.java | 41 ++++--------
 .../eagle/jpm/mr/JobConfigSerDeserTest.java     | 65 ++++++++++++++++++++
 ....running.MRRunningJobApplicationProvider.xml |  2 +-
 ...ecurity.auditlog.HdfsAuditLogAppProvider.xml |  4 +-
 ...ity.oozie.parse.OozieAuditLogAppProvider.xml |  2 +-
 ....eagle.topology.TopologyCheckAppProvider.xml |  2 +-
 8 files changed, 83 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
index 2d68da8..f33797f 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/resources/META-INF/providers/org.apache.eagle.hadoop.queue.HadoopQueueRunningAppProvider.xml
@@ -57,7 +57,7 @@
         <property>
             <name>dataSinkConfig.HADOOP_QUEUE_STREAM.topic</name>
             <displayName>Destination(Kafka Topic) Of Queue Stream Data</displayName>
-            <value>yarn_queue</value>
+            <value>yarn_queue_${siteId}</value>
             <description>topic for kafka data sink</description>
         </property>
         <property>

http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-jpm-entity/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-entity/pom.xml b/eagle-jpm/eagle-jpm-entity/pom.xml
index 98982f9..9a38725 100644
--- a/eagle-jpm/eagle-jpm-entity/pom.xml
+++ b/eagle-jpm/eagle-jpm-entity/pom.xml
@@ -42,10 +42,9 @@
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>3.8.1</version>
+            <version>${junit.version}</version>
             <scope>test</scope>
         </dependency>
-
     </dependencies>
 
 </project>

http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java
b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java
index cfa50f9..f4f58d8 100644
--- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java
+++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobConfigSerDeser.java
@@ -19,45 +19,26 @@
 package org.apache.eagle.jpm.mr.historyentity;
 
 import org.apache.eagle.log.entity.meta.EntitySerDeser;
-import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.eagle.log.entity.meta.MapSerDeser;
 
 import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TreeMap;
+
 
 public class JobConfigSerDeser implements EntitySerDeser<JobConfig> {
 
+    private static final MapSerDeser INSTANCE = new MapSerDeser();
+
     @Override
     public JobConfig deserialize(byte[] bytes) {
-        JobConfig jc = new JobConfig();
-        Map<String, String> map = new TreeMap<String, String>();
-        jc.setConfig(map);
-        String sb = Bytes.toString(bytes);
-        String[] keyValue = sb.split(",");
-        for (String pair : keyValue) {
-            String[] str = pair.split(":");
-            if (pair.equals("") || str[0].equals("")) {
-                continue;
-            }
-            String key = str[0];
-            String value = "";
-            if (str.length == 2) {
-                value = str[1];
-            }
-            map.put(key, value);
-        }
-        return jc;
+        Map map = INSTANCE.deserialize(bytes);
+        JobConfig config = new JobConfig();
+        config.setConfig(map);
+        return config;
     }
-    
+
     @Override
-    public byte[] serialize(JobConfig conf) {
-        Map<String, String> map = conf.getConfig();
-        StringBuilder sb = new StringBuilder();
-        for (Entry<String, String> entry : map.entrySet()) {
-            sb.append(entry.getKey() + ":" + entry.getValue() + ",");
-        }
-        sb.deleteCharAt(sb.length() - 1);
-        return sb.toString().getBytes();
+    public byte[] serialize(JobConfig jobConfig) {
+        return INSTANCE.serialize(jobConfig.getConfig());
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-jpm-entity/src/test/java/org/apache/eagle/jpm/mr/JobConfigSerDeserTest.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-entity/src/test/java/org/apache/eagle/jpm/mr/JobConfigSerDeserTest.java
b/eagle-jpm/eagle-jpm-entity/src/test/java/org/apache/eagle/jpm/mr/JobConfigSerDeserTest.java
new file mode 100644
index 0000000..a4b5307
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-entity/src/test/java/org/apache/eagle/jpm/mr/JobConfigSerDeserTest.java
@@ -0,0 +1,65 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.eagle.jpm.mr;
+
+import org.apache.eagle.jpm.mr.historyentity.JobConfig;
+import org.apache.eagle.jpm.mr.historyentity.JobConfigSerDeser;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class JobConfigSerDeserTest {
+
+    @Test
+    public void test() {
+        String value = "INSERT OVERWRITE TABLE kylin_intermediate_KYLIN_HIVE_METRICS_QUERY_CUBE
SELECT\n" +
+                "HIVE_METRICS_QUERY_CUBE.`CUBE_NAME`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`SEGMENT_NAME`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`CUBOID_SOURCE`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`CUBOID_TARGET`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`IF_MATCH`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`IF_SUCCESS`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`KYEAR_BEGIN_DATE`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`KMONTH_BEGIN_DATE`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`KWEEK_BEGIN_DATE`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`KDAY_DATE`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`WEIGHT_PER_HIT`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`STORAGE_CALL_COUNT`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`STORAGE_CALL_TIME_SUM`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`STORAGE_CALL_TIME_MAX`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`STORAGE_COUNT_SKIP`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`STORAGE_SIZE_SCAN`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`STORAGE_SIZE_RETURN`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`STORAGE_SIZE_AGGREGATE_FILTER`\n" +
+                ",HIVE_METRICS_QUERY_CUBE.`STORAGE_SIZE_AGGREGATE`\n" +
+                "FROM KYLIN.HIVE_METRICS_QUERY_CUBE as HIVE_METRICS_QUERY_CUBE\n" +
+                "WHERE (((HIVE_METRICS_QUERY_CUBE.KDAY_DATE = '2017-04-06' AND HIVE_METRICS_QUERY_CUBE.KDAY_TIME
>= '18:00:00') OR (HIVE_METRICS_QUERY_CUBE.KDAY_DATE > '2017-04-06')) AND ((HIVE_METRICS_QUERY_CUBE.KDAY_DATE
= '2017-04-06' AND HIVE_METRICS_QUERY_CUBE.KDAY_TIME < '20:00:00') OR (HIVE_METRICS_QUERY_CUBE.KDAY_DATE
< '2017-04-06')))";
+
+        Map<String, String> conf = new HashMap<>();
+        conf.put("test1", value);
+        JobConfig source = new JobConfig();
+        source.setConfig(conf);
+        JobConfigSerDeser serDeser = new JobConfigSerDeser();
+        byte[] bytes = serDeser.serialize(source);
+        JobConfig target = serDeser.deserialize(bytes);
+
+        Assert.assertTrue(target.getConfig().size() == 1);
+    }
+}

http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
index fe3ab83..907e35a 100644
--- a/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
+++ b/eagle-jpm/eagle-jpm-mr-running/src/main/resources/META-INF/providers/org.apache.eagle.jpm.mr.running.MRRunningJobApplicationProvider.xml
@@ -104,7 +104,7 @@
         <property>
             <name>dataSinkConfig.ACCEPTED_APP_STREAM.topic</name>
             <displayName>Destination(Kafka Topic) Of App Stream Data</displayName>
-            <value>yarn_accepted_app_{site}</value>
+            <value>yarn_accepted_app_{siteId}</value>
             <description>topic for kafka data sink</description>
             <required>true</required>
         </property>

http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
index d48340c..d453cd6 100644
--- a/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
+++ b/eagle-security/eagle-security-hdfs-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.auditlog.HdfsAuditLogAppProvider.xml
@@ -130,14 +130,14 @@
         <property>
             <name>dataSinkConfig.HDFS_AUDIT_LOG_ENRICHED_STREAM.topic</name>
             <displayName>Kafka Topic for Auditlog Event Sink</displayName>
-            <value>hdfs_audit_event_${site}</value>
+            <value>hdfs_audit_event_${siteId}</value>
             <description>topic for kafka data sink</description>
             <required>true</required>
         </property>
         <property>
             <name>dataSinkConfig.HDFS_AUDIT_LOG_TRAFFIC_STREAM.topic</name>
             <displayName>Kafka Topic for Traffic Data Sink</displayName>
-            <value>hdfs_traffic_event_${site}</value>
+            <value>hdfs_traffic_event_${siteId}</value>
             <description>topic for kafka data sink</description>
         </property>
         <property>

http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml
b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml
index ee1715c..5434a9d 100644
--- a/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml
+++ b/eagle-security/eagle-security-oozie-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.oozie.parse.OozieAuditLogAppProvider.xml
@@ -69,7 +69,7 @@
         <property>
             <name>dataSourceConfig.topic</name>
             <displayName>Kafka Topic for Data Consumption</displayName>
-            <value>oozie_audit_log</value>
+            <value>oozie_audit_log_${siteId}</value>
             <description>data source topic</description>
         </property>
         <property>

http://git-wip-us.apache.org/repos/asf/eagle/blob/7c0d83f3/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
b/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
index 4bdb1c4..aa36c53 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
+++ b/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
@@ -165,7 +165,7 @@
         <property>
             <name>dataSinkConfig.topic</name>
             <displayName>Topic For Kafka Data Sink</displayName>
-            <value>topology_check</value>
+            <value>topology_check_${siteId}</value>
             <description>topic For kafka data sink</description>
         </property>
         <property>


Mime
View raw message