kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mahong...@apache.org
Subject kylin git commit: put ext filter into project L2 cache
Date Fri, 01 Apr 2016 07:48:24 GMT
Repository: kylin
Updated Branches:
  refs/heads/master c86de1a8e -> 0e662a350


put ext filter into project L2 cache


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/0e662a35
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/0e662a35
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/0e662a35

Branch: refs/heads/master
Commit: 0e662a350b88ad4b1194874af19f25590286a52a
Parents: c86de1a
Author: Hongbin Ma <mahongbin@apache.org>
Authored: Fri Apr 1 15:47:50 2016 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Fri Apr 1 15:47:50 2016 +0800

----------------------------------------------------------------------
 .../metadata/filter/UDF/MassInTupleFilter.java  |  1 +
 .../kylin/metadata/project/ProjectL2Cache.java  | 37 +++++++++++++++-----
 .../kylin/metadata/project/ProjectManager.java  |  9 +++++
 .../controller/ExternalFilterController.java    | 37 ++++++++++----------
 .../kylin/rest/service/ExtFilterService.java    | 21 -----------
 5 files changed, 58 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
index 0cb416c..5889c61 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/filter/UDF/MassInTupleFilter.java
@@ -32,6 +32,7 @@ import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.filter.function.Functions;
 import org.apache.kylin.metadata.model.ExternalFilterDesc;
 import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.metadata.tuple.IEvaluatableTuple;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
index e8004e1..3006773 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectL2Cache.java
@@ -25,6 +25,7 @@ import java.util.Set;
 
 import org.apache.kylin.metadata.MetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.ExternalFilterDesc;
 import org.apache.kylin.metadata.model.FunctionDesc;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.TableDesc;
@@ -59,6 +60,16 @@ class ProjectL2Cache {
         projectCaches.clear();
     }
 
+    public ExternalFilterDesc getExternalFilterDesc(String project, String extFilterName)
{
+        ProjectCache prjCache = getCache(project);
+        return prjCache.extFilters.get(extFilterName);
+    }
+
+    public Map<String, ExternalFilterDesc> listExternalFilterDesc(String project) {
+        ProjectCache prjCache = getCache(project);
+        return Collections.unmodifiableMap(prjCache.extFilters);
+    }
+
     public List<TableDesc> listDefinedTables(String project) {
         ProjectCache prjCache = getCache(project);
         List<TableDesc> result = Lists.newArrayListWithCapacity(prjCache.tables.size());
@@ -159,7 +170,7 @@ class ProjectL2Cache {
 
     private ProjectCache loadCache(String project) {
         logger.info("Loading L2 project cache for " + project);
-        ProjectCache result = new ProjectCache(project);
+        ProjectCache projectCache = new ProjectCache(project);
 
         ProjectInstance pi = mgr.getProject(project);
 
@@ -171,30 +182,39 @@ class ProjectL2Cache {
         for (String tableName : pi.getTables()) {
             TableDesc tableDesc = metaMgr.getTableDesc(tableName);
             if (tableDesc != null) {
-                result.tables.put(tableDesc.getIdentity(), new TableCache(tableDesc));
+                projectCache.tables.put(tableDesc.getIdentity(), new TableCache(tableDesc));
             } else {
                 logger.warn("Table '" + tableName + "' defined under project '" + project
+ "' is not found");
             }
         }
 
+        for (String extFilterName : pi.getExtFilters()) {
+            ExternalFilterDesc filterDesc = metaMgr.getExtFilterDesc(extFilterName);
+            if (filterDesc != null) {
+                projectCache.extFilters.put(extFilterName, filterDesc);
+            } else {
+                logger.warn("External Filter '" + extFilterName + "' defined under project
'" + project + "' is not found");
+            }
+        }
+
         RealizationRegistry registry = RealizationRegistry.getInstance(mgr.getConfig());
         for (RealizationEntry entry : pi.getRealizationEntries()) {
             IRealization realization = registry.getRealization(entry.getType(), entry.getRealization());
             if (realization != null) {
-                result.realizations.add(realization);
+                projectCache.realizations.add(realization);
             } else {
                 logger.warn("Realization '" + entry + "' defined under project '" + project
+ "' is not found");
             }
         }
 
-        for (IRealization realization : result.realizations) {
-            if (sanityCheck(result, realization)) {
-                mapTableToRealization(result, realization);
-                markExposedTablesAndColumns(result, realization);
+        for (IRealization realization : projectCache.realizations) {
+            if (sanityCheck(projectCache, realization)) {
+                mapTableToRealization(projectCache, realization);
+                markExposedTablesAndColumns(projectCache, realization);
             }
         }
 
-        return result;
+        return projectCache;
     }
 
     // check all columns reported by realization does exists
@@ -257,6 +277,7 @@ class ProjectL2Cache {
         private Map<String, TableCache> tables = Maps.newHashMap();
         private Set<TableDesc> exposedTables = Sets.newHashSet();
         private Set<IRealization> realizations = Sets.newHashSet();
+        private Map<String, ExternalFilterDesc> extFilters = Maps.newHashMap();
 
         ProjectCache(String project) {
             this.project = project;

http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
index 65b3719..b42b7e2 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/project/ProjectManager.java
@@ -21,6 +21,7 @@ package org.apache.kylin.metadata.project;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -352,6 +353,14 @@ public class ProjectManager {
         return projects;
     }
 
+    public ExternalFilterDesc getExternalFilterDesc(String project, String extFilter) {
+        return l2Cache.getExternalFilterDesc(project, extFilter);
+    }
+
+    public Map<String, ExternalFilterDesc> listExternalFilterDescs(String project)
{
+        return l2Cache.listExternalFilterDesc(project);
+    }
+
     public List<TableDesc> listDefinedTables(String project) throws IOException {
         return l2Cache.listDefinedTables(norm(project));
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
b/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
index b0b7569..5801eb5 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/ExternalFilterController.java
@@ -18,6 +18,12 @@
 
 package org.apache.kylin.rest.controller;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
 import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.metadata.model.ExternalFilterDesc;
 import org.apache.kylin.rest.request.ExternalFilterRequest;
@@ -26,13 +32,14 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.ResponseBody;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import com.google.common.collect.Lists;
 
 /**
  * @author jiazhong
@@ -45,8 +52,6 @@ public class ExternalFilterController extends BasicController {
     @Autowired
     private ExtFilterService extFilterService;
 
-
-
     @RequestMapping(value = "/saveExtFilter", method = { RequestMethod.POST })
     @ResponseBody
     public Map<String, String> saveExternalFilter(@RequestBody ExternalFilterRequest
request) throws IOException {
@@ -55,8 +60,8 @@ public class ExternalFilterController extends BasicController {
         ExternalFilterDesc desc = JsonUtil.readValue(request.getExtFilter(), ExternalFilterDesc.class);
         desc.setUuid(UUID.randomUUID().toString());
         extFilterService.saveExternalFilter(desc);
-        extFilterService.syncExtFilterToProject(new String[]{desc.getName()},filterProject);
-        result.put("success","true");
+        extFilterService.syncExtFilterToProject(new String[] { desc.getName() }, filterProject);
+        result.put("success", "true");
         return result;
     }
 
@@ -66,7 +71,7 @@ public class ExternalFilterController extends BasicController {
         Map<String, String> result = new HashMap();
         ExternalFilterDesc desc = JsonUtil.readValue(request.getExtFilter(), ExternalFilterDesc.class);
         extFilterService.updateExternalFilter(desc);
-        result.put("success","true");
+        result.put("success", "true");
         return result;
     }
 
@@ -74,13 +79,12 @@ public class ExternalFilterController extends BasicController {
     @ResponseBody
     public Map<String, String> unLoadHiveTables(@PathVariable String filter, @PathVariable
String project) throws IOException {
         Map<String, String> result = new HashMap<String, String>();
-        extFilterService.removeExtFilterFromProject(filter,project);
+        extFilterService.removeExtFilterFromProject(filter, project);
         extFilterService.removeExternalFilter(filter);
         result.put("success", "true");
         return result;
     }
 
-
     /**
      * Get available table list of the input database
      *
@@ -90,12 +94,9 @@ public class ExternalFilterController extends BasicController {
     @RequestMapping(value = "", method = { RequestMethod.GET })
     @ResponseBody
     public List<ExternalFilterDesc> getExternalFilters(@RequestParam(value = "project",
required = true) String project) throws IOException {
-        List<ExternalFilterDesc> filterDescs;
-        filterDescs = extFilterService.listProjectFilters(project);
+        List<ExternalFilterDesc> filterDescs = Lists.newArrayList();
+        filterDescs.addAll(extFilterService.getProjectManager().listExternalFilterDescs(project).values());
         return filterDescs;
     }
 
-
-
-
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/0e662a35/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java b/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
index f41996d..6577a12 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/ExtFilterService.java
@@ -33,11 +33,6 @@ import org.springframework.stereotype.Component;
 import java.io.IOException;
 import java.util.List;
 
-/**
- * Stateless & lightweight service facade of cube management functions.
- *
- * @author jiazhong
- */
 @Component("extFilterService")
 public class ExtFilterService extends BasicService {
     private static final Logger logger = LoggerFactory.getLogger(ExtFilterService.class);
@@ -47,22 +42,6 @@ public class ExtFilterService extends BasicService {
     private AccessService accessService;
 
 
-
-    public List<ExternalFilterDesc> listProjectFilters(String project) throws IOException
{
-        List<ExternalFilterDesc> filterDescs = Lists.newArrayList();
-        if(project == null){
-            return filterDescs;
-        }
-        ProjectInstance projectInstance = getProjectManager().getProject(project);
-        for(String filter:projectInstance.getExtFilters()){
-            ExternalFilterDesc filterDesc = getMetadataManager().getExtFilterDesc(filter);
-            if(filterDesc!=null){
-                filterDescs.add(filterDesc);
-            }
-        }
-        return filterDescs;
-    }
-
     @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
     public void saveExternalFilter(ExternalFilterDesc desc) throws IOException {
         if(getMetadataManager().getExtFilterDesc(desc.getName()) != null){


Mime
View raw message