kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From billy...@apache.org
Subject kylin git commit: refactor
Date Tue, 27 Dec 2016 02:23:00 GMT
Repository: kylin
Updated Branches:
  refs/heads/hive-load-refactor 0b66ef1d6 -> 877b538a5


refactor


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/877b538a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/877b538a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/877b538a

Branch: refs/heads/hive-load-refactor
Commit: 877b538a508f60ab72d97eaf7af6e39d55a4b4bd
Parents: 0b66ef1
Author: Billy Liu <billyliu@apache.org>
Authored: Tue Dec 27 10:22:50 2016 +0800
Committer: Billy Liu <billyliu@apache.org>
Committed: Tue Dec 27 10:22:50 2016 +0800

----------------------------------------------------------------------
 .../kylin/rest/controller/TableController.java  | 76 ++++++++++----------
 .../apache/kylin/rest/service/TableService.java | 72 +++++++++++--------
 webapp/app/js/controllers/sourceMeta.js         | 16 ++---
 webapp/app/js/services/tables.js                |  1 -
 4 files changed, 86 insertions(+), 79 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index 0debad7..964cd4c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -24,12 +24,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.kylin.common.util.JsonUtil;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.rest.exception.InternalErrorException;
+import org.apache.kylin.rest.exception.NotFoundException;
 import org.apache.kylin.rest.request.CardinalityRequest;
 import org.apache.kylin.rest.request.HiveTableRequest;
-import org.apache.kylin.rest.request.StreamingRequest;
 import org.apache.kylin.rest.service.TableService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -66,14 +65,12 @@ public class TableController extends BasicController {
     @RequestMapping(value = "", method = { RequestMethod.GET })
     @ResponseBody
     public List<TableDesc> getTableDesc(@RequestParam(value = "ext", required = false)
boolean withExt, @RequestParam(value = "project", required = true) String project) throws
IOException {
-        List<TableDesc> tables = null;
         try {
-            tables = tableService.getTableDescByProject(project, withExt);
+            return tableService.getTableDescByProject(project, withExt);
         } catch (IOException e) {
             logger.error("Failed to get Hive Tables", e);
             throw new InternalErrorException(e.getLocalizedMessage());
         }
-        return tables;
     }
 
     /**
@@ -85,21 +82,28 @@ public class TableController extends BasicController {
     @RequestMapping(value = "/{tableName:.+}", method = { RequestMethod.GET })
     @ResponseBody
     public TableDesc getTableDesc(@PathVariable String tableName) {
-        return tableService.getTableDescByName(tableName);
+        TableDesc table = tableService.getTableDescByName(tableName, false);
+        if (table == null)
+            throw new NotFoundException("Could not find Hive table: " + tableName);
+        return table;
     }
 
     @RequestMapping(value = "/{tables}/{project}", method = { RequestMethod.POST })
     @ResponseBody
     public Map<String, String[]> loadHiveTables(@PathVariable String tables, @PathVariable
String project, @RequestBody HiveTableRequest request) throws IOException {
         String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
+        Map<String, String[]> result = new HashMap<String, String[]>();
         String[] tableNames = tables.split(",");
-        String[] loaded = tableService.loadHiveTablesToProject(tableNames, project);
-        if (request.isCalculate()) {
-            tableService.calculateCardinalityIfNotPresent(loaded, submitter);
+        try {
+            String[] loaded = tableService.loadHiveTablesToProject(tableNames, project);
+            result.put("result.loaded", loaded);
+            if (request.isCalculate()) {
+                tableService.calculateCardinalityIfNotPresent(loaded, submitter);
+            }
+        } catch (Exception e) {
+            logger.error("Failed to load Hive Table", e);
+            throw new InternalErrorException(e.getLocalizedMessage());
         }
-        Map<String, String[]> result = new HashMap<String, String[]>();
-        result.put("result.loaded", loaded);
-        result.put("result.unloaded", new String[] {});
         return result;
     }
 
@@ -109,29 +113,23 @@ public class TableController extends BasicController {
         Set<String> unLoadSuccess = Sets.newHashSet();
         Set<String> unLoadFail = Sets.newHashSet();
         Map<String, String[]> result = new HashMap<String, String[]>();
-        for (String tableName : tables.split(",")) {
-            if (tableService.unLoadHiveTable(tableName, project)) {
-                unLoadSuccess.add(tableName);
-            } else {
-                unLoadFail.add(tableName);
+        try {
+            for (String tableName : tables.split(",")) {
+                if (tableService.unLoadHiveTable(tableName, project)) {
+                    unLoadSuccess.add(tableName);
+                } else {
+                    unLoadFail.add(tableName);
+                }
             }
+        } catch (Exception e) {
+            logger.error("Failed to unload Hive Table", e);
+            throw new InternalErrorException(e.getLocalizedMessage());
         }
         result.put("result.unload.success", (String[]) unLoadSuccess.toArray(new String[unLoadSuccess.size()]));
         result.put("result.unload.fail", (String[]) unLoadFail.toArray(new String[unLoadFail.size()]));
         return result;
     }
 
-    @RequestMapping(value = "/addStreamingSrc", method = { RequestMethod.POST })
-    @ResponseBody
-    public Map<String, String> addStreamingTable(@RequestBody StreamingRequest request)
throws IOException {
-        Map<String, String> result = new HashMap<String, String>();
-        String project = request.getProject();
-        TableDesc desc = JsonUtil.readValue(request.getTableData(), TableDesc.class);
-        tableService.addStreamingTable(desc, project);
-        result.put("success", "true");
-        return result;
-    }
-
     /**
      * Regenerate table cardinality
      *
@@ -143,8 +141,13 @@ public class TableController extends BasicController {
     public CardinalityRequest generateCardinality(@PathVariable String tableNames, @RequestBody
CardinalityRequest request) throws IOException {
         String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
         String[] tables = tableNames.split(",");
-        for (String table : tables) {
-            tableService.calculateCardinality(table.trim().toUpperCase(), submitter);
+        try {
+            for (String table : tables) {
+                tableService.calculateCardinality(table.trim().toUpperCase(), submitter);
+            }
+        } catch (IOException e) {
+            logger.error("Failed to calculate cardinality", e);
+            throw new InternalErrorException(e.getLocalizedMessage());
         }
         return request;
     }
@@ -158,13 +161,11 @@ public class TableController extends BasicController {
     @RequestMapping(value = "/hive", method = { RequestMethod.GET })
     @ResponseBody
     private List<String> showHiveDatabases() throws IOException {
-        List<String> results = null;
         try {
-            results = tableService.getHiveDbNames();
+            return tableService.getHiveDbNames();
         } catch (Exception e) {
-            throw new IOException(e);
+            throw new InternalErrorException(e.getLocalizedMessage());
         }
-        return results;
     }
 
     /**
@@ -176,14 +177,11 @@ public class TableController extends BasicController {
     @RequestMapping(value = "/hive/{database}", method = { RequestMethod.GET })
     @ResponseBody
     private List<String> showHiveTables(@PathVariable String database) throws IOException
{
-        List<String> results = null;
-
         try {
-            results = tableService.getHiveTableNames(database);
+            return tableService.getHiveTableNames(database);
         } catch (Exception e) {
-            throw new IOException(e);
+            throw new InternalErrorException(e.getLocalizedMessage());
         }
-        return results;
     }
 
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index 6accfdc..8868bff 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -83,8 +83,12 @@ public class TableService extends BasicService {
         return tables;
     }
 
-    public TableDesc getTableDescByName(String tableName) {
-        return getMetadataManager().getTableDesc(tableName);
+    public TableDesc getTableDescByName(String tableName, boolean withExt) {
+        TableDesc table =  getMetadataManager().getTableDesc(tableName);
+        if(withExt){
+            table = cloneTableDesc(table);
+        }
+        return table;
     }
 
     @PreAuthorize(Constant.ACCESS_HAS_ROLE_MODELER + " or " + Constant.ACCESS_HAS_ROLE_ADMIN)
@@ -128,7 +132,7 @@ public class TableService extends BasicService {
         //remove streaming info
         String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
         tableName = dbTableName[0] + "." + dbTableName[1];
-        TableDesc desc = getTableDescByName(tableName);
+        TableDesc desc = getMetadataManager().getTableDesc(tableName);
         if (desc == null)
             return false;
         tableType = desc.getSourceType();
@@ -206,39 +210,45 @@ public class TableService extends BasicService {
         return results;
     }
 
+    private TableDescResponse cloneTableDesc(TableDesc table) {
+        TableExtDesc tableExtDesc = getMetadataManager().getTableExt(table.getIdentity());
+
+        // Clone TableDesc
+        TableDescResponse rtableDesc = new TableDescResponse(table);
+        Map<String, Long> cardinality = new HashMap<String, Long>();
+        Map<String, String> dataSourceProp = new HashMap<>();
+        String scard = tableExtDesc.getCardinality();
+        if (!StringUtils.isEmpty(scard)) {
+            String[] cards = StringUtils.split(scard, ",");
+            ColumnDesc[] cdescs = rtableDesc.getColumns();
+            for (int i = 0; i < cdescs.length; i++) {
+                ColumnDesc columnDesc = cdescs[i];
+                if (cards.length > i) {
+                    cardinality.put(columnDesc.getName(), Long.parseLong(cards[i]));
+                } else {
+                    logger.error("The result cardinality is not identical with hive table
metadata, cardinality : " + scard + " column array length: " + cdescs.length);
+                    break;
+                }
+            }
+            rtableDesc.setCardinality(cardinality);
+        }
+        dataSourceProp.putAll(tableExtDesc.getDataSourceProp());
+        dataSourceProp.put("location", tableExtDesc.getStorageLocation());
+        dataSourceProp.put("owner", tableExtDesc.getOwner());
+        dataSourceProp.put("last_access_time", tableExtDesc.getLastAccessTime());
+        dataSourceProp.put("partition_column", tableExtDesc.getPartitionColumn());
+        dataSourceProp.put("total_file_size", tableExtDesc.getTotalFileSize());
+        rtableDesc.setDescExd(dataSourceProp);
+        return rtableDesc;
+    }
+
+
     private List<TableDesc> cloneTableDesc(List<TableDesc> tables) throws IOException
{
         List<TableDesc> descs = new ArrayList<TableDesc>();
         Iterator<TableDesc> it = tables.iterator();
         while (it.hasNext()) {
             TableDesc table = it.next();
-            TableExtDesc tableExtDesc = getMetadataManager().getTableExt(table.getIdentity());
-
-            // Clone TableDesc
-            TableDescResponse rtableDesc = new TableDescResponse(table);
-            Map<String, Long> cardinality = new HashMap<String, Long>();
-            Map<String, String> dataSourceProp = new HashMap<>();
-            String scard = tableExtDesc.getCardinality();
-            if (!StringUtils.isEmpty(scard)) {
-                String[] cards = StringUtils.split(scard, ",");
-                ColumnDesc[] cdescs = rtableDesc.getColumns();
-                for (int i = 0; i < cdescs.length; i++) {
-                    ColumnDesc columnDesc = cdescs[i];
-                    if (cards.length > i) {
-                        cardinality.put(columnDesc.getName(), Long.parseLong(cards[i]));
-                    } else {
-                        logger.error("The result cardinality is not identical with hive table
metadata, cardinality : " + scard + " column array length: " + cdescs.length);
-                        break;
-                    }
-                }
-                rtableDesc.setCardinality(cardinality);
-            }
-            dataSourceProp.putAll(tableExtDesc.getDataSourceProp());
-            dataSourceProp.put("location", tableExtDesc.getStorageLocation());
-            dataSourceProp.put("owner", tableExtDesc.getOwner());
-            dataSourceProp.put("last_access_time", tableExtDesc.getLastAccessTime());
-            dataSourceProp.put("partition_column", tableExtDesc.getPartitionColumn());
-            dataSourceProp.put("total_file_size", tableExtDesc.getTotalFileSize());
-            rtableDesc.setDescExd(dataSourceProp);
+            TableDescResponse rtableDesc = cloneTableDesc(table);
             descs.add(rtableDesc);
         }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/webapp/app/js/controllers/sourceMeta.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/sourceMeta.js b/webapp/app/js/controllers/sourceMeta.js
index bbb9915..a53a35f 100755
--- a/webapp/app/js/controllers/sourceMeta.js
+++ b/webapp/app/js/controllers/sourceMeta.js
@@ -330,7 +330,7 @@ KylinApp
         }
 
         if ($scope.tableNames.trim() === "") {
-          SweetAlert.swal('', 'Please input table(s) you want to synchronize.', 'info');
+          SweetAlert.swal('', 'Please input table(s) you want to load.', 'info');
           return;
         }
 
@@ -352,13 +352,13 @@ KylinApp
           })
 
           if (result['result.unloaded'].length != 0 && result['result.loaded'].length
== 0) {
-            SweetAlert.swal('Failed!', 'Failed to synchronize following table(s): ' + unloadedTableInfo,
'error');
+            SweetAlert.swal('Failed!', 'Failed to load following table(s): ' + unloadedTableInfo,
'error');
           }
           if (result['result.loaded'].length != 0 && result['result.unloaded'].length
== 0) {
-            SweetAlert.swal('Success!', 'The following table(s) have been successfully synchronized:
' + loadTableInfo, 'success');
+            SweetAlert.swal('Success!', 'The following table(s) have been successfully loaded:
' + loadTableInfo, 'success');
           }
           if (result['result.loaded'].length != 0 && result['result.unloaded'].length
!= 0) {
-            SweetAlert.swal('Partial loaded!', 'The following table(s) have been successfully
synchronized: ' + loadTableInfo + "\n\n Failed to synchronize following table(s):" + unloadedTableInfo,
'warning');
+            SweetAlert.swal('Partial loaded!', 'The following table(s) have been successfully
loaded: ' + loadTableInfo + "\n\n Failed to load following table(s):" + unloadedTableInfo,
'warning');
           }
           loadingRequest.hide();
           scope.aceSrcTbLoaded(true);
@@ -378,7 +378,7 @@ KylinApp
 
     $scope.remove = function () {
         if ($scope.tableNames.trim() === "") {
-          SweetAlert.swal('', 'Please input table(s) you want to synchronize.', 'info');
+          SweetAlert.swal('', 'Please input table(s) you want to unload.', 'info');
           return;
         }
 
@@ -400,13 +400,13 @@ KylinApp
           })
 
           if (result['result.unload.fail'].length != 0 && result['result.unload.success'].length
== 0) {
-            SweetAlert.swal('Failed!', 'Failed to synchronize following table(s): ' + unRemovedTableInfo,
'error');
+            SweetAlert.swal('Failed!', 'Failed to unload following table(s): ' + unRemovedTableInfo,
'error');
           }
           if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length
== 0) {
-            SweetAlert.swal('Success!', 'The following table(s) have been successfully synchronized:
' + removedTableInfo, 'success');
+            SweetAlert.swal('Success!', 'The following table(s) have been successfully unloaded:
' + removedTableInfo, 'success');
           }
           if (result['result.unload.success'].length != 0 && result['result.unload.fail'].length
!= 0) {
-            SweetAlert.swal('Partial unloaded!', 'The following table(s) have been successfully
synchronized: ' + removedTableInfo + "\n\n Failed to synchronize following table(s):" + unRemovedTableInfo,
'warning');
+            SweetAlert.swal('Partial unloaded!', 'The following table(s) have been successfully
unloaded: ' + removedTableInfo + "\n\n Failed to unload following table(s):" + unRemovedTableInfo,
'warning');
           }
           loadingRequest.hide();
           scope.aceSrcTbLoaded(true);

http://git-wip-us.apache.org/repos/asf/kylin/blob/877b538a/webapp/app/js/services/tables.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/services/tables.js b/webapp/app/js/services/tables.js
index 4199d6c..4e7a7c4 100755
--- a/webapp/app/js/services/tables.js
+++ b/webapp/app/js/services/tables.js
@@ -24,7 +24,6 @@ KylinApp.factory('TableService', ['$resource', function ($resource, config)
{
     reload: {method: 'PUT', params: {action: 'reload'}, isArray: false},
     loadHiveTable: {method: 'POST', params: {}, isArray: false},
     unLoadHiveTable: {method: 'DELETE', params: {}, isArray: false},
-    addStreamingSrc: {method: 'POST', params: {action:'addStreamingSrc'}, isArray: false},
     genCardinality: {method: 'PUT', params: {action: 'cardinality'}, isArray: false},
     showHiveDatabases: {method: 'GET', params: {action:'hive'}, cache: true, isArray: true},
     showHiveTables: {method: 'GET', params: {action:'hive'}, cache: true, isArray: true}


Mime
View raw message