ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alexantone...@apache.org
Subject [5/5] ambari git commit: AMBARI-10035. Hive View: Retrieve history from ATS (alexantonenko)
Date Thu, 12 Mar 2015 20:23:05 GMT
AMBARI-10035. Hive View: Retrieve history from ATS (alexantonenko)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8dbdbf66
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8dbdbf66
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8dbdbf66

Branch: refs/heads/branch-2.0.0
Commit: 8dbdbf66db2e598df8a1f9d13f23210744b93e86
Parents: a72dbc4
Author: Alex Antonenko <hiveww@gmail.com>
Authored: Thu Mar 12 22:22:02 2015 +0200
Committer: Alex Antonenko <hiveww@gmail.com>
Committed: Thu Mar 12 22:22:57 2015 +0200

----------------------------------------------------------------------
 .../assets/javascripts/modernizr-2.6.2.min.js   |   0
 contrib/views/files/src/main/resources/view.xml |  13 +-
 contrib/views/hive/pom.xml                      |   2 +-
 .../apache/ambari/view/hive/BaseService.java    |  20 +-
 .../apache/ambari/view/hive/HelpService.java    |   2 +-
 .../org/apache/ambari/view/hive/TestBean.java   |   6 +-
 .../ambari/view/hive/client/Connection.java     |  24 +-
 .../view/hive/client/ConnectionFactory.java     |  72 ++++
 .../ambari/view/hive/client/ConnectionPool.java |  81 ----
 .../view/hive/client/IConnectionFactory.java    |  24 ++
 .../view/hive/persistence/DataStoreStorage.java |  32 +-
 .../view/hive/persistence/IStorageFactory.java  |  23 ++
 .../view/hive/persistence/KeyValueStorage.java  |  15 +-
 .../ambari/view/hive/persistence/Storage.java   |   6 +-
 .../persistence/utils/FilteringStrategy.java    |   1 +
 .../view/hive/persistence/utils/Indexed.java    |   4 +-
 .../utils/OnlyOwnersFilteringStrategy.java      |   5 +
 .../hive/persistence/utils/StorageFactory.java  |  69 ++++
 .../hive/persistence/utils/StorageUtil.java     |  94 -----
 .../hive/resources/CRUDResourceManager.java     |  42 +-
 .../view/hive/resources/IResourceManager.java   |  37 ++
 .../resources/PersonalCRUDResourceManager.java  |  20 +-
 .../resources/SharedCRUDResourceManager.java    |  12 +-
 .../resources/browser/HiveBrowserService.java   |  24 +-
 .../view/hive/resources/files/FileService.java  |  47 ++-
 .../view/hive/resources/jobs/Aggregator.java    | 210 ++++++++++
 .../resources/jobs/ConnectionController.java    |  21 +-
 .../jobs/IOperationHandleResourceManager.java   |  36 ++
 .../ambari/view/hive/resources/jobs/Job.java    |  90 ----
 .../view/hive/resources/jobs/JobController.java |  46 ---
 .../resources/jobs/JobControllerFactory.java    |  43 --
 .../hive/resources/jobs/JobControllerImpl.java  | 326 ---------------
 .../view/hive/resources/jobs/JobImpl.java       | 203 ---------
 .../hive/resources/jobs/JobResourceManager.java | 107 -----
 .../resources/jobs/JobResourceProvider.java     |  12 +-
 .../view/hive/resources/jobs/JobService.java    |  37 +-
 .../view/hive/resources/jobs/LogParser.java     |  79 ++--
 .../jobs/OperationHandleController.java         |  22 +-
 .../jobs/OperationHandleControllerFactory.java  |  25 +-
 .../jobs/OperationHandleResourceManager.java    |  41 +-
 .../resources/jobs/StoredOperationHandle.java   |  12 +-
 .../hive/resources/jobs/atsJobs/ATSParser.java  | 139 +++++++
 .../jobs/atsJobs/ATSParserFactory.java          |  42 ++
 .../jobs/atsJobs/ATSRequestsDelegate.java       |  29 ++
 .../jobs/atsJobs/ATSRequestsDelegateImpl.java   |  86 ++++
 .../resources/jobs/atsJobs/HiveQueryId.java     |  37 ++
 .../hive/resources/jobs/atsJobs/IATSParser.java |  29 ++
 .../hive/resources/jobs/atsJobs/TezDagId.java   |  26 ++
 .../jobs/viewJobs/IJobControllerFactory.java    |  23 ++
 .../view/hive/resources/jobs/viewJobs/Job.java  |  99 +++++
 .../resources/jobs/viewJobs/JobController.java  |  46 +++
 .../jobs/viewJobs/JobControllerFactory.java     |  42 ++
 .../jobs/viewJobs/JobControllerImpl.java        | 343 ++++++++++++++++
 .../hive/resources/jobs/viewJobs/JobImpl.java   | 226 ++++++++++
 .../jobs/viewJobs/JobResourceManager.java       | 107 +++++
 .../resources/resources/FileResourceItem.java   |   6 +-
 .../resources/FileResourceResourceManager.java  |   9 +-
 .../resources/FileResourceResourceProvider.java |  10 +-
 .../resources/FileResourceService.java          |  11 +-
 .../hive/resources/savedQueries/SavedQuery.java |   6 +-
 .../savedQueries/SavedQueryResourceManager.java |  42 +-
 .../SavedQueryResourceProvider.java             |  18 +-
 .../savedQueries/SavedQueryService.java         |  10 +-
 .../ambari/view/hive/resources/udfs/UDF.java    |   6 +-
 .../hive/resources/udfs/UDFResourceManager.java |   9 +-
 .../resources/udfs/UDFResourceProvider.java     |  11 +-
 .../view/hive/resources/udfs/UDFService.java    |  15 +-
 .../ambari/view/hive/utils/FilePaginator.java   |   6 +-
 .../apache/ambari/view/hive/utils/HdfsApi.java  |  45 +-
 .../apache/ambari/view/hive/utils/HdfsUtil.java |  17 +-
 .../view/hive/utils/SharedObjectsFactory.java   | 163 ++++++++
 .../hive-web/app/components/typeahead-widget.js |   2 +-
 .../ui/hive-web/app/controllers/history.js      |   9 +-
 .../ui/hive-web/app/controllers/index.js        |  56 ++-
 .../app/controllers/index/history-query/logs.js |  12 +-
 .../controllers/index/history-query/results.js  |   5 +-
 .../ui/hive-web/app/controllers/job.js          |   5 +-
 .../ui/hive-web/app/controllers/open-queries.js |  13 +-
 .../ui/hive-web/app/controllers/queries.js      |   5 +-
 .../ui/hive-web/app/controllers/settings.js     | 129 +++---
 .../ui/hive-web/app/controllers/tez-ui.js       |  22 +
 .../hive-web/app/controllers/visual-explain.js  |  22 +
 .../ui/hive-web/app/helpers/all-uppercase.js    |  25 ++
 .../ui/hive-web/app/initializers/i18n.js        |   7 +-
 .../resources/ui/hive-web/app/models/job.js     |  14 +-
 .../ui/hive-web/app/routes/application.js       |   4 +-
 .../app/routes/index/history-query/index.js     |  13 +-
 .../resources/ui/hive-web/app/styles/app.scss   | 100 +++--
 .../templates/components/_typeahead-widget.hbs  |  17 +
 .../app/templates/components/popover-widget.hbs |   2 +-
 .../app/templates/databases-search-results.hbs  |   2 +-
 .../ui/hive-web/app/templates/databases.hbs     |   4 +-
 .../ui/hive-web/app/templates/history.hbs       |   4 +-
 .../ui/hive-web/app/templates/index.hbs         | 124 +++---
 .../ui/hive-web/app/templates/queries.hbs       |  58 +--
 .../ui/hive-web/app/templates/settings.hbs      |  63 +--
 .../ui/hive-web/app/templates/tez-ui.hbs        |  22 +
 .../hive-web/app/templates/visual-explain.hbs   |  22 +
 .../ui/hive-web/app/utils/constants.js          |  87 +++-
 .../ui/hive-web/app/utils/functions.js          |  19 +-
 .../resources/ui/hive-web/app/views/tez-ui.js   |  35 ++
 .../ui/hive-web/app/views/visual-explain.js     |  35 ++
 .../src/main/resources/ui/hive-web/bower.json   |   8 +-
 .../src/main/resources/ui/hive-web/package.json |  12 +-
 .../tests/integration/query-editor-test.js      |   2 +-
 .../tests/unit/controllers/history-test.js      |   4 +-
 .../tests/unit/controllers/queries-test.js      |  15 -
 .../tests/unit/controllers/settings-test.js     |  97 +++++
 .../tests/unit/controllers/tez-ui-test.js       |  33 ++
 .../unit/controllers/visual-explain-test.js     |  33 ++
 .../tests/unit/views/visual-explain-test.js     |  30 ++
 contrib/views/hive/src/main/resources/view.xml  |  52 ++-
 .../hive/resources/files/FileServiceTest.java   |   7 +-
 .../view/hive/resources/jobs/ATSParserTest.java | 411 +++++++++++++++++++
 .../hive/resources/jobs/AggregatorTest.java     | 396 ++++++++++++++++++
 .../hive/resources/jobs/JobServiceTest.java     |  16 +-
 .../view/hive/resources/jobs/LogParserTest.java |   8 +-
 .../resources/FileResourceServiceTest.java      |   4 +-
 .../savedQueries/SavedQueryServiceTest.java     |  16 +-
 .../hive/resources/udfs/UDFServiceTest.java     |   4 +-
 120 files changed, 4083 insertions(+), 1645 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/files/src/main/resources/ui/app/assets/javascripts/modernizr-2.6.2.min.js
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/main/resources/ui/app/assets/javascripts/modernizr-2.6.2.min.js b/contrib/views/files/src/main/resources/ui/app/assets/javascripts/modernizr-2.6.2.min.js
old mode 100644
new mode 100755

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/files/src/main/resources/view.xml
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/main/resources/view.xml b/contrib/views/files/src/main/resources/view.xml
index 206128d..eaf1a02 100644
--- a/contrib/views/files/src/main/resources/view.xml
+++ b/contrib/views/files/src/main/resources/view.xml
@@ -19,19 +19,26 @@
     <label>Files</label>
     <version>0.1.0</version>
 
+    <min-ambari-version>1.7.*</min-ambari-version>
+
     <parameter>
         <name>webhdfs.url</name>
-        <description>WebHDFS FileSystem URI (example: webhdfs://namenode:50070)</description>
+        <description>Enter the WebHDFS FileSystem URI. Typically this is the dfs.namenode.http-address property in the hdfs-site.xml configuration. URL must be accessible from Ambari Server.</description>
+        <label>WebHDFS FileSystem URI</label>
+        <placeholder>webhdfs://namenode:50070</placeholder>
         <required>true</required>
     </parameter>
     <parameter>
         <name>webhdfs.username</name>
-        <description>doAs for proxy user for HDFS</description>
+        <description>doAs for proxy user for HDFS. By default, uses the currently logged-in Ambari user.</description>
+        <label>WebHDFS Username</label>
         <required>false</required>
     </parameter>
     <parameter>
         <name>webhdfs.auth</name>
-        <description>Semicolon-separated authentication configs. Default: auth=SIMPLE</description>
+        <description>Semicolon-separated authentication configs.</description>
+        <placeholder>auth=SIMPLE</placeholder>
+        <default-value>auth=SIMPLE</default-value>
         <required>false</required>
     </parameter>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/views/hive/pom.xml b/contrib/views/hive/pom.xml
index 0ed6af4..e381719 100644
--- a/contrib/views/hive/pom.xml
+++ b/contrib/views/hive/pom.xml
@@ -195,7 +195,7 @@
         <artifactId>frontend-maven-plugin</artifactId>
         <version>0.0.14</version>
         <configuration>
-          <nodeVersion>v0.10.26</nodeVersion>
+          <nodeVersion>v0.10.32</nodeVersion>
           <npmVersion>1.4.3</npmVersion>
           <workingDirectory>src/main/resources/ui/hive-web/</workingDirectory>
         </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/BaseService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/BaseService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/BaseService.java
index e28193d..a963975 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/BaseService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/BaseService.java
@@ -20,9 +20,8 @@ package org.apache.ambari.view.hive;
 
 import com.google.inject.Inject;
 import org.apache.ambari.view.ViewContext;
-import org.apache.ambari.view.hive.persistence.Storage;
-import org.apache.ambari.view.hive.persistence.utils.StorageUtil;
 import org.apache.ambari.view.hive.utils.HdfsApi;
+import org.apache.ambari.view.hive.utils.SharedObjectsFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -37,19 +36,16 @@ public class BaseService {
   protected final static Logger LOG =
       LoggerFactory.getLogger(BaseService.class);
 
-  private Storage storage;
-  protected Storage getStorage() {
-    if (storage == null) {
-      storage = StorageUtil.getInstance(context).getStorage();
+  private SharedObjectsFactory sharedObjectsFactory;
+  public SharedObjectsFactory getSharedObjectsFactory() {
+    if (sharedObjectsFactory == null) {
+      sharedObjectsFactory = new SharedObjectsFactory(context);
     }
-    return storage;
+    return sharedObjectsFactory;
   }
 
-  private HdfsApi hdfsApi = null;
-  protected HdfsApi getHdfsApi()  {
-    if (hdfsApi == null)
-      hdfsApi = HdfsApi.getInstance(context);
-    return hdfsApi;
+  public void setSharedObjectsFactory(SharedObjectsFactory sharedObjectsFactory) {
+    this.sharedObjectsFactory = sharedObjectsFactory;
   }
 
   public BaseService() {

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/HelpService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/HelpService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/HelpService.java
index f975393..870e31a 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/HelpService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/HelpService.java
@@ -66,7 +66,7 @@ public class HelpService extends BaseService {
   public Response testStorage(){
     TestBean test = new TestBean();
     test.someData = "hello world";
-    getStorage().store(TestBean.class, test);
+    getSharedObjectsFactory().getStorage().store(TestBean.class, test);
     return Response.ok("OK").build();
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/TestBean.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/TestBean.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/TestBean.java
index fce0177..d298931 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/TestBean.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/TestBean.java
@@ -22,15 +22,15 @@ import org.apache.ambari.view.hive.persistence.utils.Indexed;
 
 public class TestBean implements Indexed {
   public String someData;
-  public Integer id;
+  public String id;
 
   @Override
-  public Integer getId() {
+  public String getId() {
     return id;
   }
 
   @Override
-  public void setId(Integer id) {
+  public void setId(String id) {
     this.id = id;
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java
index e713aba..44c90ae 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/Connection.java
@@ -18,7 +18,9 @@
 
 package org.apache.ambari.view.hive.client;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.auth.KerberosSaslHelper;
 import org.apache.hive.service.auth.PlainSaslHelper;
@@ -54,11 +56,13 @@ public class Connection {
   private TTransport transport;
 
   private DDLDelegator ddl;
+  private String username;
 
-  public Connection(String host, int port, Map<String, String> authParams) throws HiveClientException {
+  public Connection(String host, int port, Map<String, String> authParams, String username) throws HiveClientException {
     this.host = host;
     this.port = port;
     this.authParams = authParams;
+    this.username = username;
 
     openConnection();
     ddl = new DDLDelegator(this);
@@ -78,11 +82,11 @@ public class Connection {
           + host + ":" + port + ": " + e.toString(), e);
     }
     LOG.info("Hive connection opened");
-    openSession();
   }
 
   /**
    * Based on JDBC implementation of HiveConnection.createBinaryTransport
+   *
    * @return transport
    * @throws HiveClientException
    */
@@ -107,6 +111,11 @@ public class Connection {
           }
           saslProps.put(Sasl.QOP, saslQOP.toString());
           saslProps.put(Sasl.SERVER_AUTH, "true");
+
+          Configuration conf = new Configuration();
+          conf.set("hadoop.security.authentication", "kerberos");
+          UserGroupInformation.setConfiguration(conf);
+
           transport = KerberosSaslHelper.getKerberosTransport(
               authParams.get(Utils.HiveAuthenticationParams.AUTH_PRINCIPAL), host,
               HiveAuthFactory.getSocketTransport(host, port, 10000), saslProps,
@@ -119,7 +128,7 @@ public class Connection {
                 host, HiveAuthFactory.getSocketTransport(host, port, 10000), saslProps);
           } else {
             // we are using PLAIN Sasl connection with user/password
-            String userName = getAuthParamDefault(Utils.HiveAuthenticationParams.AUTH_USER, Utils.HiveAuthenticationParams.ANONYMOUS_USER);
+            String userName = getAuthParamDefault(Utils.HiveAuthenticationParams.AUTH_USER, getUsername());
             String passwd = getAuthParamDefault(Utils.HiveAuthenticationParams.AUTH_PASSWD, Utils.HiveAuthenticationParams.ANONYMOUS_USER);
             // Note: Thrift returns an SSL socket that is already bound to the specified host:port
             // Therefore an open called on this would be a no-op later
@@ -250,6 +259,7 @@ public class Connection {
         public TExecuteStatementResp body() throws HiveClientException {
 
           TExecuteStatementReq execReq = null;
+          openSession();
           execReq = new TExecuteStatementReq(getSessHandle(), oneCmd);
           execReq.setRunAsync(async);
           execReq.setConfOverlay(new HashMap<String, String>()); //maybe it's hive configuration? use it, Luke!
@@ -398,4 +408,12 @@ public class Connection {
   public void setAuthParams(Map<String, String> authParams) {
     this.authParams = authParams;
   }
+
+  public String getUsername() {
+    return username;
+  }
+
+  public void setUsername(String username) {
+    this.username = username;
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java
new file mode 100644
index 0000000..6886f57
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionFactory.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.client;
+
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.hive.utils.ServiceFormattedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ConnectionFactory implements IConnectionFactory {
+  private final static Logger LOG =
+      LoggerFactory.getLogger(ConnectionFactory.class);
+  private ViewContext context;
+
+  public ConnectionFactory(ViewContext context) {
+    this.context = context;
+  }
+
+  @Override
+  public Connection getHiveConnection() {
+    try {
+      return new Connection(getHiveHost(), Integer.valueOf(getHivePort()),
+          getHiveAuthParams(), context.getUsername());
+    } catch (HiveClientException e) {
+      throw new ServiceFormattedException("Couldn't open connection to Hive: " + e.toString(), e);
+    }
+  }
+
+  private String getHiveHost() {
+    return context.getProperties().get("hive.host");
+  }
+
+  private String getHivePort() {
+    return context.getProperties().get("hive.port");
+  }
+
+  private Map<String, String> getHiveAuthParams() {
+    String auth = context.getProperties().get("hive.auth");
+    Map<String, String> params = new HashMap<String, String>();
+    if (auth == null || auth.isEmpty()) {
+      auth = "auth=NOSASL";
+    }
+    for(String param : auth.split(";")) {
+      String[] keyvalue = param.split("=");
+      if (keyvalue.length != 2) {
+        LOG.error("Can not parse authentication param " + param + " in " + auth);
+        continue;
+      }
+      params.put(keyvalue[0], keyvalue[1]);
+    }
+    return params;
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionPool.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionPool.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionPool.java
deleted file mode 100644
index 6d07067..0000000
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/ConnectionPool.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.view.hive.client;
-
-import org.apache.ambari.view.ViewContext;
-import org.apache.ambari.view.hive.utils.ServiceFormattedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class ConnectionPool {
-  private final static Logger LOG =
-      LoggerFactory.getLogger(ConnectionPool.class);
-
-  private static Map<String, Connection> viewSingletonObjects = new HashMap<String, Connection>();
-  /**
-   * Returns HdfsApi object specific to instance
-   * @param context View Context instance
-   * @return Hdfs business delegate object
-   */
-  public static Connection getConnection(ViewContext context) {
-    if (!viewSingletonObjects.containsKey(context.getInstanceName()))
-      viewSingletonObjects.put(context.getInstanceName(), connectToHive(context));
-    return viewSingletonObjects.get(context.getInstanceName());
-  }
-
-  private static Connection connectToHive(ViewContext context) {
-    try {
-      return new Connection(getHiveHost(context), Integer.valueOf(getHivePort(context)), getHiveAuthParams(context));
-    } catch (HiveClientException e) {
-      throw new ServiceFormattedException("Couldn't open connection to Hive: " + e.toString(), e);
-    }
-  }
-
-  public static void setInstance(ViewContext context, Connection api) {
-    viewSingletonObjects.put(context.getInstanceName(), api);
-  }
-
-  private static String getHiveHost(ViewContext context) {
-    return context.getProperties().get("hive.host");
-  }
-
-  private static String getHivePort(ViewContext context) {
-    return context.getProperties().get("hive.port");
-  }
-
-  private static Map<String, String> getHiveAuthParams(ViewContext context) {
-    String auth = context.getProperties().get("hive.auth");
-    Map<String, String> params = new HashMap<String, String>();
-    if (auth == null || auth.isEmpty()) {
-      auth = "auth=NOSASL";
-    }
-    for(String param : auth.split(";")) {
-      String[] keyvalue = param.split("=");
-      if (keyvalue.length != 2) {
-        LOG.error("Can not parse authentication param " + param + " in " + auth);
-        continue;
-      }
-      params.put(keyvalue[0], keyvalue[1]);
-    }
-    return params;
-  }
-}

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/IConnectionFactory.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/IConnectionFactory.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/IConnectionFactory.java
new file mode 100644
index 0000000..7a3df06
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/client/IConnectionFactory.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.client;
+
+
+public interface IConnectionFactory {
+  Connection getHiveConnection();
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/DataStoreStorage.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/DataStoreStorage.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/DataStoreStorage.java
index 7a4ff40..b4bc415 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/DataStoreStorage.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/DataStoreStorage.java
@@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory;
 
 import javax.ws.rs.WebApplicationException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 
@@ -55,7 +54,7 @@ public class DataStoreStorage implements Storage {
   public synchronized void store(Class model, Indexed obj) {
     try {
       if (obj.getId() == null) {
-        int id = nextIdForEntity(context, model);
+        String id = nextIdForEntity(context, model);
         obj.setId(id);
       }
       context.getDataStore().store(obj);
@@ -64,7 +63,7 @@ public class DataStoreStorage implements Storage {
     }
   }
 
-  private static synchronized int nextIdForEntity(ViewContext context, Class aClass) {
+  private static synchronized String nextIdForEntity(ViewContext context, Class aClass) {
     // auto increment id implementation
     String lastId = context.getInstanceData(aClass.getName());
     int newId;
@@ -74,12 +73,12 @@ public class DataStoreStorage implements Storage {
       newId = Integer.parseInt(lastId) + 1;
     }
     context.putInstanceData(aClass.getName(), String.valueOf(newId));
-    return newId;
+    return String.valueOf(newId);
   }
 
   @Override
-  public synchronized <T extends Indexed> T load(Class<T> model, Integer id) throws ItemNotFound {
-    LOG.debug(String.format("Loading %s #%d", model.getName(), id));
+  public synchronized <T extends Indexed> T load(Class<T> model, Object id) throws ItemNotFound {
+    LOG.debug(String.format("Loading %s #%s", model.getName(), id));
     try {
       T obj = context.getDataStore().find(model, id);
       if (obj != null) {
@@ -97,10 +96,9 @@ public class DataStoreStorage implements Storage {
     LinkedList<T> list = new LinkedList<T>();
     LOG.debug(String.format("Loading all %s-s", model.getName()));
     try {
-      for(T item: context.getDataStore().findAll(model, null)) {
-        if ((filter == null) || filter.isConform(item)) {
-          list.add(item);
-        }
+      //TODO: use WHERE statement instead of this ugly filter
+      for(T item: context.getDataStore().findAll(model, filter.whereStatement())) {
+        list.add(item);
       }
     } catch (PersistenceException e) {
       throw new ServiceFormattedException("Error while finding all objects in DataStorage", e);
@@ -124,8 +122,8 @@ public class DataStoreStorage implements Storage {
   }
 
   @Override
-  public synchronized void delete(Class model, int id) throws ItemNotFound {
-    LOG.debug(String.format("Deleting %s:%d", model.getName(), id));
+  public synchronized void delete(Class model, Object id) throws ItemNotFound {
+    LOG.debug(String.format("Deleting %s:%s", model.getName(), id));
     Object obj = load(model, id);
     try {
       context.getDataStore().remove(obj);
@@ -135,7 +133,7 @@ public class DataStoreStorage implements Storage {
   }
 
   @Override
-  public boolean exists(Class model, Integer id) {
+  public boolean exists(Class model, Object id) {
     try {
       return context.getDataStore().find(model, id) != null;
     } catch (PersistenceException e) {
@@ -151,7 +149,7 @@ public class DataStoreStorage implements Storage {
       storage.store(SmokeTestEntity.class, entity);
 
       if (entity.getId() == null) throw new ServiceFormattedException("Ambari Views instance data DB doesn't work properly (auto increment id doesn't work)", null);
-      Integer id = entity.getId();
+      Object id = entity.getId();
       SmokeTestEntity entity2 = storage.load(SmokeTestEntity.class, id);
       boolean status = entity2.getData().compareTo("42") == 0;
       storage.delete(SmokeTestEntity.class, id);
@@ -164,14 +162,14 @@ public class DataStoreStorage implements Storage {
   }
 
   public static class SmokeTestEntity implements Indexed {
-    private Integer id = null;
+    private String id = null;
     private String data = null;
 
-    public Integer getId() {
+    public String getId() {
       return id;
     }
 
-    public void setId(Integer id) {
+    public void setId(String id) {
       this.id = id;
     }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/IStorageFactory.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/IStorageFactory.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/IStorageFactory.java
new file mode 100644
index 0000000..298d4c8
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/IStorageFactory.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.persistence;
+
+public interface IStorageFactory {
+  Storage getStorage();
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/KeyValueStorage.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/KeyValueStorage.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/KeyValueStorage.java
index e8a904c..6e88063 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/KeyValueStorage.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/KeyValueStorage.java
@@ -62,7 +62,7 @@ public abstract class KeyValueStorage implements Storage {
       int lastIndex = getConfig().getInt(modelIndexingPropName, 0);
       lastIndex ++;
       getConfig().setProperty(modelIndexingPropName, lastIndex);
-      obj.setId(lastIndex);
+      obj.setId(String.valueOf(lastIndex));
     }
 
     String modelPropName = getItemPropertyName(model, obj.getId());
@@ -71,12 +71,13 @@ public abstract class KeyValueStorage implements Storage {
   }
 
   @Override
-  public <T extends Indexed> T load(Class<T> model, Integer id) throws ItemNotFound {
+  public <T extends Indexed> T load(Class<T> model, Object id) throws ItemNotFound {
     String modelPropName = getItemPropertyName(model, id);
     LOG.debug(String.format("Loading %s", modelPropName));
     if (getConfig().containsKey(modelPropName)) {
       String json = read(modelPropName);
       LOG.debug(String.format("json: %s", json));
+
       return deserialize(model, json);
     } else {
       throw new ItemNotFound();
@@ -141,14 +142,14 @@ public abstract class KeyValueStorage implements Storage {
   }
 
   @Override
-  public synchronized void delete(Class model, int id) {
-    LOG.debug(String.format("Deleting %s:%d", model.getName(), id));
+  public synchronized void delete(Class model, Object id) {
+    LOG.debug(String.format("Deleting %s:%s", model.getName(), id));
     String modelPropName = getItemPropertyName(model, id);
     clear(modelPropName);
   }
 
   @Override
-  public boolean exists(Class model, Integer id) {
+  public boolean exists(Class model, Object id) {
     return getConfig().containsKey(getItemPropertyName(model, id));
   }
 
@@ -156,7 +157,7 @@ public abstract class KeyValueStorage implements Storage {
     return String.format("%s:index", model.getName());
   }
 
-  private String getItemPropertyName(Class model, int id) {
-    return String.format("%s.%d", model.getName(), id);
+  private String getItemPropertyName(Class model, Object id) {
+    return String.format("%s.%s", model.getName(), id);
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/Storage.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/Storage.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/Storage.java
index fb75670..188282e 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/Storage.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/Storage.java
@@ -41,7 +41,7 @@ public interface Storage {
    * @return bean instance
    * @throws ItemNotFound thrown if item with id was not found in DB
    */
-  <T extends Indexed> T load(Class<T> model, Integer id) throws ItemNotFound;
+  <T extends Indexed> T load(Class<T> model, Object id) throws ItemNotFound;
 
   /**
    * Load all objects of given bean class
@@ -74,7 +74,7 @@ public interface Storage {
    * @param model bean class
    * @param id identifier
    */
-  void delete(Class model, int id) throws ItemNotFound;
+  void delete(Class model, Object id) throws ItemNotFound;
 
   /**
    * Check is object exists
@@ -82,5 +82,5 @@ public interface Storage {
    * @param id identifier
    * @return true if exists
    */
-  boolean exists(Class model, Integer id);
+  boolean exists(Class model, Object id);
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/FilteringStrategy.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/FilteringStrategy.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/FilteringStrategy.java
index 79aa2f9..eba572e 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/FilteringStrategy.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/FilteringStrategy.java
@@ -28,4 +28,5 @@ public interface FilteringStrategy {
    * @return true if item conforms this filter
    */
   boolean isConform(Indexed item);
+  String whereStatement();
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/Indexed.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/Indexed.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/Indexed.java
index 1d80920..82b7d57 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/Indexed.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/Indexed.java
@@ -26,11 +26,11 @@ public interface Indexed {
    * Get the ID
    * @return ID
    */
-  Integer getId();
+  String getId();
 
   /**
    * Set ID
    * @param id ID
    */
-  void setId(Integer id);
+  void setId(String id);
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/OnlyOwnersFilteringStrategy.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/OnlyOwnersFilteringStrategy.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/OnlyOwnersFilteringStrategy.java
index 9f2b4a2..620f440 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/OnlyOwnersFilteringStrategy.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/OnlyOwnersFilteringStrategy.java
@@ -30,4 +30,9 @@ public class OnlyOwnersFilteringStrategy implements FilteringStrategy {
     Owned object = (Owned) item;
     return object.getOwner().compareTo(username) == 0;
   }
+
+  @Override
+  public String whereStatement() {
+    return "owner = '" + username + "'";
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageFactory.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageFactory.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageFactory.java
new file mode 100644
index 0000000..88a6d66
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageFactory.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.persistence.utils;
+
+import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.hive.persistence.DataStoreStorage;
+import org.apache.ambari.view.hive.persistence.IStorageFactory;
+import org.apache.ambari.view.hive.persistence.LocalKeyValueStorage;
+import org.apache.ambari.view.hive.persistence.Storage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Storage factory, creates storage of Local or Persistence API type.
+ * Type depends on context configuration: if "dataworker.storagePath" is set,
+ * storage of Local type will be created.  Otherwise, Persistence API will be used.
+ *
+ * Storage is singleton.
+ */
+public class StorageFactory implements IStorageFactory {
+  protected final static Logger LOG =
+      LoggerFactory.getLogger(StorageFactory.class);
+
+  private ViewContext context;
+
+  /**
+   * Constructor of storage factory
+   * @param context View Context instance
+   */
+  public StorageFactory(ViewContext context) {
+    this.context = context;
+  }
+
+  /**
+   * Creates storage instance
+   * @return storage instance
+   */
+  public Storage getStorage() {
+    String fileName = context.getProperties().get("dataworker.storagePath");
+
+    Storage storageInstance;
+    if (fileName != null) {
+      LOG.debug("Using local storage in " + fileName + " to store data");
+      // If specifed, use LocalKeyValueStorage - key-value file based storage
+      storageInstance = new LocalKeyValueStorage(context);
+    } else {
+      LOG.debug("Using Persistence API to store data");
+      // If not specifed, use ambari-views Persistence API
+      storageInstance = new DataStoreStorage(context);
+    }
+    return storageInstance;
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageUtil.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageUtil.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageUtil.java
deleted file mode 100644
index 1e67c02..0000000
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/persistence/utils/StorageUtil.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.view.hive.persistence.utils;
-
-import org.apache.ambari.view.ViewContext;
-import org.apache.ambari.view.hive.persistence.DataStoreStorage;
-import org.apache.ambari.view.hive.persistence.LocalKeyValueStorage;
-import org.apache.ambari.view.hive.persistence.Storage;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Storage factory, creates storage of Local or Persistence API type.
- * Type depends on context configuration: if "dataworker.storagePath" is set,
- * storage of Local type will be created.  Otherwise, Persistence API will be used.
- *
- * Storage is singleton.
- */
-public class StorageUtil {
-  private Storage storageInstance = null;
-
-  protected final static Logger LOG =
-      LoggerFactory.getLogger(StorageUtil.class);
-
-
-  private static Map<String, StorageUtil> viewSingletonObjects = new HashMap<String, StorageUtil>();
-  public static StorageUtil getInstance(ViewContext context) {
-    if (!viewSingletonObjects.containsKey(context.getInstanceName()))
-      viewSingletonObjects.put(context.getInstanceName(), new StorageUtil(context));
-    return viewSingletonObjects.get(context.getInstanceName());
-  }
-
-  public static void dropAllConnections() {
-    viewSingletonObjects.clear();
-  }
-
-  private ViewContext context;
-
-  /**
-   * Constructor of storage util
-   * @param context View Context instance
-   */
-  public StorageUtil(ViewContext context) {
-    this.context = context;
-  }
-
-  /**
-   * Get storage instance. If one is not created, creates instance.
-   * @return storage instance
-   */
-  public synchronized Storage getStorage() {
-    if (storageInstance == null) {
-      String fileName = context.getProperties().get("dataworker.storagePath");
-      if (fileName != null) {
-        LOG.debug("Using local storage in " + fileName + " to store data");
-        // If specifed, use LocalKeyValueStorage - key-value file based storage
-        storageInstance = new LocalKeyValueStorage(context);
-      } else {
-        LOG.debug("Using Persistence API to store data");
-        // If not specifed, use ambari-views Persistence API
-        storageInstance = new DataStoreStorage(context);
-      }
-    }
-    return storageInstance;
-  }
-
-  /**
-   * Set storage to use across all application.
-   * Used in unit tests.
-   * @param storage storage instance
-   */
-  public void setStorage(Storage storage) {
-    storageInstance = storage;
-  }
-}

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java
index b8dec17..28cf4d1 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/CRUDResourceManager.java
@@ -18,12 +18,11 @@
 
 package org.apache.ambari.view.hive.resources;
 
-import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.hive.persistence.IStorageFactory;
 import org.apache.ambari.view.hive.persistence.Storage;
 import org.apache.ambari.view.hive.persistence.utils.FilteringStrategy;
 import org.apache.ambari.view.hive.persistence.utils.Indexed;
 import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
-import org.apache.ambari.view.hive.persistence.utils.StorageUtil;
 import org.apache.ambari.view.hive.utils.ServiceFormattedException;
 
 import java.util.List;
@@ -32,18 +31,20 @@ import java.util.List;
  * CRUD resource manager
  * @param <T> Data type with ID
  */
-abstract public class CRUDResourceManager<T extends Indexed> {
+abstract public class CRUDResourceManager<T extends Indexed> implements IResourceManager<T> {
   //TODO: refactor: generic parameter gets Fabric for Indexed objects, not objects itself
   private Storage storage = null;
 
   protected final Class<? extends T> resourceClass;
+  protected IStorageFactory storageFabric;
 
   /**
    * Constructor
    * @param resourceClass model class
    */
-  public CRUDResourceManager(Class<? extends T> resourceClass) {
+  public CRUDResourceManager(Class<? extends T> resourceClass, IStorageFactory storageFabric) {
     this.resourceClass = resourceClass;
+    this.storageFabric = storageFabric;
   }
   // CRUD operations
 
@@ -52,7 +53,8 @@ abstract public class CRUDResourceManager<T extends Indexed> {
    * @param object object
    * @return model object
    */
-  protected T create(T object) {
+  @Override
+  public T create(T object) {
     object.setId(null);
     return this.save(object);
   }
@@ -63,9 +65,10 @@ abstract public class CRUDResourceManager<T extends Indexed> {
    * @return model object
    * @throws org.apache.ambari.view.hive.persistence.utils.ItemNotFound
    */
-  protected T read(Integer id) throws ItemNotFound {
+  @Override
+  public T read(Object id) throws ItemNotFound {
     T object = null;
-    object = getStorage().load(this.resourceClass, id);
+    object = storageFabric.getStorage().load(this.resourceClass, id);
     if (!checkPermissions(object))
       throw new ItemNotFound();
     return object;
@@ -76,8 +79,9 @@ abstract public class CRUDResourceManager<T extends Indexed> {
    * @param filteringStrategy filtering strategy
    * @return list of filtered objects
    */
-  protected List<T> readAll(FilteringStrategy filteringStrategy) {
-    return getStorage().loadAll(this.resourceClass, filteringStrategy);
+  @Override
+  public List<T> readAll(FilteringStrategy filteringStrategy) {
+    return storageFabric.getStorage().loadAll(this.resourceClass, filteringStrategy);
   }
 
   /**
@@ -87,7 +91,8 @@ abstract public class CRUDResourceManager<T extends Indexed> {
    * @return model object
    * @throws org.apache.ambari.view.hive.persistence.utils.ItemNotFound
    */
-  protected T update(T newObject, Integer id) throws ItemNotFound {
+  @Override
+  public T update(T newObject, String id) throws ItemNotFound {
     newObject.setId(id);
     this.save(newObject);
     return newObject;
@@ -98,29 +103,22 @@ abstract public class CRUDResourceManager<T extends Indexed> {
    * @param resourceId object identifier
    * @throws org.apache.ambari.view.hive.persistence.utils.ItemNotFound
    */
-  protected void delete(Integer resourceId) throws ItemNotFound {
-    if (!getStorage().exists(this.resourceClass, resourceId)) {
+  @Override
+  public void delete(Object resourceId) throws ItemNotFound {
+    if (!storageFabric.getStorage().exists(this.resourceClass, resourceId)) {
       throw new ItemNotFound();
     }
-    getStorage().delete(this.resourceClass, resourceId);
+    storageFabric.getStorage().delete(this.resourceClass, resourceId);
   }
 
   // UTILS
 
   protected T save(T object) {
-    getStorage().store(resourceClass, object);
+    storageFabric.getStorage().store(resourceClass, object);
     return object;
   }
 
-  protected Storage getStorage() {
-    if (storage == null) {
-      storage = StorageUtil.getInstance(getContext()).getStorage();
-    }
-    return storage;
-  }
-
   protected abstract boolean checkPermissions(T object);
-  protected abstract ViewContext getContext();
 
   protected void cleanupAfterErrorAndThrowAgain(Indexed object, ServiceFormattedException e) {
     try {

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/IResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/IResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/IResourceManager.java
new file mode 100644
index 0000000..222d695
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/IResourceManager.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources;
+
+import org.apache.ambari.view.hive.persistence.utils.FilteringStrategy;
+import org.apache.ambari.view.hive.persistence.utils.Indexed;
+import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
+
+import java.util.List;
+
+public interface IResourceManager<T extends Indexed> {
+  T create(T object);
+
+  T read(Object id) throws ItemNotFound;
+
+  List<T> readAll(FilteringStrategy filteringStrategy);
+
+  T update(T newObject, String id) throws ItemNotFound;
+
+  void delete(Object resourceId) throws ItemNotFound;
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java
index d1223fb..15fcf22 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/PersonalCRUDResourceManager.java
@@ -19,6 +19,7 @@
 package org.apache.ambari.view.hive.resources;
 
 import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.hive.persistence.IStorageFactory;
 import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
 import org.apache.ambari.view.hive.persistence.utils.PersonalResource;
 import org.slf4j.Logger;
@@ -31,24 +32,24 @@ import java.util.concurrent.Callable;
  * @param <T> Data type with ID and Owner
  */
 public class PersonalCRUDResourceManager<T extends PersonalResource> extends CRUDResourceManager<T> {
-  protected ViewContext context;
   protected boolean ignorePermissions = false;
 
   private final static Logger LOG =
       LoggerFactory.getLogger(PersonalCRUDResourceManager.class);
+  protected ViewContext context;
+
   /**
    * Constructor
    * @param resourceClass model class
-   * @param context View Context instance
    */
-  public PersonalCRUDResourceManager(Class<? extends T> resourceClass, ViewContext context) {
-    super(resourceClass);
+  public PersonalCRUDResourceManager(Class<? extends T> resourceClass, IStorageFactory storageFabric, ViewContext context) {
+    super(resourceClass, storageFabric);
     this.context = context;
   }
 
   @Override
-  public T update(T newObject, Integer id) throws ItemNotFound {
-    T object = getStorage().load(this.resourceClass, id);
+  public T update(T newObject, String id) throws ItemNotFound {
+    T object = storageFabric.getStorage().load(this.resourceClass, id);
     if (object.getOwner().compareTo(this.context.getUsername()) != 0) {
       throw new ItemNotFound();
     }
@@ -74,18 +75,13 @@ public class PersonalCRUDResourceManager<T extends PersonalResource> extends CRU
     return object.getOwner().compareTo(this.context.getUsername()) == 0;
   }
 
-  @Override
-  public ViewContext getContext() {
-    return context;
-  }
-
   /**
    * Execute action ignoring objects owner
    * @param actions callable to execute
    * @return value returned from actions
    * @throws Exception
    */
-  public <T> T ignorePermissions(Callable<T> actions) throws Exception {
+  public T ignorePermissions(Callable<T> actions) throws Exception {
     ignorePermissions = true;
     T result;
     try {

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/SharedCRUDResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/SharedCRUDResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/SharedCRUDResourceManager.java
index 0d2b297..9c4ca36 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/SharedCRUDResourceManager.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/SharedCRUDResourceManager.java
@@ -19,6 +19,7 @@
 package org.apache.ambari.view.hive.resources;
 
 import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.hive.persistence.IStorageFactory;
 import org.apache.ambari.view.hive.persistence.utils.Indexed;
 
 /**
@@ -31,20 +32,13 @@ public class SharedCRUDResourceManager<T extends Indexed> extends CRUDResourceMa
   /**
    * Constructor
    * @param responseClass model class
-   * @param context View Context instance
    */
-  public SharedCRUDResourceManager(Class<T> responseClass, ViewContext context) {
-    super(responseClass);
-    this.context = context;
+  public SharedCRUDResourceManager(Class<T> responseClass, IStorageFactory storageFabric) {
+    super(responseClass, storageFabric);
   }
 
   @Override
   protected boolean checkPermissions(T object) {
     return true; //everyone has permission
   }
-
-  @Override
-  protected ViewContext getContext() {
-    return context;
-  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java
index e5983b9..bb1a0a2 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/browser/HiveBrowserService.java
@@ -21,13 +21,13 @@ package org.apache.ambari.view.hive.resources.browser;
 import com.google.inject.Inject;
 import org.apache.ambari.view.ViewContext;
 import org.apache.ambari.view.ViewResourceHandler;
-import org.apache.ambari.view.hive.BaseService;
 import org.apache.ambari.view.hive.client.ColumnDescription;
-import org.apache.ambari.view.hive.client.ConnectionPool;
 import org.apache.ambari.view.hive.client.Cursor;
+import org.apache.ambari.view.hive.client.IConnectionFactory;
 import org.apache.ambari.view.hive.resources.jobs.ResultsPaginationController;
 import org.apache.ambari.view.hive.utils.BadRequestFormattedException;
 import org.apache.ambari.view.hive.utils.ServiceFormattedException;
+import org.apache.ambari.view.hive.utils.SharedObjectsFactory;
 import org.apache.commons.collections4.map.PassiveExpiringMap;
 import org.json.simple.JSONObject;
 import org.slf4j.Logger;
@@ -55,6 +55,8 @@ public class HiveBrowserService {
 
   private static final long EXPIRING_TIME = 10*60*1000;  // 10 minutes
   private static Map<String, Cursor> resultsCache;
+  private IConnectionFactory connectionFactory;
+
   public static Map<String, Cursor> getResultsCache() {
     if (resultsCache == null) {
       PassiveExpiringMap<String, Cursor> resultsCacheExpiringMap =
@@ -64,6 +66,12 @@ public class HiveBrowserService {
     return resultsCache;
   }
 
+  private IConnectionFactory getConnectionFactory() {
+    if (connectionFactory == null)
+      connectionFactory = new SharedObjectsFactory(context);
+    return new SharedObjectsFactory(context);
+  }
+
   /**
    * Returns list of databases
    */
@@ -81,7 +89,7 @@ public class HiveBrowserService {
     String curl = null;
     try {
       JSONObject response = new JSONObject();
-      List<String> tables = ConnectionPool.getConnection(context).ddl().getDBList(like);
+      List<String> tables = getConnectionFactory().getHiveConnection().ddl().getDBList(like);
       response.put("databases", tables);
       return Response.ok(response).build();
     } catch (WebApplicationException ex) {
@@ -116,7 +124,7 @@ public class HiveBrowserService {
               new Callable<Cursor>() {
                 @Override
                 public Cursor call() throws Exception {
-                  return ConnectionPool.getConnection(context).ddl().getDBListCursor(finalLike);
+                  return getConnectionFactory().getHiveConnection().ddl().getDBListCursor(finalLike);
                 }
               }).build();
     } catch (WebApplicationException ex) {
@@ -146,7 +154,7 @@ public class HiveBrowserService {
     String curl = null;
     try {
       JSONObject response = new JSONObject();
-      List<String> tables = ConnectionPool.getConnection(context).ddl().getTableList(db, like);
+      List<String> tables = getConnectionFactory().getHiveConnection().ddl().getTableList(db, like);
       response.put("tables", tables);
       response.put("database", db);
       return Response.ok(response).build();
@@ -183,7 +191,7 @@ public class HiveBrowserService {
               new Callable<Cursor>() {
                 @Override
                 public Cursor call() throws Exception {
-                  Cursor cursor = ConnectionPool.getConnection(context).ddl().getTableListCursor(db, finalLike);
+                  Cursor cursor = getConnectionFactory().getHiveConnection().ddl().getTableListCursor(db, finalLike);
                   cursor.selectColumns(requestedColumns);
                   return cursor;
                 }
@@ -212,7 +220,7 @@ public class HiveBrowserService {
     String curl = null;
     try {
       JSONObject response = new JSONObject();
-      List<ColumnDescription> columnDescriptions = ConnectionPool.getConnection(context).ddl()
+      List<ColumnDescription> columnDescriptions = getConnectionFactory().getHiveConnection().ddl()
           .getTableDescription(db, table, like, extendedTableDescription);
       response.put("columns", columnDescriptions);
       response.put("database", db);
@@ -247,7 +255,7 @@ public class HiveBrowserService {
               new Callable<Cursor>() {
                 @Override
                 public Cursor call() throws Exception {
-                  Cursor cursor = ConnectionPool.getConnection(context).ddl().getTableDescriptionCursor(db, table, like);
+                  Cursor cursor = getConnectionFactory().getHiveConnection().ddl().getTableDescriptionCursor(db, table, like);
                   cursor.selectColumns(requestedColumns);
                   return cursor;
                 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
index 8d886d5..3f5b3b8 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/files/FileService.java
@@ -23,6 +23,7 @@ import org.apache.ambari.view.ViewContext;
 import org.apache.ambari.view.ViewResourceHandler;
 import org.apache.ambari.view.hive.BaseService;
 import org.apache.ambari.view.hive.utils.*;
+import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.json.simple.JSONObject;
@@ -51,6 +52,8 @@ import java.io.IOException;
  *      update file content
  */
 public class FileService extends BaseService {
+  public static final String FAKE_FILE = "fakefile://";
+
   @Inject
   ViewResourceHandler handler;
 
@@ -66,17 +69,23 @@ public class FileService extends BaseService {
   public Response getFilePage(@PathParam("filePath") String filePath, @QueryParam("page") Long page) throws IOException, InterruptedException {
     LOG.debug("Reading file " + filePath);
     try {
-      FilePaginator paginator = new FilePaginator(filePath, context);
+      FileResource file = new FileResource();
 
       if (page == null)
         page = 0L;
 
-      FileResource file = new FileResource();
-      file.setFilePath(filePath);
-      file.setFileContent(paginator.readPage(page));
-      file.setHasNext(paginator.pageCount() > page + 1);
-      file.setPage(page);
-      file.setPageCount(paginator.pageCount());
+      if (filePath.startsWith(FAKE_FILE)) {
+        if (page > 1)
+          throw new IllegalArgumentException("There's only one page in fake files");
+
+        String content = filePath.substring(FAKE_FILE.length());
+
+        fillFakeFileObject(filePath, file, content);
+      } else {
+        FilePaginator paginator = new FilePaginator(filePath, getSharedObjectsFactory().getHdfsApi());
+
+        fillRealFileObject(filePath, page, file, paginator);
+      }
 
       JSONObject object = new JSONObject();
       object.put("file", file);
@@ -92,6 +101,24 @@ public class FileService extends BaseService {
     }
   }
 
+  public void fillRealFileObject(String filePath, Long page, FileResource file, FilePaginator paginator) throws IOException, InterruptedException {
+    file.setFilePath(filePath);
+    file.setFileContent(paginator.readPage(page));
+    file.setHasNext(paginator.pageCount() > page + 1);
+    file.setPage(page);
+    file.setPageCount(paginator.pageCount());
+  }
+
+  public void fillFakeFileObject(String filePath, FileResource file, String encodedContent) {
+    String content = new String(Base64.decodeBase64(encodedContent));
+
+    file.setFilePath(filePath);
+    file.setFileContent(content);
+    file.setHasNext(false);
+    file.setPage(0);
+    file.setPageCount(1);
+  }
+
   /**
    * Delete single item
    */
@@ -100,7 +127,7 @@ public class FileService extends BaseService {
   public Response deleteFile(@PathParam("filePath") String filePath) throws IOException, InterruptedException {
     try {
       LOG.debug("Deleting file " + filePath);
-      if (getHdfsApi().delete(filePath, false)) {
+      if (getSharedObjectsFactory().getHdfsApi().delete(filePath, false)) {
         return Response.status(204).build();
       }
       throw new NotFoundFormattedException("FileSystem.delete returned false", null);
@@ -121,7 +148,7 @@ public class FileService extends BaseService {
                              @PathParam("filePath") String filePath) throws IOException, InterruptedException {
     try {
       LOG.debug("Rewriting file " + filePath);
-      FSDataOutputStream output = getHdfsApi().create(filePath, true);
+      FSDataOutputStream output = getSharedObjectsFactory().getHdfsApi().create(filePath, true);
       output.writeBytes(request.file.getFileContent());
       output.close();
       return Response.status(204).build();
@@ -143,7 +170,7 @@ public class FileService extends BaseService {
     try {
       LOG.debug("Creating file " + request.file.getFilePath());
       try {
-        FSDataOutputStream output = getHdfsApi().create(request.file.getFilePath(), false);
+        FSDataOutputStream output = getSharedObjectsFactory().getHdfsApi().create(request.file.getFilePath(), false);
         if (request.file.getFileContent() != null) {
           output.writeBytes(request.file.getFileContent());
         }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java
new file mode 100644
index 0000000..ce82e15
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Aggregator.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.jobs;
+
+import org.apache.ambari.view.hive.persistence.utils.FilteringStrategy;
+import org.apache.ambari.view.hive.persistence.utils.Indexed;
+import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
+import org.apache.ambari.view.hive.persistence.utils.OnlyOwnersFilteringStrategy;
+import org.apache.ambari.view.hive.resources.IResourceManager;
+import org.apache.ambari.view.hive.resources.jobs.atsJobs.HiveQueryId;
+import org.apache.ambari.view.hive.resources.jobs.atsJobs.IATSParser;
+import org.apache.ambari.view.hive.resources.jobs.atsJobs.TezDagId;
+import org.apache.ambari.view.hive.resources.jobs.viewJobs.Job;
+import org.apache.ambari.view.hive.resources.jobs.viewJobs.JobImpl;
+import org.apache.commons.beanutils.PropertyUtils;
+import org.apache.commons.codec.binary.Base64;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * View Jobs and ATS Jobs aggregator
+ * Not all ViewJobs create ATS job
+ */
+public class Aggregator {
+  protected final static Logger LOG =
+      LoggerFactory.getLogger(Aggregator.class);
+
+  private final IATSParser ats;
+  private final IOperationHandleResourceManager operationHandleResourceManager;
+  private IResourceManager<Job> viewJobResourceManager;
+
+  public Aggregator(IResourceManager<Job> jobResourceManager,
+                    IOperationHandleResourceManager operationHandleResourceManager,
+                    IATSParser ats) {
+    this.viewJobResourceManager = jobResourceManager;
+    this.operationHandleResourceManager = operationHandleResourceManager;
+    this.ats = ats;
+  }
+
+  public List<Job> readAll(String username) {
+      Set<String> addedOperationIds = new HashSet<String>();
+
+    List<Job> allJobs = new LinkedList<Job>();
+    for (HiveQueryId atsHiveQuery : ats.getHiveQuieryIdsList(username)) {
+
+      TezDagId atsTezDag;
+      if (atsHiveQuery.dagNames != null && atsHiveQuery.dagNames.size() > 0) {
+        String dagName = atsHiveQuery.dagNames.get(0);
+
+        atsTezDag = ats.getTezDAGByName(dagName);
+      } else {
+        atsTezDag = new TezDagId();
+      }
+
+      JobImpl atsJob;
+      if (hasOperationId(atsHiveQuery)) {
+        try {
+          Job viewJob = getJobByOperationId(urlSafeBase64ToHexString(atsHiveQuery.operationId));
+          saveJobInfoIfNeeded(atsHiveQuery, atsTezDag, viewJob);
+
+          atsJob = mergeAtsJobWithViewJob(atsHiveQuery, atsTezDag, viewJob);
+        } catch (ItemNotFound itemNotFound) {
+          // Executed from HS2, but outside of Hive View
+          atsJob = atsOnlyJob(atsHiveQuery, atsTezDag);
+        }
+      } else {
+        atsJob = atsOnlyJob(atsHiveQuery, atsTezDag);
+      }
+      allJobs.add(atsJob);
+
+      addedOperationIds.add(atsHiveQuery.operationId);
+    }
+
+    //cover case when operationId is present, but not exists in ATS
+    //e.g. optimized queries without executing jobs, like "SELECT * FROM TABLE"
+    for (Job job : viewJobResourceManager.readAll(new OnlyOwnersFilteringStrategy(username))) {
+      List<StoredOperationHandle> operationHandles = operationHandleResourceManager.readJobRelatedHandles(job);
+      assert operationHandles.size() <= 1;
+
+      if (operationHandles.size() > 0) {
+        StoredOperationHandle operationHandle = operationHandles.get(0);
+
+        if (!addedOperationIds.contains(hexStringToUrlSafeBase64(operationHandle.getGuid()))) {
+          //e.g. query without hadoop job: select * from table
+          allJobs.add(job);
+        }
+      }
+    }
+
+    return allJobs;
+  }
+
+  protected boolean hasOperationId(HiveQueryId atsHiveQuery) {
+    return atsHiveQuery.operationId != null;
+  }
+
+  protected JobImpl mergeAtsJobWithViewJob(HiveQueryId atsHiveQuery, TezDagId atsTezDag, Job viewJob) {
+    JobImpl atsJob;
+    try {
+      atsJob = new JobImpl(PropertyUtils.describe(viewJob));
+    }catch(IllegalAccessException e){
+      LOG.error("Can't instantiate JobImpl", e);
+      return null;
+    }catch(InvocationTargetException e){
+      LOG.error("Can't instantiate JobImpl", e);
+      return null;
+    }catch(NoSuchMethodException e){
+      LOG.error("Can't instantiate JobImpl", e);
+      return null;
+    }
+    fillAtsJobFields(atsJob, atsHiveQuery, atsTezDag);
+    return atsJob;
+  }
+
+  protected void saveJobInfoIfNeeded(HiveQueryId hiveQueryId, TezDagId tezDagId, Job viewJob) throws ItemNotFound {
+    if (viewJob.getDagName() == null) {
+      viewJob.setDagName(tezDagId.dagName);
+      viewJobResourceManager.update(viewJob, viewJob.getId());
+    }
+    if (viewJob.getStatus().equals(tezDagId.status)) {
+      viewJob.setStatus(tezDagId.status);
+      viewJobResourceManager.update(viewJob, viewJob.getId());
+    }
+  }
+
+  protected JobImpl atsOnlyJob(HiveQueryId atsHiveQuery, TezDagId atsTezDag) {
+    JobImpl atsJob = new JobImpl();
+    atsJob.setId(atsHiveQuery.entity);
+    fillAtsJobFields(atsJob, atsHiveQuery, atsTezDag);
+
+    String query = atsHiveQuery.query;
+    atsJob.setTitle(query.substring(0, (query.length() > 42)?42:query.length()));
+
+    atsJob.setQueryFile("fakefile://" + Base64.encodeBase64URLSafeString(query.getBytes()));  // fake queryFile
+    return atsJob;
+  }
+
+  protected JobImpl fillAtsJobFields(JobImpl atsJob, HiveQueryId atsHiveQuery, TezDagId atsTezDag) {
+    atsJob.setApplicationId(atsTezDag.applicationId);
+
+    atsJob.setDagName(atsTezDag.dagName);
+    if (!atsTezDag.status.equals(TezDagId.STATUS_UNKNOWN))
+      atsJob.setStatus(atsTezDag.status);
+    if (atsHiveQuery.starttime != 0)
+      atsJob.setDateSubmitted(atsHiveQuery.starttime);
+    atsJob.setDuration(atsHiveQuery.duration);
+    return atsJob;
+  }
+
+  protected Job getJobByOperationId(final String opId) throws ItemNotFound {
+    List<StoredOperationHandle> operationHandles = operationHandleResourceManager.readAll(new FilteringStrategy() {
+      @Override
+      public boolean isConform(Indexed item) {
+        StoredOperationHandle opHandle = (StoredOperationHandle) item;
+        return opHandle.getGuid().equals(opId);
+      }
+
+      @Override
+      public String whereStatement() {
+        return "guid='" + opId + "'";
+      }
+    });
+
+    if (operationHandles.size() != 1)
+      throw new ItemNotFound();
+
+    return viewJobResourceManager.read(operationHandles.get(0).getJobId());
+  }
+
+  protected static String urlSafeBase64ToHexString(String urlsafeBase64){
+    byte[] decoded = Base64.decodeBase64(urlsafeBase64);
+
+    StringBuilder sb = new StringBuilder();
+    for(byte b : decoded){
+      sb.append(String.format("%02x", b));
+    }
+    return sb.toString();
+  }
+
+  protected static String hexStringToUrlSafeBase64(String hexString){
+    byte[] decoded = new byte[hexString.length() / 2];
+
+    for(int i=0; i<hexString.length(); i+=2) {
+       decoded[i / 2] = (byte) Integer.parseInt(String.format("%c%c", hexString.charAt(i), hexString.charAt(i+1)), 16);
+    }
+    return Base64.encodeBase64URLSafeString(decoded);
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java
index 8f143e7..b70abe2 100644
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/ConnectionController.java
@@ -18,32 +18,19 @@
 
 package org.apache.ambari.view.hive.resources.jobs;
 
-import org.apache.ambari.view.ViewContext;
 import org.apache.ambari.view.hive.client.Connection;
-import org.apache.ambari.view.hive.client.ConnectionPool;
 import org.apache.ambari.view.hive.client.HiveClientException;
 import org.apache.ambari.view.hive.utils.ServiceFormattedException;
 import org.apache.hive.service.cli.thrift.TOperationHandle;
 
-import java.util.HashMap;
-import java.util.Map;
 
 public class ConnectionController {
-  private ViewContext context;
-  private Connection connection;
   private OperationHandleControllerFactory operationHandleControllerFactory;
+  private Connection connection;
 
-  private ConnectionController(ViewContext context) {
-    this.context = context;
-    connection = ConnectionPool.getConnection(context);
-    operationHandleControllerFactory = OperationHandleControllerFactory.getInstance(context);
-  }
-
-  private static Map<String, ConnectionController> viewSingletonObjects = new HashMap<String, ConnectionController>();
-  public static ConnectionController getInstance(ViewContext context) {
-    if (!viewSingletonObjects.containsKey(context.getInstanceName()))
-      viewSingletonObjects.put(context.getInstanceName(), new ConnectionController(context));
-    return viewSingletonObjects.get(context.getInstanceName());
+  public ConnectionController(OperationHandleControllerFactory operationHandleControllerFactory, Connection connection) {
+    this.connection = connection;
+    this.operationHandleControllerFactory = operationHandleControllerFactory;
   }
 
   public void selectDatabase(String database) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/IOperationHandleResourceManager.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/IOperationHandleResourceManager.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/IOperationHandleResourceManager.java
new file mode 100644
index 0000000..185402e
--- /dev/null
+++ b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/IOperationHandleResourceManager.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.hive.resources.jobs;
+
+import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
+import org.apache.ambari.view.hive.resources.IResourceManager;
+import org.apache.ambari.view.hive.resources.jobs.viewJobs.Job;
+import org.apache.hive.service.cli.thrift.TOperationHandle;
+
+import java.util.List;
+
+public interface IOperationHandleResourceManager extends IResourceManager<StoredOperationHandle> {
+  List<StoredOperationHandle> readJobRelatedHandles(Job job);
+
+  void putHandleForJob(TOperationHandle h, Job job);
+
+  boolean containsHandleForJob(Job job);
+
+  TOperationHandle getHandleForJob(Job job) throws ItemNotFound;
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Job.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Job.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Job.java
deleted file mode 100644
index e6f7a63..0000000
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/Job.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.view.hive.resources.jobs;
-
-import org.apache.ambari.view.hive.persistence.utils.Indexed;
-import org.apache.ambari.view.hive.persistence.utils.PersonalResource;
-
-import java.io.Serializable;
-
-/**
- * Interface for Job bean to create Proxy for it
- */
-public interface Job extends Serializable,Indexed,PersonalResource {
-  public static final String JOB_STATE_UNKNOWN = "Unknown";
-  public static final String JOB_STATE_INITIALIZED = "Initialized";
-  public static final String JOB_STATE_RUNNING = "Running";
-  public static final String JOB_STATE_FINISHED = "Finished";
-  public static final String JOB_STATE_CANCELED = "Canceled";
-  public static final String JOB_STATE_CLOSED = "Closed";
-  public static final String JOB_STATE_ERROR = "Error";
-  public static final String JOB_STATE_PENDING = "Pending";
-
-  Integer getId();
-
-  void setId(Integer id);
-
-  String getOwner();
-
-  void setOwner(String owner);
-
-  String getTitle();
-
-  void setTitle(String title);
-
-  String getQueryFile();
-
-  void setQueryFile(String queryFile);
-
-  Long getDateSubmitted();
-
-  void setDateSubmitted(Long dateSubmitted);
-
-  Long getDuration();
-
-  void setDuration(Long duration);
-
-  String getStatus();
-
-  void setStatus(String status);
-
-  String getForcedContent();
-
-  void setForcedContent(String forcedContent);
-
-  Integer getQueryId();
-
-  void setQueryId(Integer queryId);
-
-  String getStatusDir();
-
-  void setStatusDir(String statusDir);
-
-  String getDataBase();
-
-  void setDataBase(String dataBase);
-
-  String getLogFile();
-
-  void setLogFile(String logFile);
-
-  String getConfFile();
-
-  void setConfFile(String confFile);
-}

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobController.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobController.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobController.java
deleted file mode 100644
index 3d5189e..0000000
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobController.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.view.hive.resources.jobs;
-
-import org.apache.ambari.view.hive.client.Cursor;
-import org.apache.ambari.view.hive.persistence.utils.ItemNotFound;
-
-public interface JobController {
-  void submit();
-
-  void cancel() throws ItemNotFound;
-
-  Job getJob();
-
-  /**
-   * Use carefully. Returns unproxied bean object
-   * @return unproxied bean object
-   */
-  Job getJobPOJO();
-
-  Cursor getResults() throws ItemNotFound;
-
-  void afterCreation();
-
-  void onRead();
-
-  boolean isModified();
-
-  void clearModified();
-}

http://git-wip-us.apache.org/repos/asf/ambari/blob/8dbdbf66/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobControllerFactory.java
----------------------------------------------------------------------
diff --git a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobControllerFactory.java b/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobControllerFactory.java
deleted file mode 100644
index f6ec5b1..0000000
--- a/contrib/views/hive/src/main/java/org/apache/ambari/view/hive/resources/jobs/JobControllerFactory.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.view.hive.resources.jobs;
-
-import org.apache.ambari.view.ViewContext;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class JobControllerFactory {
-  private ViewContext context;
-
-  private JobControllerFactory(ViewContext context) {
-    this.context = context;
-  }
-
-  private static Map<String, JobControllerFactory> viewSingletonObjects = new HashMap<String, JobControllerFactory>();
-  public static JobControllerFactory getInstance(ViewContext context) {
-    if (!viewSingletonObjects.containsKey(context.getInstanceName()))
-      viewSingletonObjects.put(context.getInstanceName(), new JobControllerFactory(context));
-    return viewSingletonObjects.get(context.getInstanceName());
-  }
-
-  public JobController createControllerForJob(Job job) {
-    return new JobControllerImpl(context, job);
-  }
-}


Mime
View raw message