Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id C6429200B48 for ; Mon, 18 Jul 2016 13:54:45 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id C4CE7160A87; Mon, 18 Jul 2016 11:54:45 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id E4EF5160A6D for ; Mon, 18 Jul 2016 13:54:44 +0200 (CEST) Received: (qmail 22741 invoked by uid 500); 18 Jul 2016 11:54:44 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 22732 invoked by uid 99); 18 Jul 2016 11:54:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 18 Jul 2016 11:54:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id EAD3DDFAF5; Mon, 18 Jul 2016 11:54:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: nitiraj@apache.org To: commits@ambari.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-17605. fixed Upload table using Hive view will fail if Hive database is created with Location option (nitirajrathore) Date: Mon, 18 Jul 2016 11:54:43 +0000 (UTC) archived-at: Mon, 18 Jul 2016 11:54:46 -0000 Repository: ambari Updated Branches: refs/heads/branch-2.4 21190abe1 -> 4ac42ee5e AMBARI-17605. fixed Upload table using Hive view will fail if Hive database is created with Location option (nitirajrathore) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4ac42ee5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4ac42ee5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4ac42ee5 Branch: refs/heads/branch-2.4 Commit: 4ac42ee5eef6cd387380344d2d0966955701300d Parents: 21190ab Author: Nitiraj Rathore Authored: Mon Jul 18 17:21:03 2016 +0530 Committer: Nitiraj Rathore Committed: Mon Jul 18 17:23:52 2016 +0530 ---------------------------------------------------------------------- .../ambari/view/hive2/client/DDLDelegator.java | 2 + .../view/hive2/client/DDLDelegatorImpl.java | 13 +++++ .../resources/browser/HiveBrowserService.java | 4 +- .../hive2/resources/uploads/UploadService.java | 61 +++++++++++++++----- 4 files changed, 64 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4ac42ee5/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegator.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegator.java b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegator.java index 20b91f1..1774a65 100644 --- a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegator.java +++ b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegator.java @@ -26,6 +26,8 @@ public interface DDLDelegator { List getTableList(ConnectionConfig config, String database, String like); + List getTableDescriptionFormatted(ConnectionConfig config, String database, String table); + List getTableDescription(ConnectionConfig config, String database, String table, String like, boolean extended); Cursor getDbListCursor(ConnectionConfig config, String like); http://git-wip-us.apache.org/repos/asf/ambari/blob/4ac42ee5/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegatorImpl.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegatorImpl.java b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegatorImpl.java index 0171626..dff9471 100644 --- a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegatorImpl.java +++ b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/client/DDLDelegatorImpl.java @@ -81,6 +81,19 @@ public class DDLDelegatorImpl implements DDLDelegator { } @Override + public List getTableDescriptionFormatted(ConnectionConfig config, String database, String table) { + Optional rowsFromDB = getRowsFromDB(config, getTableDescriptionStatements(database, table)); + return rowsFromDB.isPresent() ? rowsFromDB.get().getRows() : null; + } + + private String[] getTableDescriptionStatements(String database, String table) { + return new String[]{ + String.format("use %s",database), + String.format("describe formatted %s", table) + }; + } + + @Override public List getTableDescription(ConnectionConfig config, String database, String table, String like, boolean extended) { Optional resultOptional = getTableDescription(config, database, table, like); List descriptions = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/4ac42ee5/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/browser/HiveBrowserService.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/browser/HiveBrowserService.java b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/browser/HiveBrowserService.java index 0da49c4..7481f9e 100644 --- a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/browser/HiveBrowserService.java +++ b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/browser/HiveBrowserService.java @@ -20,6 +20,8 @@ package org.apache.ambari.view.hive2.resources.browser; import org.apache.ambari.view.ViewContext; import org.apache.ambari.view.ViewResourceHandler; +import org.apache.ambari.view.hive2.ConnectionFactory; +import org.apache.ambari.view.hive2.ConnectionSystem; import org.apache.ambari.view.hive2.client.ColumnDescription; import org.apache.ambari.view.hive2.client.ConnectionConfig; import org.apache.ambari.view.hive2.client.Cursor; @@ -29,8 +31,6 @@ import org.apache.ambari.view.hive2.client.Row; import org.apache.ambari.view.hive2.resources.jobs.ResultsPaginationController; import org.apache.ambari.view.hive2.utils.BadRequestFormattedException; import org.apache.ambari.view.hive2.utils.ServiceFormattedException; -import org.apache.ambari.view.hive2.ConnectionFactory; -import org.apache.ambari.view.hive2.ConnectionSystem; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/ambari/blob/4ac42ee5/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/UploadService.java ---------------------------------------------------------------------- diff --git a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/UploadService.java b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/UploadService.java index 18da1ff..08d91e9 100644 --- a/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/UploadService.java +++ b/contrib/views/hive-next/src/main/java/org/apache/ambari/view/hive2/resources/uploads/UploadService.java @@ -20,8 +20,14 @@ package org.apache.ambari.view.hive2.resources.uploads; import com.sun.jersey.core.header.FormDataContentDisposition; import com.sun.jersey.multipart.FormDataParam; +import org.apache.ambari.view.ViewContext; import org.apache.ambari.view.hive.resources.uploads.CSVParams; import org.apache.ambari.view.hive2.BaseService; +import org.apache.ambari.view.hive2.ConnectionFactory; +import org.apache.ambari.view.hive2.ConnectionSystem; +import org.apache.ambari.view.hive2.client.DDLDelegator; +import org.apache.ambari.view.hive2.client.DDLDelegatorImpl; +import org.apache.ambari.view.hive2.client.Row; import org.apache.ambari.view.hive2.resources.jobs.viewJobs.Job; import org.apache.ambari.view.hive2.resources.jobs.viewJobs.JobController; import org.apache.ambari.view.hive2.resources.jobs.viewJobs.JobImpl; @@ -45,6 +51,7 @@ import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.inject.Inject; import javax.ws.rs.Consumes; import javax.ws.rs.POST; import javax.ws.rs.Path; @@ -57,6 +64,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.net.URI; +import java.net.URISyntaxException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -80,6 +89,9 @@ public class UploadService extends BaseService { private final static Logger LOG = LoggerFactory.getLogger(UploadService.class); + @Inject + protected ViewContext context; + private AmbariApi ambariApi; protected JobResourceManager resourceManager; @@ -348,22 +360,9 @@ public class UploadService extends BaseService { private String uploadIntoTable(Reader reader, String databaseName, String tempTableName) { try { - String basePath = getHiveMetaStoreLocation(); - - if (!basePath.endsWith("/")) { - basePath = basePath + "/"; - } - - if (databaseName != null && !databaseName.equals(HIVE_DEFAULT_DB)) { - basePath = basePath + databaseName + ".db/"; - } - - String fullPath = basePath + tempTableName + "/" + tempTableName + ".csv"; - + String fullPath = getHiveMetaStoreLocation(databaseName, tempTableName); LOG.info("Uploading file into : {}", fullPath); - uploadFile(fullPath, new ReaderInputStream(reader)); - return fullPath; } catch (WebApplicationException e) { LOG.error(getErrorMessage(e), e); @@ -418,6 +417,40 @@ public class UploadService extends BaseService { return job; } + private String getHiveMetaStoreLocation(String db, String table) { + String locationColValue = "Location:"; + String urlString = null; + DDLDelegator delegator = new DDLDelegatorImpl(context, ConnectionSystem.getInstance().getActorSystem(), ConnectionSystem.getInstance().getOperationController(context)); + List result = delegator.getTableDescriptionFormatted(ConnectionFactory.create(context), db, table); + for (Row row : result) { + if (row != null && row.getRow().length > 1 && row.getRow()[0] != null && row.getRow()[0].toString().trim().equals(locationColValue)) { + urlString = row.getRow()[1] == null ? null : row.getRow()[1].toString(); + break; + } + } + + String tablePath = null; + if (null != urlString) { + try { + URI uri = new URI(urlString); + tablePath = uri.getPath(); + } catch (URISyntaxException e) { + LOG.debug("Error occurred while parsing as url : ", urlString, e); + } + } else { + String basePath = getHiveMetaStoreLocation(); + if (!basePath.endsWith("/")) { + basePath = basePath + "/"; + } + if (db != null && !db.equals(HIVE_DEFAULT_DB)) { + basePath = basePath + db + ".db/"; + } + tablePath = basePath + table; + } + + return tablePath + "/" + table ; + } + private String getHiveMetaStoreLocation() { String dir = context.getProperties().get(HIVE_METASTORE_LOCATION_KEY_VIEW_PROPERTY); if (dir != null && !dir.trim().isEmpty()) {