Return-Path: X-Original-To: apmail-atlas-commits-archive@minotaur.apache.org Delivered-To: apmail-atlas-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BC9B418F6A for ; Tue, 16 Jun 2015 23:04:50 +0000 (UTC) Received: (qmail 30922 invoked by uid 500); 16 Jun 2015 23:04:50 -0000 Delivered-To: apmail-atlas-commits-archive@atlas.apache.org Received: (qmail 30900 invoked by uid 500); 16 Jun 2015 23:04:50 -0000 Mailing-List: contact commits-help@atlas.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@atlas.incubator.apache.org Delivered-To: mailing list commits@atlas.incubator.apache.org Received: (qmail 30891 invoked by uid 99); 16 Jun 2015 23:04:50 -0000 Received: from Unknown (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Jun 2015 23:04:50 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id BB851182829 for ; Tue, 16 Jun 2015 23:04:49 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.77 X-Spam-Level: * X-Spam-Status: No, score=1.77 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, T_RP_MATCHES_RCVD=-0.01] autolearn=disabled Received: from mx1-us-east.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id nSAOmUVotmSr for ; Tue, 16 Jun 2015 23:04:37 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-east.apache.org (ASF Mail Server at mx1-us-east.apache.org) with SMTP id 5300747C16 for ; Tue, 16 Jun 2015 23:04:36 +0000 (UTC) Received: (qmail 30319 invoked by uid 99); 16 Jun 2015 23:04:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Jun 2015 23:04:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B3E8BE3C56; Tue, 16 Jun 2015 23:04:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: venkatesh@apache.org To: commits@atlas.incubator.apache.org Date: Tue, 16 Jun 2015 23:04:37 -0000 Message-Id: <2d18be0325a84e809075d6732891faeb@git.apache.org> In-Reply-To: <29f8990c10f64784b72a9cafbbc9676e@git.apache.org> References: <29f8990c10f64784b72a9cafbbc9676e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [03/58] [abbrv] incubator-atlas git commit: Refactor packages and scripts to Atlas (cherry picked from commit 414beba) http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java new file mode 100755 index 0000000..be6ae8b --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java @@ -0,0 +1,679 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.resources; + +import com.google.common.collect.ImmutableList; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import org.apache.atlas.MetadataServiceClient; +import org.apache.atlas.MetadataServiceException; +import org.apache.atlas.typesystem.IStruct; +import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.json.InstanceSerialization; +import org.apache.atlas.typesystem.json.InstanceSerialization$; +import org.apache.atlas.typesystem.json.TypesSerialization; +import org.apache.atlas.typesystem.json.TypesSerialization$; +import org.apache.atlas.typesystem.persistence.Id; +import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.EnumTypeDefinition; +import org.apache.atlas.typesystem.types.EnumValue; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.Multiplicity; +import org.apache.atlas.typesystem.types.StructTypeDefinition; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeUtils; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.apache.atlas.web.util.Servlets; +import org.apache.commons.lang.RandomStringUtils; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.Response; +import java.util.List; +import java.util.UUID; + +/** + * Integration tests for Entity Jersey Resource. + */ +public class EntityJerseyResourceIT extends BaseResourceIT { + + private static final Logger LOG = LoggerFactory.getLogger(EntityJerseyResourceIT.class); + + private static final String DATABASE_TYPE = "hive_database"; + private static final String DATABASE_NAME = "foo"; + private static final String TABLE_TYPE = "hive_table_type"; + private static final String TABLE_NAME = "bar"; + private static final String TRAITS = "traits"; + + private Referenceable tableInstance; + private Id tableId; + private String traitName; + + @BeforeClass + public void setUp() throws Exception { + super.setUp(); + + createHiveTypes(); + } + + @Test + public void testSubmitEntity() throws Exception { + tableInstance = createHiveTableInstance(); + tableId = createInstance(tableInstance); + + final String guid = tableId._getId(); + try { + Assert.assertNotNull(UUID.fromString(guid)); + } catch (IllegalArgumentException e) { + Assert.fail("Response is not a guid, " + guid); + } + } + + @DataProvider + public Object[][] invalidAttrValues() { + return new Object[][]{ + {null}, {""}, {" "}}; + } + + @Test(dataProvider = "invalidAttrValues") + public void testEntityInvalidValue(String value) throws Exception { + Referenceable databaseInstance = new Referenceable(DATABASE_TYPE); + databaseInstance.set("name", randomString()); + databaseInstance.set("description", value); + + try { + createInstance(databaseInstance); + Assert.fail("Exptected MetadataServiceException"); + } catch(MetadataServiceException e) { + Assert.assertEquals(e.getStatus(), ClientResponse.Status.BAD_REQUEST); + } + } + + @Test + public void testSubmitEntityWithBadDateFormat() throws Exception { + + try { + Referenceable databaseInstance = new Referenceable(DATABASE_TYPE); + databaseInstance.set("name", DATABASE_NAME); + databaseInstance.set("description", "foo database"); + + Referenceable tableInstance = new Referenceable(TABLE_TYPE, + "classification", "pii", "phi", "pci", "sox", "sec", "finance"); + tableInstance.set("name", TABLE_NAME); + tableInstance.set("description", "bar table"); + tableInstance.set("date", "2014-07-11"); + tableInstance.set("type", "managed"); + tableInstance.set("level", 2); + tableInstance.set("tableType", 1); // enum + tableInstance.set("database", databaseInstance); + tableInstance.set("compressed", false); + + Struct traitInstance = (Struct) tableInstance.getTrait("classification"); + traitInstance.set("tag", "foundation_etl"); + + Struct serde1Instance = new Struct("serdeType"); + serde1Instance.set("name", "serde1"); + serde1Instance.set("serde", "serde1"); + tableInstance.set("serde1", serde1Instance); + + Struct serde2Instance = new Struct("serdeType"); + serde2Instance.set("name", "serde2"); + serde2Instance.set("serde", "serde2"); + tableInstance.set("serde2", serde2Instance); + + tableId = createInstance(tableInstance); + Assert.fail("Was expecting an exception here "); + } catch (MetadataServiceException e) { + Assert.assertTrue( + e.getMessage().contains("\"error\":\"Cannot convert value '2014-07-11' to datatype date\"")); + } + } + + @Test(dependsOnMethods = "testSubmitEntity") + public void testAddProperty() throws Exception { + final String guid = tableId._getId(); + //add property + String description = "bar table - new desc"; + ClientResponse clientResponse = addProperty(guid, "description", description); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String entityRef = getEntityDefinition(getEntityDefinition(guid)); + Assert.assertNotNull(entityRef); + + tableInstance.set("description", description); + + //invalid property for the type + clientResponse = addProperty(guid, "invalid_property", "bar table"); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode()); + + //non-string property, update + clientResponse = addProperty(guid, "level", "4"); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + entityRef = getEntityDefinition(getEntityDefinition(guid)); + Assert.assertNotNull(entityRef); + + tableInstance.set("level", 4); + } + + @Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class) + public void testAddNullProperty() throws Exception { + final String guid = tableId._getId(); + //add property + addProperty(guid, null, "foo bar"); + Assert.fail(); + } + + @Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class) + public void testAddNullPropertyValue() throws Exception { + final String guid = tableId._getId(); + //add property + addProperty(guid, "description", null); + Assert.fail(); + } + + @Test(dependsOnMethods = "testSubmitEntity") + public void testAddReferenceProperty() throws Exception { + //Create new db instance + Referenceable databaseInstance = new Referenceable(DATABASE_TYPE); + databaseInstance.set("name", "newdb"); + databaseInstance.set("description", "new database"); + + Id dbInstance = createInstance(databaseInstance); + String dbId = dbInstance._getId(); + + //Add reference property + final String guid = tableId._getId(); + ClientResponse clientResponse = addProperty(guid, "database", dbId); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + } + + @Test(dependsOnMethods = "testSubmitEntity") + public void testGetEntityDefinition() throws Exception { + final String guid = tableId._getId(); + ClientResponse clientResponse = getEntityDefinition(guid); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + final String definition = response.getString(MetadataServiceClient.DEFINITION); + Assert.assertNotNull(definition); + LOG.debug("tableInstanceAfterGet = " + definition); + InstanceSerialization.fromJsonReferenceable(definition, true); + } + + private ClientResponse addProperty(String guid, String property, String value) { + WebResource resource = service + .path("api/atlas/entities") + .path(guid); + + return resource.queryParam("property", property).queryParam("value", value) + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.PUT, ClientResponse.class); + } + + private ClientResponse getEntityDefinition(String guid) { + WebResource resource = service + .path("api/atlas/entities") + .path(guid); + return resource.accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + } + + private String getEntityDefinition(ClientResponse clientResponse) throws Exception { + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + JSONObject response = new JSONObject(clientResponse.getEntity(String.class)); + final String definition = response.getString(MetadataServiceClient.DEFINITION); + Assert.assertNotNull(definition); + + return definition; + } + + @Test + public void testGetInvalidEntityDefinition() throws Exception { + WebResource resource = service + .path("api/atlas/entities") + .path("blah"); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.ERROR)); + Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE)); + } + + @Test(dependsOnMethods = "testSubmitEntity") + public void testGetEntityList() throws Exception { + ClientResponse clientResponse = service + .path("api/atlas/entities") + .queryParam("type", TABLE_TYPE) + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS); + Assert.assertNotNull(list); + Assert.assertEquals(list.length(), 1); + } + + @Test + public void testGetEntityListForBadEntityType() throws Exception { + ClientResponse clientResponse = service + .path("api/atlas/entities") + .queryParam("type", "blah") + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.ERROR)); + Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE)); + } + + + @Test + public void testGetEntityListForNoInstances() throws Exception { + addNewType(); + + ClientResponse clientResponse = service + .path("api/atlas/entities") + .queryParam("type", "test") + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS); + Assert.assertEquals(list.length(), 0); + } + + private void addNewType() throws Exception { + HierarchicalTypeDefinition testTypeDefinition = + TypesUtil.createClassTypeDef("test", ImmutableList.of(), + TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE)); + + String typesAsJSON = TypesSerialization.toJson(testTypeDefinition); + createType(typesAsJSON); + } + + @Test(dependsOnMethods = "testSubmitEntity") + public void testGetTraitNames() throws Exception { + final String guid = tableId._getId(); + ClientResponse clientResponse = service + .path("api/atlas/entities") + .path(guid) + .path(TRAITS) + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + Assert.assertNotNull(response.get("GUID")); + + final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS); + Assert.assertEquals(list.length(), 7); + } + + @Test(dependsOnMethods = "testGetTraitNames") + public void testAddTrait() throws Exception { + traitName = "PII_Trait" + randomString(); + HierarchicalTypeDefinition piiTrait = + TypesUtil.createTraitTypeDef(traitName, ImmutableList.of()); + String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true); + LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON); + createType(traitDefinitionAsJSON); + + Struct traitInstance = new Struct(traitName); + String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true); + LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON); + + final String guid = tableId._getId(); + ClientResponse clientResponse = service + .path("api/atlas/entities") + .path(guid) + .path(TRAITS) + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + Assert.assertNotNull(response.get(MetadataServiceClient.GUID)); + } + + @Test(dependsOnMethods = "testAddTrait") + public void testAddExistingTrait() throws Exception { + final String traitName = "PII_Trait" + randomString(); + + Struct traitInstance = new Struct(traitName); + String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true); + LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON); + + final String guid = tableId._getId(); + ClientResponse clientResponse = service + .path("api/atlas/entities") + .path(guid) + .path(TRAITS) + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode()); + } + + @Test(dependsOnMethods = "testGetTraitNames") + public void testAddTraitWithAttribute() throws Exception { + final String traitName = "PII_Trait" + randomString(); + HierarchicalTypeDefinition piiTrait = + TypesUtil.createTraitTypeDef(traitName, ImmutableList.of(), + TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE)); + String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true); + LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON); + createType(traitDefinitionAsJSON); + + Struct traitInstance = new Struct(traitName); + traitInstance.set("type", "SSN"); + String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true); + LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON); + + final String guid = tableId._getId(); + ClientResponse clientResponse = service + .path("api/atlas/entities") + .path(guid) + .path(TRAITS) + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + Assert.assertNotNull(response.get(MetadataServiceClient.GUID)); + + // verify the response + clientResponse = getEntityDefinition(guid); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + final String definition = response.getString(MetadataServiceClient.DEFINITION); + Assert.assertNotNull(definition); + Referenceable entityRef = InstanceSerialization.fromJsonReferenceable(definition, true); + IStruct traitRef = entityRef.getTrait(traitName); + String type = (String) traitRef.get("type"); + Assert.assertEquals(type, "SSN"); + } + + @Test + public void testAddTraitWithNoRegistration() throws Exception { + final String traitName = "PII_Trait" + randomString(); + HierarchicalTypeDefinition piiTrait = + TypesUtil.createTraitTypeDef(traitName, ImmutableList.of()); + String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true); + LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON); + + Struct traitInstance = new Struct(traitName); + String traitInstanceAsJSON = InstanceSerialization$.MODULE$.toJson(traitInstance, true); + LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON); + + ClientResponse clientResponse = service + .path("api/atlas/entities") + .path("random") + .path(TRAITS) + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode()); + } + + @Test(dependsOnMethods = "testAddTrait") + public void testDeleteTrait() throws Exception { + final String guid = tableId._getId(); + + ClientResponse clientResponse = service + .path("api/atlas/entities") + .path(guid) + .path(TRAITS) + .path(traitName) + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.DELETE, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + Assert.assertNotNull(response.get("GUID")); + Assert.assertNotNull(response.get("traitName")); + } + + @Test + public void testDeleteTraitNonExistent() throws Exception { + final String traitName = "blah_trait"; + + ClientResponse clientResponse = service + .path("api/atlas/entities") + .path("random") + .path(TRAITS) + .path(traitName) + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.DELETE, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.ERROR)); + Assert.assertEquals(response.getString(MetadataServiceClient.ERROR), + "trait=" + traitName + " should be defined in type system before it can be deleted"); + Assert.assertNotNull(response.get(MetadataServiceClient.STACKTRACE)); + } + + private String random() { + return RandomStringUtils.random(10); + } + + private String randomString() { + return RandomStringUtils.randomAlphanumeric(10); + } + + @Test + public void testUTF8() throws Exception { + String classType = random(); + String attrName = random(); + String attrValue = random(); + + HierarchicalTypeDefinition classTypeDefinition = + TypesUtil.createClassTypeDef(classType, ImmutableList.of(), + TypesUtil.createUniqueRequiredAttrDef(attrName, DataTypes.STRING_TYPE)); + TypesDef typesDef = TypeUtils.getTypesDef(ImmutableList.of(), + ImmutableList.of(), ImmutableList.>of(), + ImmutableList.of(classTypeDefinition)); + createType(typesDef); + + Referenceable instance = new Referenceable(classType); + instance.set(attrName, attrValue); + Id guid = createInstance(instance); + + ClientResponse response = getEntityDefinition(guid._getId()); + String definition = getEntityDefinition(response); + Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true); + Assert.assertEquals(getReferenceable.get(attrName), attrValue); + } + + private void createHiveTypes() throws Exception { + HierarchicalTypeDefinition databaseTypeDefinition = + TypesUtil.createClassTypeDef(DATABASE_TYPE, + ImmutableList.of(), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE)); + + StructTypeDefinition structTypeDefinition = + new StructTypeDefinition("serdeType", + new AttributeDefinition[]{ + TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("serde", DataTypes.STRING_TYPE) + }); + + EnumValue values[] = { + new EnumValue("MANAGED", 1), + new EnumValue("EXTERNAL", 2), + }; + + EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("tableType", values); + + HierarchicalTypeDefinition tableTypeDefinition = + TypesUtil.createClassTypeDef(TABLE_TYPE, + ImmutableList.of(), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), + TypesUtil.createOptionalAttrDef("description", DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("date", DataTypes.DATE_TYPE), + TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE), + new AttributeDefinition("tableType", "tableType", + Multiplicity.REQUIRED, false, null), + new AttributeDefinition("serde1", + "serdeType", Multiplicity.REQUIRED, false, null), + new AttributeDefinition("serde2", + "serdeType", Multiplicity.REQUIRED, false, null), + new AttributeDefinition("database", + DATABASE_TYPE, Multiplicity.REQUIRED, true, null), + new AttributeDefinition("compressed", + DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, true, null)); + + HierarchicalTypeDefinition classificationTraitDefinition = + TypesUtil.createTraitTypeDef("classification", + ImmutableList.of(), + TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition piiTrait = + TypesUtil.createTraitTypeDef("pii", ImmutableList.of()); + HierarchicalTypeDefinition phiTrait = + TypesUtil.createTraitTypeDef("phi", ImmutableList.of()); + HierarchicalTypeDefinition pciTrait = + TypesUtil.createTraitTypeDef("pci", ImmutableList.of()); + HierarchicalTypeDefinition soxTrait = + TypesUtil.createTraitTypeDef("sox", ImmutableList.of()); + HierarchicalTypeDefinition secTrait = + TypesUtil.createTraitTypeDef("sec", ImmutableList.of()); + HierarchicalTypeDefinition financeTrait = + TypesUtil.createTraitTypeDef("finance", ImmutableList.of()); + + TypesDef typesDef = TypeUtils.getTypesDef( + ImmutableList.of(enumTypeDefinition), + ImmutableList.of(structTypeDefinition), + ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait, + soxTrait, secTrait, financeTrait), + ImmutableList.of(databaseTypeDefinition, tableTypeDefinition)); + createType(typesDef); + } + + private Referenceable createHiveTableInstance() throws Exception { + Referenceable databaseInstance = new Referenceable(DATABASE_TYPE); + databaseInstance.set("name", DATABASE_NAME); + databaseInstance.set("description", "foo database"); + + Referenceable tableInstance = new Referenceable(TABLE_TYPE, + "classification", "pii", "phi", "pci", "sox", "sec", "finance"); + tableInstance.set("name", TABLE_NAME); + tableInstance.set("description", "bar table"); + tableInstance.set("date", "2014-07-11T08:00:00.000Z"); + tableInstance.set("type", "managed"); + tableInstance.set("level", 2); + tableInstance.set("tableType", 1); // enum + tableInstance.set("database", databaseInstance); + tableInstance.set("compressed", false); + + Struct traitInstance = (Struct) tableInstance.getTrait("classification"); + traitInstance.set("tag", "foundation_etl"); + + Struct serde1Instance = new Struct("serdeType"); + serde1Instance.set("name", "serde1"); + serde1Instance.set("serde", "serde1"); + tableInstance.set("serde1", serde1Instance); + + Struct serde2Instance = new Struct("serdeType"); + serde2Instance.set("name", "serde2"); + serde2Instance.set("serde", "serde2"); + tableInstance.set("serde2", serde2Instance); + + List traits = tableInstance.getTraits(); + Assert.assertEquals(traits.size(), 7); + + return tableInstance; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/resources/HiveLineageJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/HiveLineageJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/HiveLineageJerseyResourceIT.java new file mode 100644 index 0000000..421ce15 --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/resources/HiveLineageJerseyResourceIT.java @@ -0,0 +1,399 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.resources; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import org.apache.atlas.MetadataServiceClient; +import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.persistence.Id; +import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.EnumTypeDefinition; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.IDataType; +import org.apache.atlas.typesystem.types.Multiplicity; +import org.apache.atlas.typesystem.types.StructTypeDefinition; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeUtils; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.apache.atlas.web.util.Servlets; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONObject; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.Response; +import java.util.List; + +/** + * Hive Lineage Integration Tests. + */ +public class HiveLineageJerseyResourceIT extends BaseResourceIT { + + private static final String BASE_URI = "api/atlas/lineage/hive/table/"; + + @BeforeClass + public void setUp() throws Exception { + super.setUp(); + + setUpTypes(); + setupInstances(); + } + + @Test + public void testInputsGraph() throws Exception { + WebResource resource = service + .path(BASE_URI) + .path("sales_fact_monthly_mv") + .path("inputs") + .path("graph"); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + System.out.println("inputs graph = " + responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS); + Assert.assertNotNull(results); + + JSONObject values = results.getJSONObject("values"); + Assert.assertNotNull(values); + + final JSONObject vertices = values.getJSONObject("vertices"); + Assert.assertEquals(vertices.length(), 4); + + final JSONObject edges = values.getJSONObject("edges"); + Assert.assertEquals(edges.length(), 4); + } + + @Test + public void testOutputsGraph() throws Exception { + WebResource resource = service + .path(BASE_URI) + .path("sales_fact") + .path("outputs") + .path("graph"); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + System.out.println("outputs graph= " + responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS); + Assert.assertNotNull(results); + + JSONObject values = results.getJSONObject("values"); + Assert.assertNotNull(values); + + final JSONObject vertices = values.getJSONObject("vertices"); + Assert.assertEquals(vertices.length(), 3); + + final JSONObject edges = values.getJSONObject("edges"); + Assert.assertEquals(edges.length(), 4); + } + + @Test + public void testSchema() throws Exception { + WebResource resource = service + .path(BASE_URI) + .path("sales_fact") + .path("schema"); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + System.out.println("schema = " + responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS); + Assert.assertNotNull(results); + + JSONArray rows = results.getJSONArray("rows"); + Assert.assertEquals(rows.length(), 4); + + for (int index = 0; index < rows.length(); index++) { + final JSONObject row = rows.getJSONObject(index); + Assert.assertNotNull(row.getString("name")); + Assert.assertNotNull(row.getString("comment")); + Assert.assertNotNull(row.getString("dataType")); + Assert.assertEquals(row.getString("$typeName$"), "hive_column"); + } + } + + @Test + public void testSchemaForEmptyTable() throws Exception { + WebResource resource = service + .path(BASE_URI) + .path("") + .path("schema"); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), + Response.Status.NOT_FOUND.getStatusCode()); + } + + @Test + public void testSchemaForInvalidTable() throws Exception { + WebResource resource = service + .path(BASE_URI) + .path("blah") + .path("schema"); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), + Response.Status.NOT_FOUND.getStatusCode()); + } + + private void setUpTypes() throws Exception { + TypesDef typesDef = createTypeDefinitions(); + createType(typesDef); + } + + private static final String DATABASE_TYPE = "hive_db"; + private static final String HIVE_TABLE_TYPE = "hive_table"; + private static final String COLUMN_TYPE = "hive_column"; + private static final String HIVE_PROCESS_TYPE = "hive_process"; + + private TypesDef createTypeDefinitions() { + HierarchicalTypeDefinition dbClsDef + = TypesUtil.createClassTypeDef(DATABASE_TYPE, null, + attrDef("name", DataTypes.STRING_TYPE), + attrDef("description", DataTypes.STRING_TYPE), + attrDef("locationUri", DataTypes.STRING_TYPE), + attrDef("owner", DataTypes.STRING_TYPE), + attrDef("createTime", DataTypes.INT_TYPE) + ); + + HierarchicalTypeDefinition columnClsDef = + TypesUtil.createClassTypeDef(COLUMN_TYPE, null, + attrDef("name", DataTypes.STRING_TYPE), + attrDef("dataType", DataTypes.STRING_TYPE), + attrDef("comment", DataTypes.STRING_TYPE) + ); + + HierarchicalTypeDefinition tblClsDef = + TypesUtil.createClassTypeDef(HIVE_TABLE_TYPE, ImmutableList.of("DataSet"), + attrDef("owner", DataTypes.STRING_TYPE), + attrDef("createTime", DataTypes.INT_TYPE), + attrDef("lastAccessTime", DataTypes.INT_TYPE), + attrDef("tableType", DataTypes.STRING_TYPE), + attrDef("temporary", DataTypes.BOOLEAN_TYPE), + new AttributeDefinition("db", DATABASE_TYPE, + Multiplicity.REQUIRED, false, null), + new AttributeDefinition("columns", + DataTypes.arrayTypeName(COLUMN_TYPE), + Multiplicity.COLLECTION, true, null) + ); + + HierarchicalTypeDefinition loadProcessClsDef = + TypesUtil.createClassTypeDef(HIVE_PROCESS_TYPE, ImmutableList.of("Process"), + attrDef("userName", DataTypes.STRING_TYPE), + attrDef("startTime", DataTypes.INT_TYPE), + attrDef("endTime", DataTypes.INT_TYPE), + attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), + attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), + attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED), + attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED) + ); + + HierarchicalTypeDefinition dimTraitDef = + TypesUtil.createTraitTypeDef("Dimension", null); + + HierarchicalTypeDefinition factTraitDef = + TypesUtil.createTraitTypeDef("Fact", null); + + HierarchicalTypeDefinition metricTraitDef = + TypesUtil.createTraitTypeDef("Metric", null); + + HierarchicalTypeDefinition etlTraitDef = + TypesUtil.createTraitTypeDef("ETL", null); + + + HierarchicalTypeDefinition piiTraitDef = + TypesUtil.createTraitTypeDef("PII", null); + + return TypeUtils.getTypesDef( + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(dimTraitDef, factTraitDef, metricTraitDef, etlTraitDef, piiTraitDef), + ImmutableList.of(dbClsDef, columnClsDef, tblClsDef, loadProcessClsDef) + ); + } + + AttributeDefinition attrDef(String name, IDataType dT) { + return attrDef(name, dT, Multiplicity.OPTIONAL, false, null); + } + + AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) { + return attrDef(name, dT, m, false, null); + } + + AttributeDefinition attrDef(String name, IDataType dT, + Multiplicity m, boolean isComposite, String reverseAttributeName) { + Preconditions.checkNotNull(name); + Preconditions.checkNotNull(dT); + return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName); + } + + private void setupInstances() throws Exception { + Id salesDB = database( + "Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales"); + + List salesFactColumns = ImmutableList.of( + column("time_id", "int", "time id"), + column("product_id", "int", "product id"), + column("customer_id", "int", "customer id", "PII"), + column("sales", "double", "product id", "Metric") + ); + + Id salesFact = table("sales_fact", "sales fact table", + salesDB, "Joe", "Managed", salesFactColumns, "Fact"); + + List timeDimColumns = ImmutableList.of( + column("time_id", "int", "time id"), + column("dayOfYear", "int", "day Of Year"), + column("weekDay", "int", "week Day") + ); + + Id timeDim = table("time_dim", "time dimension table", + salesDB, "John Doe", "External", timeDimColumns, "Dimension"); + + Id reportingDB = database("Reporting", "reporting database", "Jane BI", + "hdfs://host:8000/apps/warehouse/reporting"); + + Id salesFactDaily = table("sales_fact_daily_mv", + "sales fact daily materialized view", + reportingDB, "Joe BI", "Managed", salesFactColumns, "Metric"); + + loadProcess("loadSalesDaily", "John ETL", + ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily), + "create table as select ", "plan", "id", "graph", + "ETL"); + + Id salesFactMonthly = table("sales_fact_monthly_mv", + "sales fact monthly materialized view", + reportingDB, "Jane BI", "Managed", salesFactColumns, "Metric"); + + loadProcess("loadSalesMonthly", "John ETL", + ImmutableList.of(salesFactDaily), ImmutableList.of(salesFactMonthly), + "create table as select ", "plan", "id", "graph", + "ETL"); + } + + Id database(String name, String description, + String owner, String locationUri, + String... traitNames) throws Exception { + Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames); + referenceable.set("name", name); + referenceable.set("description", description); + referenceable.set("owner", owner); + referenceable.set("locationUri", locationUri); + referenceable.set("createTime", System.currentTimeMillis()); + + return createInstance(referenceable); + } + + Referenceable column(String name, String dataType, String comment, + String... traitNames) throws Exception { + Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames); + referenceable.set("name", name); + referenceable.set("dataType", dataType); + referenceable.set("comment", comment); + + return referenceable; + } + + Id table(String name, String description, Id dbId, + String owner, String tableType, + List columns, + String... traitNames) throws Exception { + Referenceable referenceable = new Referenceable(HIVE_TABLE_TYPE, traitNames); + referenceable.set("name", name); + referenceable.set("description", description); + referenceable.set("owner", owner); + referenceable.set("tableType", tableType); + referenceable.set("createTime", System.currentTimeMillis()); + referenceable.set("lastAccessTime", System.currentTimeMillis()); + referenceable.set("retention", System.currentTimeMillis()); + + referenceable.set("db", dbId); + referenceable.set("columns", columns); + + return createInstance(referenceable); + } + + Id loadProcess(String name, String user, + List inputTables, + List outputTables, + String queryText, String queryPlan, + String queryId, String queryGraph, + String... traitNames) throws Exception { + Referenceable referenceable = new Referenceable(HIVE_PROCESS_TYPE, traitNames); + referenceable.set("name", name); + referenceable.set("user", user); + referenceable.set("startTime", System.currentTimeMillis()); + referenceable.set("endTime", System.currentTimeMillis() + 10000); + + referenceable.set("inputs", inputTables); + referenceable.set("outputs", outputTables); + + referenceable.set("queryText", queryText); + referenceable.set("queryPlan", queryPlan); + referenceable.set("queryId", queryId); + referenceable.set("queryGraph", queryGraph); + + return createInstance(referenceable); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java new file mode 100755 index 0000000..cf27ac2 --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/resources/MetadataDiscoveryJerseyResourceIT.java @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.resources; + +import com.google.common.collect.ImmutableList; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import org.apache.atlas.MetadataServiceClient; +import org.apache.atlas.typesystem.Referenceable; +import org.apache.atlas.typesystem.Struct; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.persistence.Id; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.EnumTypeDefinition; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.StructTypeDefinition; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.TypeUtils; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.apache.atlas.web.util.Servlets; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONObject; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.Response; +import java.util.List; + +/** + * Search Integration Tests. + */ +public class MetadataDiscoveryJerseyResourceIT extends BaseResourceIT { + + @BeforeClass + public void setUp() throws Exception { + super.setUp(); + + createTypes(); + createInstance(); + } + + @Test + public void testSearchByDSL() throws Exception { + String dslQuery = "from dsl_test_type"; + WebResource resource = service + .path("api/atlas/discovery/search/dsl") + .queryParam("query", dslQuery); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + Assert.assertEquals(response.getString("query"), dslQuery); + Assert.assertEquals(response.getString("queryType"), "dsl"); + + JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS); + Assert.assertNotNull(results); + + JSONArray rows = results.getJSONArray(MetadataServiceClient.ROWS); + Assert.assertEquals(rows.length(), 1); + + int numRows = response.getInt(MetadataServiceClient.COUNT); + Assert.assertEquals(numRows, 1); + + } + + @Test + public void testSearchByDSLForUnknownType() throws Exception { + String dslQuery = "from blah"; + WebResource resource = service + .path("api/atlas/discovery/search/dsl") + .queryParam("query", dslQuery); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), + Response.Status.BAD_REQUEST.getStatusCode()); + } + + @Test + public void testSearchUsingGremlin() throws Exception { + String query = "g.V.has('type', 'dsl_test_type').toList()"; + WebResource resource = service + .path("api/atlas/discovery/search") + .queryParam("query", query); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + Assert.assertEquals(response.getString("query"), query); + Assert.assertEquals(response.getString("queryType"), "gremlin"); + } + + @Test + public void testSearchUsingDSL() throws Exception { + String query = "from dsl_test_type"; + WebResource resource = service + .path("api/atlas/discovery/search") + .queryParam("query", query); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + Assert.assertEquals(response.getString("query"), query); + Assert.assertEquals(response.getString("queryType"), "dsl"); + } + + @Test(enabled = false) + public void testSearchUsingFullText() throws Exception { + String query = "foundation_etl"; + JSONObject response = serviceClient.searchByFullText(query); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + Assert.assertEquals(response.getString("query"), query); + Assert.assertEquals(response.getString("queryType"), "full-text"); + + JSONArray results = response.getJSONArray(MetadataServiceClient.RESULTS); + Assert.assertEquals(results.length(), 1); + + JSONObject row = results.getJSONObject(0); + Assert.assertNotNull(row.get("guid")); + Assert.assertEquals(row.getString("typeName"), "dsl_test_type"); + Assert.assertNotNull(row.get("score")); + + int numRows = response.getInt(MetadataServiceClient.COUNT); + Assert.assertEquals(numRows, 1); + } + + private void createTypes() throws Exception { + HierarchicalTypeDefinition dslTestTypeDefinition = + TypesUtil.createClassTypeDef("dsl_test_type", + ImmutableList.of(), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE)); + + HierarchicalTypeDefinition classificationTraitDefinition = + TypesUtil.createTraitTypeDef("Classification", + ImmutableList.of(), + TypesUtil.createRequiredAttrDef("tag", DataTypes.STRING_TYPE)); + HierarchicalTypeDefinition piiTrait = + TypesUtil.createTraitTypeDef("PII_TYPE", ImmutableList.of()); + HierarchicalTypeDefinition phiTrait = + TypesUtil.createTraitTypeDef("PHI", ImmutableList.of()); + HierarchicalTypeDefinition pciTrait = + TypesUtil.createTraitTypeDef("PCI", ImmutableList.of()); + HierarchicalTypeDefinition soxTrait = + TypesUtil.createTraitTypeDef("SOX", ImmutableList.of()); + HierarchicalTypeDefinition secTrait = + TypesUtil.createTraitTypeDef("SEC", ImmutableList.of()); + HierarchicalTypeDefinition financeTrait = + TypesUtil.createTraitTypeDef("Finance", ImmutableList.of()); + + TypesDef typesDef = TypeUtils.getTypesDef( + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(classificationTraitDefinition, piiTrait, phiTrait, pciTrait, + soxTrait, secTrait, financeTrait), + ImmutableList.of(dslTestTypeDefinition)); + createType(typesDef); + } + + private Id createInstance() throws Exception { + Referenceable entityInstance = new Referenceable("dsl_test_type", + "Classification", "PII_TYPE", "PHI", "PCI", "SOX", "SEC", "Finance"); + entityInstance.set("name", "foo name"); + entityInstance.set("description", "bar description"); + + Struct traitInstance = (Struct) entityInstance.getTrait("Classification"); + traitInstance.set("tag", "foundation_etl"); + + List traits = entityInstance.getTraits(); + Assert.assertEquals(traits.size(), 7); + + return createInstance(entityInstance); + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/resources/RexsterGraphJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/RexsterGraphJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/RexsterGraphJerseyResourceIT.java new file mode 100755 index 0000000..e3f29ae --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/resources/RexsterGraphJerseyResourceIT.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.resources; + +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import org.apache.atlas.web.util.Servlets; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.Response; + +/** + * Integration tests for Rexster Graph Jersey Resource. + */ +@Test +public class RexsterGraphJerseyResourceIT extends BaseResourceIT { + + @BeforeClass + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @Test(enabled = false) + public void testGetVertex() throws Exception { + // todo: add a vertex before fetching it + + WebResource resource = service + .path("api/atlas/graph/vertices") + .path("0"); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + String response = clientResponse.getEntity(String.class); + Assert.assertNotNull(response); + } + + public void testGetVertexWithInvalidId() throws Exception { + WebResource resource = service + .path("api/atlas/graph/vertices/blah"); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + } + + public void testGetVertexProperties() throws Exception { + + } + + public void testGetVertices() throws Exception { + + } + + public void testGetVertexEdges() throws Exception { + + } + + public void testGetEdge() throws Exception { + + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java new file mode 100755 index 0000000..bd3cbed --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java @@ -0,0 +1,235 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.resources; + +import com.google.common.collect.ImmutableList; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import org.apache.atlas.MetadataServiceClient; +import org.apache.atlas.typesystem.TypesDef; +import org.apache.atlas.typesystem.json.TypesSerialization; +import org.apache.atlas.typesystem.json.TypesSerialization$; +import org.apache.atlas.typesystem.types.AttributeDefinition; +import org.apache.atlas.typesystem.types.ClassType; +import org.apache.atlas.typesystem.types.DataTypes; +import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition; +import org.apache.atlas.typesystem.types.Multiplicity; +import org.apache.atlas.typesystem.types.TraitType; +import org.apache.atlas.typesystem.types.utils.TypesUtil; +import org.apache.atlas.web.util.Servlets; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONObject; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.Response; +import java.util.ArrayList; +import java.util.List; + +/** + * Integration test for types jersey resource. + */ +public class TypesJerseyResourceIT extends BaseResourceIT { + + private List typeDefinitions; + + @BeforeClass + public void setUp() throws Exception { + super.setUp(); + + typeDefinitions = createHiveTypes(); + } + + @AfterClass + public void tearDown() throws Exception { + typeDefinitions.clear(); + } + + @Test + public void testSubmit() throws Exception { + for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) { + String typesAsJSON = TypesSerialization.toJson(typeDefinition); + System.out.println("typesAsJSON = " + typesAsJSON); + + WebResource resource = service + .path("api/atlas/types"); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.POST, ClientResponse.class, typesAsJSON); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + JSONArray typesAdded = response.getJSONArray(MetadataServiceClient.TYPES); + Assert.assertEquals(typesAdded.length(), 1); + Assert.assertEquals(typesAdded.getJSONObject(0).getString("name"), typeDefinition.typeName); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + } + } + + @Test(dependsOnMethods = "testSubmit") + public void testGetDefinition() throws Exception { + for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) { + System.out.println("typeName = " + typeDefinition.typeName); + + WebResource resource = service + .path("api/atlas/types") + .path(typeDefinition.typeName); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.DEFINITION)); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + String typesJson = response.getString(MetadataServiceClient.DEFINITION); + final TypesDef typesDef = TypesSerialization.fromJson(typesJson); + List> hierarchicalTypeDefinitions = typesDef.classTypesAsJavaList(); + for(HierarchicalTypeDefinition classType : hierarchicalTypeDefinitions) { + for(AttributeDefinition attrDef : classType.attributeDefinitions) { + if("name".equals(attrDef.name)) { + Assert.assertEquals(attrDef.isIndexable, true); + Assert.assertEquals(attrDef.isUnique, true); + } + } + } + } + } + + @Test + public void testGetDefinitionForNonexistentType() throws Exception { + WebResource resource = service + .path("api/atlas/types") + .path("blah"); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + } + + @Test(dependsOnMethods = "testSubmit") + public void testGetTypeNames() throws Exception { + WebResource resource = service + .path("api/atlas/types"); + + ClientResponse clientResponse = resource + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS); + Assert.assertNotNull(list); + } + + @Test + public void testGetTraitNames() throws Exception { + String[] traitsAdded = addTraits(); + + WebResource resource = service + .path("api/atlas/types"); + + ClientResponse clientResponse = resource + .queryParam("type", DataTypes.TypeCategory.TRAIT.name()) + .accept(Servlets.JSON_MEDIA_TYPE) + .type(Servlets.JSON_MEDIA_TYPE) + .method(HttpMethod.GET, ClientResponse.class); + Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode()); + + String responseAsString = clientResponse.getEntity(String.class); + Assert.assertNotNull(responseAsString); + + JSONObject response = new JSONObject(responseAsString); + Assert.assertNotNull(response.get(MetadataServiceClient.REQUEST_ID)); + + final JSONArray list = response.getJSONArray(MetadataServiceClient.RESULTS); + Assert.assertNotNull(list); + Assert.assertTrue(list.length() >= traitsAdded.length); + } + + private String[] addTraits() throws Exception { + String[] traitNames = { + "class_trait", + "secure_trait", + "pii_trait", + "ssn_trait", + "salary_trait", + "sox_trait", + }; + + for (String traitName : traitNames) { + HierarchicalTypeDefinition traitTypeDef = + TypesUtil.createTraitTypeDef(traitName, ImmutableList.of()); + String json = TypesSerialization$.MODULE$.toJson(traitTypeDef, true); + createType(json); + } + + return traitNames; + } + + private List createHiveTypes() throws Exception { + ArrayList typeDefinitions = new ArrayList<>(); + + HierarchicalTypeDefinition databaseTypeDefinition = + TypesUtil.createClassTypeDef("database", + ImmutableList.of(), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE)); + typeDefinitions.add(databaseTypeDefinition); + + HierarchicalTypeDefinition tableTypeDefinition = TypesUtil.createClassTypeDef( + "table", + ImmutableList.of(), + TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE), + TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE), + new AttributeDefinition("database", + "database", Multiplicity.REQUIRED, false, "database")); + typeDefinitions.add(tableTypeDefinition); + + HierarchicalTypeDefinition fetlTypeDefinition = TypesUtil.createTraitTypeDef( + "fetl", + ImmutableList.of(), + TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE)); + typeDefinitions.add(fetlTypeDefinition); + + return typeDefinitions; + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java new file mode 100644 index 0000000..ee06cdc --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerIT.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.atlas.web.service; + +import org.apache.commons.configuration.PropertiesConfiguration; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.net.HttpURLConnection; +import java.net.URL; + +import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH; + +public class SecureEmbeddedServerIT extends SecureEmbeddedServerITBase{ + @Test + public void testServerConfiguredUsingCredentialProvider() throws Exception { + // setup the configuration + final PropertiesConfiguration configuration = new PropertiesConfiguration(); + configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl); + // setup the credential provider + setupCredentials(); + + SecureEmbeddedServer secureEmbeddedServer = null; + try { + String appPath = System.getProperty("user.dir") + getWarPath(); + secureEmbeddedServer = new SecureEmbeddedServer(21443, appPath) { + @Override + protected PropertiesConfiguration getConfiguration() { + return configuration; + } + }; + secureEmbeddedServer.server.start(); + + URL url = new URL("https://localhost:21443/"); + HttpURLConnection connection = (HttpURLConnection)url.openConnection(); + connection.setRequestMethod("GET"); + connection.connect(); + + // test to see whether server is up and root page can be served + Assert.assertEquals(connection.getResponseCode(), 200); + } finally { + secureEmbeddedServer.server.stop(); + } + + } +} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java new file mode 100755 index 0000000..0f3286f --- /dev/null +++ b/webapp/src/test/java/org/apache/atlas/web/service/SecureEmbeddedServerITBase.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.web.service; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import org.apache.atlas.web.resources.AdminJerseyResourceIT; +import org.apache.atlas.web.resources.BaseResourceIT; +import org.apache.atlas.web.resources.EntityJerseyResourceIT; +import org.apache.atlas.web.resources.MetadataDiscoveryJerseyResourceIT; +import org.apache.atlas.web.resources.RexsterGraphJerseyResourceIT; +import org.apache.atlas.web.resources.TypesJerseyResourceIT; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.security.alias.JavaKeyStoreProvider; +import org.mortbay.jetty.webapp.WebAppContext; +import org.testng.Assert; +import org.testng.TestListenerAdapter; +import org.testng.TestNG; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import javax.ws.rs.core.UriBuilder; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +import static org.apache.atlas.security.SecurityProperties.CERT_STORES_CREDENTIAL_PROVIDER_PATH; +import static org.apache.atlas.security.SecurityProperties.DEFAULT_KEYSTORE_FILE_LOCATION; +import static org.apache.atlas.security.SecurityProperties.KEYSTORE_PASSWORD_KEY; +import static org.apache.atlas.security.SecurityProperties.SERVER_CERT_PASSWORD_KEY; +import static org.apache.atlas.security.SecurityProperties.TRUSTSTORE_PASSWORD_KEY; + +/** + * Secure Test class for jersey resources. + */ +public class SecureEmbeddedServerITBase { + + + private SecureEmbeddedServer secureEmbeddedServer; + protected String providerUrl; + private Path jksPath; + protected WebResource service; + + static { + //for localhost testing only + javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier( + new javax.net.ssl.HostnameVerifier(){ + + public boolean verify(String hostname, + javax.net.ssl.SSLSession sslSession) { + if (hostname.equals("localhost")) { + return true; + } + return false; + } + }); + System.setProperty("javax.net.ssl.trustStore", DEFAULT_KEYSTORE_FILE_LOCATION); + System.setProperty("javax.net.ssl.trustStorePassword", "keypass"); + System.setProperty("javax.net.ssl.trustStoreType", "JKS"); + } + + @BeforeClass + public void setupServerURI () throws Exception { + BaseResourceIT.baseUrl = "https://localhost:21443"; + } + + @AfterClass + public void resetServerURI() throws Exception { + BaseResourceIT.baseUrl = "http://localhost:21000"; + } + + @BeforeMethod + public void setup() throws Exception { + jksPath = new Path(Files.createTempDirectory("tempproviders").toString(), "test.jks"); + providerUrl = JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri(); + + String baseUrl = "https://localhost:21443/"; + + DefaultClientConfig config = new DefaultClientConfig(); + Client client = Client.create(config); + client.resource(UriBuilder.fromUri(baseUrl).build()); + + service = client.resource(UriBuilder.fromUri(baseUrl).build()); + } + + @Test + public void testNoConfiguredCredentialProvider() throws Exception { + + try { + secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/apache-atlas"); + WebAppContext webapp = new WebAppContext(); + webapp.setContextPath("/"); + webapp.setWar(System.getProperty("user.dir") + getWarPath()); + secureEmbeddedServer.server.setHandler(webapp); + + secureEmbeddedServer.server.start(); + + Assert.fail("Should have thrown an exception"); + } catch (IOException e) { + Assert.assertEquals("No credential provider path configured for storage of certificate store passwords", e.getMessage()); + } finally { + secureEmbeddedServer.server.stop(); + } + } + + @Test + public void testMissingEntriesInCredentialProvider() throws Exception { + // setup the configuration + final PropertiesConfiguration configuration = new PropertiesConfiguration(); + configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl); + + try { + secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/apache-atlas") { + @Override + protected PropertiesConfiguration getConfiguration() { + return configuration; + } + }; + Assert.fail("No entries should generate an exception"); + } catch (IOException e) { + Assert.assertTrue(e.getMessage().startsWith("No credential entry found for")); + } finally { + secureEmbeddedServer.server.stop(); + } + + } + + /** + * Runs the existing webapp test cases, this time against the initiated secure server instance. + * @throws Exception + */ + @Test + public void runOtherSuitesAgainstSecureServer() throws Exception { + final PropertiesConfiguration configuration = new PropertiesConfiguration(); + configuration.setProperty(CERT_STORES_CREDENTIAL_PROVIDER_PATH, providerUrl); + // setup the credential provider + setupCredentials(); + + try { + secureEmbeddedServer = new SecureEmbeddedServer(21443, "webapp/target/apache-atlas") { + @Override + protected PropertiesConfiguration getConfiguration() { + return configuration; + } + }; + WebAppContext webapp = new WebAppContext(); + webapp.setContextPath("/"); + webapp.setWar(System.getProperty("user.dir") + getWarPath()); + secureEmbeddedServer.server.setHandler(webapp); + + secureEmbeddedServer.server.start(); + + TestListenerAdapter tla = new TestListenerAdapter(); + TestNG testng = new TestNG(); + testng.setTestClasses(new Class[] { AdminJerseyResourceIT.class, EntityJerseyResourceIT.class, + MetadataDiscoveryJerseyResourceIT.class, RexsterGraphJerseyResourceIT.class, TypesJerseyResourceIT.class}); + testng.addListener(tla); + testng.run(); + + } finally { + secureEmbeddedServer.server.stop(); + } + + } + + protected String getWarPath() { + return String.format("/target/atlas-webapp-%s", + System.getProperty("project.version", "0.1-incubating-SNAPSHOT")); + } + + protected void setupCredentials() throws Exception { + Configuration conf = new Configuration(false); + + File file = new File(jksPath.toUri().getPath()); + file.delete(); + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerUrl); + + CredentialProvider provider = + CredentialProviderFactory.getProviders(conf).get(0); + + // create new aliases + try { + + char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'}; + provider.createCredentialEntry( + KEYSTORE_PASSWORD_KEY, storepass); + + char[] trustpass = {'k', 'e', 'y', 'p', 'a', 's', 's'}; + provider.createCredentialEntry( + TRUSTSTORE_PASSWORD_KEY, trustpass); + + char[] certpass = {'k', 'e', 'y', 'p', 'a', 's', 's'}; + provider.createCredentialEntry( + SERVER_CERT_PASSWORD_KEY, certpass); + + // write out so that it can be found in checks + provider.flush(); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + } +}