Return-Path: X-Original-To: apmail-asterixdb-commits-archive@minotaur.apache.org Delivered-To: apmail-asterixdb-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4F6CF182C4 for ; Wed, 25 Nov 2015 07:20:09 +0000 (UTC) Received: (qmail 31422 invoked by uid 500); 25 Nov 2015 07:20:09 -0000 Delivered-To: apmail-asterixdb-commits-archive@asterixdb.apache.org Received: (qmail 31388 invoked by uid 500); 25 Nov 2015 07:20:09 -0000 Mailing-List: contact commits-help@asterixdb.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.incubator.apache.org Delivered-To: mailing list commits@asterixdb.incubator.apache.org Received: (qmail 31378 invoked by uid 99); 25 Nov 2015 07:20:09 -0000 Received: from Unknown (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Nov 2015 07:20:09 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 9C38FC6F65 for ; Wed, 25 Nov 2015 07:20:08 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.218 X-Spam-Level: * X-Spam-Status: No, score=1.218 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-0.562] autolearn=disabled Received: from mx1-eu-west.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id OxNt_dx_2Vd7 for ; Wed, 25 Nov 2015 07:20:07 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-eu-west.apache.org (ASF Mail Server at mx1-eu-west.apache.org) with SMTP id B75C120229 for ; Wed, 25 Nov 2015 07:20:05 +0000 (UTC) Received: (qmail 31367 invoked by uid 99); 25 Nov 2015 07:20:04 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Nov 2015 07:20:04 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C8A33E03E8; Wed, 25 Nov 2015 07:20:04 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: buyingyi@apache.org To: commits@asterixdb.incubator.apache.org Date: Wed, 25 Nov 2015 07:20:04 -0000 Message-Id: <88d9baad845a4c8882a1cfa6206c345d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] incubator-asterixdb git commit: Fix ASTERIXDB-1194, ASTERIXDB-1195, ASTERIXDB-1196, ASTERIXDB-1197. Repository: incubator-asterixdb Updated Branches: refs/heads/master d7585c59b -> 751315f87 http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/751315f8/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java ---------------------------------------------------------------------- diff --git a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java index 0473f6c..52bad6f 100644 --- a/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java +++ b/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java @@ -25,14 +25,11 @@ import java.util.logging.Logger; import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil; import org.apache.asterix.common.api.IAsterixAppRuntimeContext; -import org.apache.asterix.common.config.AsterixPropertiesAccessor; -import org.apache.asterix.common.config.AsterixTransactionProperties; import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.external.dataset.adapter.FileSystemBasedAdapter; import org.apache.asterix.external.util.IdentitiyResolverFactory; import org.apache.asterix.testframework.xml.TestGroup; import org.apache.asterix.testframework.xml.TestSuite; -import org.apache.commons.io.FileUtils; import org.apache.hyracks.control.nc.NodeControllerService; import org.apache.hyracks.storage.common.buffercache.BufferCache; @@ -44,8 +41,6 @@ public class ExecutionTestUtil { protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration.xml"; - protected static AsterixTransactionProperties txnProperties; - protected static TestGroup FailedGroup; public static void setUp() throws Exception { @@ -55,15 +50,10 @@ public class ExecutionTestUtil { } System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, TEST_CONFIG_FILE_NAME); - AsterixPropertiesAccessor apa = new AsterixPropertiesAccessor(); - txnProperties = new AsterixTransactionProperties(apa); - - deleteTransactionLogs(); - if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("initializing pseudo cluster"); } - AsterixHyracksIntegrationUtil.init(); + AsterixHyracksIntegrationUtil.init(true); if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("initializing HDFS"); @@ -81,19 +71,11 @@ public class ExecutionTestUtil { FailedGroup.setName("failed"); } - private static void deleteTransactionLogs() throws Exception { - for (String ncId : AsterixHyracksIntegrationUtil.getNcNames()) { - File log = new File(txnProperties.getLogDirectory(ncId)); - if (log.exists()) { - FileUtils.deleteDirectory(log); - } - } - } - private static void validateBufferCacheState() { - for(NodeControllerService nc: AsterixHyracksIntegrationUtil.ncs){ - IAsterixAppRuntimeContext appCtx = (IAsterixAppRuntimeContext) nc.getApplicationContext().getApplicationObject(); - if(!((BufferCache)appCtx.getBufferCache()).isClean()){ + for (NodeControllerService nc : AsterixHyracksIntegrationUtil.ncs) { + IAsterixAppRuntimeContext appCtx = (IAsterixAppRuntimeContext) nc.getApplicationContext() + .getApplicationObject(); + if (!((BufferCache) appCtx.getBufferCache()).isClean()) { throw new IllegalStateException(); } } @@ -101,7 +83,7 @@ public class ExecutionTestUtil { public static void tearDown() throws Exception { validateBufferCacheState(); - AsterixHyracksIntegrationUtil.deinit(); + AsterixHyracksIntegrationUtil.deinit(true); File outdir = new File(PATH_ACTUAL); File[] files = outdir.listFiles(); if (files == null || files.length == 0) { http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/751315f8/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java ---------------------------------------------------------------------- diff --git a/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java b/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java index 46a4a4f..42f612a 100644 --- a/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java +++ b/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixCompilerProperties.java @@ -31,6 +31,9 @@ public class AsterixCompilerProperties extends AbstractAsterixProperties { private static final String COMPILER_FRAMESIZE_KEY = "compiler.framesize"; private static int COMPILER_FRAMESIZE_DEFAULT = (32 << 10); // 32KB + private static final String COMPILER_PREGELIX_HOME = "compiler.pregelix.home"; + private static final String COMPILER_PREGELIX_HOME_DEFAULT = "~/pregelix"; + public AsterixCompilerProperties(AsterixPropertiesAccessor accessor) { super(accessor); } @@ -55,4 +58,8 @@ public class AsterixCompilerProperties extends AbstractAsterixProperties { PropertyInterpreters.getIntegerPropertyInterpreter()); } + public String getPregelixHome() { + return accessor.getProperty(COMPILER_PREGELIX_HOME, COMPILER_PREGELIX_HOME_DEFAULT, + PropertyInterpreters.getStringPropertyInterpreter()); + } } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/751315f8/asterix-installer/src/main/resources/conf/asterix-configuration.xml ---------------------------------------------------------------------- diff --git a/asterix-installer/src/main/resources/conf/asterix-configuration.xml b/asterix-installer/src/main/resources/conf/asterix-configuration.xml index ab64cae..622ed3a 100644 --- a/asterix-installer/src/main/resources/conf/asterix-configuration.xml +++ b/asterix-installer/src/main/resources/conf/asterix-configuration.xml @@ -219,6 +219,11 @@ + + compiler.pregelix.home + ~/pregelix + + web.port 19001 http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/751315f8/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockManager.java ---------------------------------------------------------------------- diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockManager.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockManager.java index 867ce75..5d345b9 100644 --- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockManager.java +++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockManager.java @@ -225,7 +225,7 @@ public class MetadataLockManager { public void releaseFeedWriteLock(String feedName) { feedsLocks.get(feedName).writeLock().unlock(); } - + public void acquireFeedPolicyWriteLock(String policyName) { ReentrantReadWriteLock fLock = feedPolicyLocks.get(policyName); if (fLock == null) { @@ -444,7 +444,7 @@ public class MetadataLockManager { releaseFeedWriteLock(feedFullyQualifiedName); releaseDataverseReadLock(dataverseName); } - + public void dropFeedPolicyBegin(String dataverseName, String policyName) { releaseFeedWriteLock(policyName); releaseDataverseReadLock(dataverseName); @@ -486,7 +486,7 @@ public class MetadataLockManager { releaseFeedPolicyWriteLock(policyName); releaseDataverseReadLock(dataverseName); } - + public void disconnectFeedBegin(String dataverseName, String datasetFullyQualifiedName, String feedFullyQualifiedName) { acquireDataverseReadLock(dataverseName); @@ -499,14 +499,13 @@ public class MetadataLockManager { releaseDatasetReadLock(datasetFullyQualifiedName); releaseDataverseReadLock(dataverseName); } - - public void subscribeFeedBegin(String dataverseName, String datasetFullyQualifiedName, - String feedFullyQualifiedName) { + + public void subscribeFeedBegin(String dataverseName, String datasetFullyQualifiedName, String feedFullyQualifiedName) { acquireDataverseReadLock(dataverseName); acquireDatasetReadLock(datasetFullyQualifiedName); acquireFeedReadLock(feedFullyQualifiedName); } - + public void subscribeFeedEnd(String dataverseName, String datasetFullyQualifiedName, String feedFullyQualifiedName) { releaseFeedReadLock(feedFullyQualifiedName); releaseDatasetReadLock(datasetFullyQualifiedName); @@ -575,25 +574,4 @@ public class MetadataLockManager { releaseExternalDatasetRefreshLock(datasetFullyQualifiedName); releaseDataverseReadLock(dataverseName); } - - public void pregelixBegin(String dataverseName, String datasetFullyQualifiedNameFrom, - String datasetFullyQualifiedNameTo) { - acquireDataverseReadLock(dataverseName); - - if (datasetFullyQualifiedNameFrom.compareTo(datasetFullyQualifiedNameTo) < 0) { - acquireDatasetReadLock(datasetFullyQualifiedNameFrom); - acquireDatasetWriteLock(datasetFullyQualifiedNameTo); - } else { - acquireDatasetWriteLock(datasetFullyQualifiedNameTo); - acquireDatasetReadLock(datasetFullyQualifiedNameFrom); - } - } - - public void pregelixEnd(String dataverseName, String datasetFullyQualifiedNameFrom, - String datasetFullyQualifiedNameTo) { - - releaseDatasetReadLock(datasetFullyQualifiedNameFrom); - releaseDatasetWriteLock(datasetFullyQualifiedNameTo); - releaseDataverseReadLock(dataverseName); - } } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/751315f8/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java ---------------------------------------------------------------------- diff --git a/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java b/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java index e65f545..42ea000 100644 --- a/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java +++ b/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java @@ -94,6 +94,7 @@ public class ARecordType extends AbstractComplexType { @Override public String toString() { StringBuilder sb = new StringBuilder(); + sb.append(typeName + ": "); if (isOpen) { sb.append("open "); } else { @@ -290,6 +291,8 @@ public class ARecordType extends AbstractComplexType { @Override public JSONObject toJSON() throws JSONException { JSONObject type = new JSONObject(); + type.put("type", ARecordType.class.getName()); + type.put("name", typeName); if (isOpen) { type.put("open", true); } else { http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/751315f8/asterix-om/src/main/java/org/apache/asterix/om/util/JSONDeserializerForTypes.java ---------------------------------------------------------------------- diff --git a/asterix-om/src/main/java/org/apache/asterix/om/util/JSONDeserializerForTypes.java b/asterix-om/src/main/java/org/apache/asterix/om/util/JSONDeserializerForTypes.java index 801bfb9..b646ea1 100644 --- a/asterix-om/src/main/java/org/apache/asterix/om/util/JSONDeserializerForTypes.java +++ b/asterix-om/src/main/java/org/apache/asterix/om/util/JSONDeserializerForTypes.java @@ -44,23 +44,21 @@ public class JSONDeserializerForTypes { * @throws Exception */ public static IAType convertFromJSON(JSONObject typeInJSON) throws Exception { - boolean typeNameExists = typeInJSON.has("type"); - String typeName = typeNameExists ? typeInJSON.getString("type") : null; - + String typeName = typeInJSON.getString("type"); // Deals with ordered list. - if (typeNameExists && typeName.equals(AOrderedListType.class.getName())) { + if (typeName.equals(AOrderedListType.class.getName())) { IAType itemType = convertFromJSON((JSONObject) typeInJSON.get("item-type")); return new AOrderedListType(itemType, "ordered-list"); } // Deals with unordered list. - if (typeNameExists && typeName.equals(AUnorderedListType.class.getName())) { + if (typeName.equals(AUnorderedListType.class.getName())) { IAType itemType = convertFromJSON((JSONObject) typeInJSON.get("item-type")); return new AUnorderedListType(itemType, "unordered-list"); } // Deals with Union Type. - if (typeNameExists && typeName.equals(AUnionType.class.getName())) { + if (typeName.equals(AUnionType.class.getName())) { List unionTypes = new ArrayList(); JSONArray fields = (JSONArray) typeInJSON.get("fields"); for (int i = 0; i < fields.length(); i++) { @@ -70,26 +68,26 @@ public class JSONDeserializerForTypes { return new AUnionType(unionTypes, "union"); } - // Deals with primitive types. - if (typeNameExists) { - Class cl = BuiltinType.class; - Field typeField = cl.getDeclaredField(typeName.toUpperCase()); - return (IAType) typeField.get(null); - } - // Deals with record types. - boolean openType = typeInJSON.getBoolean("open"); - JSONArray fields = typeInJSON.getJSONArray("fields"); - String[] fieldNames = new String[fields.length()]; - IAType[] fieldTypes = new IAType[fields.length()]; - for (int i = 0; i < fields.length(); ++i) { - JSONObject field = (JSONObject) fields.get(i); - JSONArray names = field.names(); - String fieldName = names.getString(0); - fieldNames[i] = fieldName; - fieldTypes[i] = convertFromJSON((JSONObject) field.get(fieldName)); + if (typeName.equals(ARecordType.class.getName())) { + String name = typeInJSON.getString("name"); + boolean openType = typeInJSON.getBoolean("open"); + JSONArray fields = typeInJSON.getJSONArray("fields"); + String[] fieldNames = new String[fields.length()]; + IAType[] fieldTypes = new IAType[fields.length()]; + for (int i = 0; i < fields.length(); ++i) { + JSONObject field = (JSONObject) fields.get(i); + JSONArray names = field.names(); + String fieldName = names.getString(0); + fieldNames[i] = fieldName; + fieldTypes[i] = convertFromJSON((JSONObject) field.get(fieldName)); + } + return new ARecordType(name, fieldNames, fieldTypes, openType); } - return new ARecordType("record", fieldNames, fieldTypes, openType); - } + // Deals with primitive types. + Class cl = BuiltinType.class; + Field typeField = cl.getDeclaredField(typeName.toUpperCase()); + return (IAType) typeField.get(null); + } } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/751315f8/asterix-tools/pom.xml ---------------------------------------------------------------------- diff --git a/asterix-tools/pom.xml b/asterix-tools/pom.xml index cbb322d..50eee60 100644 --- a/asterix-tools/pom.xml +++ b/asterix-tools/pom.xml @@ -224,13 +224,13 @@ org.apache.httpcomponents httpclient - 4.2.2 + 4.3 compile org.apache.httpcomponents httpcore - 4.2.2 + 4.3 compile http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/751315f8/asterix-yarn/src/main/resources/base-asterix-configuration.xml ---------------------------------------------------------------------- diff --git a/asterix-yarn/src/main/resources/base-asterix-configuration.xml b/asterix-yarn/src/main/resources/base-asterix-configuration.xml index ab64cae..622ed3a 100644 --- a/asterix-yarn/src/main/resources/base-asterix-configuration.xml +++ b/asterix-yarn/src/main/resources/base-asterix-configuration.xml @@ -219,6 +219,11 @@ + + compiler.pregelix.home + ~/pregelix + + web.port 19001 http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/751315f8/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml ---------------------------------------------------------------------- diff --git a/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml b/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml index ab64cae..622ed3a 100644 --- a/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml +++ b/asterix-yarn/src/main/resources/configs/base-asterix-configuration.xml @@ -219,6 +219,11 @@ + + compiler.pregelix.home + ~/pregelix + + web.port 19001