Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 9D730200BEE for ; Fri, 16 Dec 2016 19:11:52 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 9C3ED160B37; Fri, 16 Dec 2016 18:11:52 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 54D81160B44 for ; Fri, 16 Dec 2016 19:11:50 +0100 (CET) Received: (qmail 53044 invoked by uid 500); 16 Dec 2016 18:11:49 -0000 Mailing-List: contact commits-help@asterixdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.apache.org Delivered-To: mailing list commits@asterixdb.apache.org Received: (qmail 52288 invoked by uid 99); 16 Dec 2016 18:11:48 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 16 Dec 2016 18:11:48 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1F73CE3902; Fri, 16 Dec 2016 18:11:48 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sjaco002@apache.org To: commits@asterixdb.apache.org Date: Fri, 16 Dec 2016 18:12:02 -0000 Message-Id: <9cc33f41efc0431fa2bcd8bf9a191f27@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [16/23] asterixdb git commit: ASTERIXDB-1711: remove more mentions of Asterix archived-at: Fri, 16 Dec 2016 18:11:52 -0000 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/BuildProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/BuildProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/BuildProperties.java new file mode 100644 index 0000000..a1d4703 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/BuildProperties.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.common.config; + +import java.util.Map; + +public class BuildProperties extends AbstractProperties { + + public BuildProperties(PropertiesAccessor accessor) { + super(accessor); + } + + public String getUserEmail() { + return accessor.getProperty("git.build.user.email", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getBuildHost() { + return accessor.getProperty("git.build.host", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getDirty() { + return accessor.getProperty("git.dirty", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getRemoteOriginUrl() { + return accessor.getProperty("git.remote.origin.url", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getClosestTagName() { + return accessor.getProperty("git.closest.tag.name", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getCommitIdDescribeShort() { + return accessor.getProperty("git.commit.id.describe-short", "", + PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getCommitUserEmail() { + return accessor.getProperty("git.commit.user.email", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getCommitTime() { + return accessor.getProperty("git.commit.time", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getCommitMessage() { + return accessor.getProperty("git.commit.message.full", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getBuildVersion() { + return accessor.getProperty("git.build.version", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getCommitMessageShort() { + return accessor.getProperty("git.commit.message.short", "", + PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getShortCommitId() { + return accessor.getProperty("git.commit.id.abbrev", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getBranch() { + return accessor.getProperty("git.branch", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getBuildUserName() { + return accessor.getProperty("git.build.user.name", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getClosestTagCommitCount() { + return accessor.getProperty("git.closest.tag.commit.count", "", + PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getCommitIdDescribe() { + return accessor.getProperty("git.commit.id.describe", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getCommitId() { + return accessor.getProperty("git.commit.id", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getTags() { + return accessor.getProperty("git.tags", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getBuildTime() { + return accessor.getProperty("git.build.time", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getCommitUserName() { + return accessor.getProperty("git.commit.user.name", "", PropertyInterpreters.getStringPropertyInterpreter()); + } + + public Map getAllProps() { + return accessor.getBuildProperties(); + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java new file mode 100644 index 0000000..14cf70b --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +import org.apache.hyracks.util.StorageUtil; + +import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE; +import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE; + +public class CompilerProperties extends AbstractProperties { + private static final String COMPILER_SORTMEMORY_KEY = "compiler.sortmemory"; + private static final long COMPILER_SORTMEMORY_DEFAULT = StorageUtil.getSizeInBytes(32, MEGABYTE); + + private static final String COMPILER_GROUPMEMORY_KEY = "compiler.groupmemory"; + private static final long COMPILER_GROUPMEMORY_DEFAULT = StorageUtil.getSizeInBytes(32, MEGABYTE); + + private static final String COMPILER_JOINMEMORY_KEY = "compiler.joinmemory"; + private static final long COMPILER_JOINMEMORY_DEFAULT = StorageUtil.getSizeInBytes(32, MEGABYTE); + + private static final String COMPILER_FRAMESIZE_KEY = "compiler.framesize"; + private static final int COMPILER_FRAMESIZE_DEFAULT = StorageUtil.getSizeInBytes(32, KILOBYTE); + + private static final String COMPILER_PREGELIX_HOME = "compiler.pregelix.home"; + private static final String COMPILER_PREGELIX_HOME_DEFAULT = "~/pregelix"; + + public CompilerProperties(PropertiesAccessor accessor) { + super(accessor); + } + + @PropertyKey(COMPILER_SORTMEMORY_KEY) + public long getSortMemorySize() { + return accessor.getProperty(COMPILER_SORTMEMORY_KEY, COMPILER_SORTMEMORY_DEFAULT, + PropertyInterpreters.getLongBytePropertyInterpreter()); + } + + @PropertyKey(COMPILER_JOINMEMORY_KEY) + public long getJoinMemorySize() { + return accessor.getProperty(COMPILER_JOINMEMORY_KEY, COMPILER_JOINMEMORY_DEFAULT, + PropertyInterpreters.getLongBytePropertyInterpreter()); + } + + @PropertyKey(COMPILER_GROUPMEMORY_KEY) + public long getGroupMemorySize() { + return accessor.getProperty(COMPILER_GROUPMEMORY_KEY, COMPILER_GROUPMEMORY_DEFAULT, + PropertyInterpreters.getLongBytePropertyInterpreter()); + } + + @PropertyKey(COMPILER_FRAMESIZE_KEY) + public int getFrameSize() { + return accessor.getProperty(COMPILER_FRAMESIZE_KEY, COMPILER_FRAMESIZE_DEFAULT, + PropertyInterpreters.getIntegerBytePropertyInterpreter()); + } + + @PropertyKey(COMPILER_PREGELIX_HOME) + public String getPregelixHome() { + return accessor.getProperty(COMPILER_PREGELIX_HOME, COMPILER_PREGELIX_HOME_DEFAULT, + PropertyInterpreters.getStringPropertyInterpreter()); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExtensionProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExtensionProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExtensionProperties.java new file mode 100644 index 0000000..edf955f --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExtensionProperties.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +import java.util.List; + +public class ExtensionProperties extends AbstractProperties { + + public ExtensionProperties(PropertiesAccessor accessor) { + super(accessor); + } + + public List getExtensions() { + return accessor.getExtensions(); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java new file mode 100644 index 0000000..dac154c --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ExternalProperties.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +import java.util.logging.Level; + +public class ExternalProperties extends AbstractProperties { + + private static final String EXTERNAL_WEBPORT_KEY = "web.port"; + private static final int EXTERNAL_WEBPORT_DEFAULT = 19001; + + private static final String EXTERNAL_SECONDARY_WEBPORT_KEY = "web.secondary.port"; + private static final int EXTERNAL_SECONDARY_WEBPORT_DEFAULT = 19005; + + private static final String QUERY_WEBPORT_KEY = "web.queryinterface.port"; + private static final int QUERY_WEBPORT_DEFAULT = 19006; + + private static final String EXTERNAL_LOGLEVEL_KEY = "log.level"; + private static final Level EXTERNAL_LOGLEVEL_DEFAULT = Level.WARNING; + + private static final String EXTERNAL_APISERVER_KEY = "api.port"; + private static final int EXTERNAL_APISERVER_DEFAULT = 19002; + + private static final String EXTERNAL_FEEDSERVER_KEY = "feed.port"; + private static final int EXTERNAL_FEEDSERVER_DEFAULT = 19003; + + private static final String EXTERNAL_CC_JAVA_OPTS_KEY = "cc.java.opts"; + private static final String EXTERNAL_CC_JAVA_OPTS_DEFAULT = "-Xmx1024m"; + + private static final String EXTERNAL_NC_JAVA_OPTS_KEY = "nc.java.opts"; + private static final String EXTERNAL_NC_JAVA_OPTS_DEFAULT = "-Xmx1024m"; + + private static final String EXTERNAL_MAX_WAIT_FOR_ACTIVE_CLUSTER = "max.wait.active.cluster"; + private static final int EXTERNAL_MAX_WAIT_FOR_ACTIVE_CLUSTER_DEFAULT = 60; + + private static final String EXTERNAL_PLOT_ACTIVATE = "plot.activate"; + private static final boolean EXTERNAL_PLOT_ACTIVATE_DEFAULT = false; + + public ExternalProperties(PropertiesAccessor accessor) { + super(accessor); + } + + @PropertyKey(EXTERNAL_WEBPORT_KEY) + public int getWebInterfacePort() { + return accessor.getProperty(EXTERNAL_WEBPORT_KEY, EXTERNAL_WEBPORT_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(EXTERNAL_SECONDARY_WEBPORT_KEY) + public int getSecondaryWebInterfacePort() { + return accessor.getProperty(EXTERNAL_SECONDARY_WEBPORT_KEY, EXTERNAL_SECONDARY_WEBPORT_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(QUERY_WEBPORT_KEY) + public int getQueryWebInterfacePort() { + return accessor.getProperty(QUERY_WEBPORT_KEY, QUERY_WEBPORT_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(EXTERNAL_APISERVER_KEY) + public int getAPIServerPort() { + return accessor.getProperty(EXTERNAL_APISERVER_KEY, EXTERNAL_APISERVER_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(EXTERNAL_FEEDSERVER_KEY) + public int getFeedServerPort() { + return accessor.getProperty(EXTERNAL_FEEDSERVER_KEY, EXTERNAL_FEEDSERVER_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(EXTERNAL_LOGLEVEL_KEY) + public Level getLogLevel() { + return accessor.getProperty(EXTERNAL_LOGLEVEL_KEY, EXTERNAL_LOGLEVEL_DEFAULT, + PropertyInterpreters.getLevelPropertyInterpreter()); + } + + public String getNCJavaParams() { + return accessor.getProperty(EXTERNAL_NC_JAVA_OPTS_KEY, EXTERNAL_NC_JAVA_OPTS_DEFAULT, + PropertyInterpreters.getStringPropertyInterpreter()); + } + + public String getCCJavaParams() { + return accessor.getProperty(EXTERNAL_CC_JAVA_OPTS_KEY, EXTERNAL_CC_JAVA_OPTS_DEFAULT, + PropertyInterpreters.getStringPropertyInterpreter()); + } + + @PropertyKey(EXTERNAL_MAX_WAIT_FOR_ACTIVE_CLUSTER) + public int getMaxWaitClusterActive() { + return accessor.getProperty(EXTERNAL_MAX_WAIT_FOR_ACTIVE_CLUSTER, EXTERNAL_MAX_WAIT_FOR_ACTIVE_CLUSTER_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(EXTERNAL_PLOT_ACTIVATE) + public Boolean getIsPlottingEnabled() { + return accessor.getProperty(EXTERNAL_PLOT_ACTIVATE, EXTERNAL_PLOT_ACTIVATE_DEFAULT, + PropertyInterpreters.getBooleanPropertyInterpreter()); + } + +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/FeedProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/FeedProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/FeedProperties.java new file mode 100644 index 0000000..0afbbd3 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/FeedProperties.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +import org.apache.hyracks.util.StorageUtil; + +import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE; + +public class FeedProperties extends AbstractProperties { + + private static final String FEED_CENTRAL_MANAGER_PORT_KEY = "feed.central.manager.port"; + private static final int FEED_CENTRAL_MANAGER_PORT_DEFAULT = 4500; + + private static final String FEED_MEMORY_GLOBALBUDGET_KEY = "feed.memory.global.budget"; + private static final long FEED_MEMORY_GLOBALBUDGET_DEFAULT = StorageUtil.getSizeInBytes(64, MEGABYTE); + // i.e. 2048 frames (assuming 32768 as frame size) + + private static final String FEED_MEMORY_AVAILABLE_WAIT_TIMEOUT_KEY = "feed.memory.available.wait.timeout"; + private static final long FEED_MEMORY_AVAILABLE_WAIT_TIMEOUT_DEFAULT = 10; // 10 seconds + + private static final String FEED_PENDING_WORK_THRESHOLD_KEY = "feed.pending.work.threshold"; + private static final int FEED_PENDING_WORK_THRESHOLD_DEFAULT = 50; + + private static final String FEED_MAX_SUCCESSIVE_THRESHOLD_PERIOD_KEY = "feed.max.threshold.period"; + private static final int FEED_MAX_SUCCESSIVE_THRESHOLD_PERIOD_DEFAULT = 5; + + public FeedProperties(PropertiesAccessor accessor) { + super(accessor); + } + + @PropertyKey(FEED_MEMORY_GLOBALBUDGET_KEY) + public long getMemoryComponentGlobalBudget() { + return accessor.getProperty(FEED_MEMORY_GLOBALBUDGET_KEY, FEED_MEMORY_GLOBALBUDGET_DEFAULT, + PropertyInterpreters.getLongBytePropertyInterpreter()); + } + + @PropertyKey(FEED_MEMORY_AVAILABLE_WAIT_TIMEOUT_KEY) + public long getMemoryAvailableWaitTimeout() { + return accessor.getProperty(FEED_MEMORY_AVAILABLE_WAIT_TIMEOUT_KEY, FEED_MEMORY_AVAILABLE_WAIT_TIMEOUT_DEFAULT, + PropertyInterpreters.getLongPropertyInterpreter()); + } + + /** + * @return port at which the Central Feed Manager listens for control messages from local Feed Managers + */ + @PropertyKey(FEED_CENTRAL_MANAGER_PORT_KEY) + public int getFeedCentralManagerPort() { + return accessor.getProperty(FEED_CENTRAL_MANAGER_PORT_KEY, FEED_CENTRAL_MANAGER_PORT_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + /** + * @return maximum length of input queue before triggering corrective action + */ + @PropertyKey(FEED_PENDING_WORK_THRESHOLD_KEY) + public int getPendingWorkThreshold() { + return accessor.getProperty(FEED_PENDING_WORK_THRESHOLD_KEY, FEED_PENDING_WORK_THRESHOLD_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(FEED_MAX_SUCCESSIVE_THRESHOLD_PERIOD_KEY) + public int getMaxSuccessiveThresholdPeriod() { + return accessor.getProperty(FEED_MAX_SUCCESSIVE_THRESHOLD_PERIOD_KEY, + FEED_MAX_SUCCESSIVE_THRESHOLD_PERIOD_DEFAULT, PropertyInterpreters.getIntegerPropertyInterpreter()); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IAsterixPropertiesProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IAsterixPropertiesProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IAsterixPropertiesProvider.java deleted file mode 100644 index 43425ab..0000000 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IAsterixPropertiesProvider.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.common.config; - -public interface IAsterixPropertiesProvider { - public AsterixStorageProperties getStorageProperties(); - - public AsterixTransactionProperties getTransactionProperties(); - - public AsterixCompilerProperties getCompilerProperties(); - - public AsterixMetadataProperties getMetadataProperties(); - - public AsterixExternalProperties getExternalProperties(); - - public AsterixFeedProperties getFeedProperties(); - - AsterixBuildProperties getBuildProperties(); - - public AsterixReplicationProperties getReplicationProperties(); - - public MessagingProperties getMessagingProperties(); -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IPropertiesProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IPropertiesProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IPropertiesProvider.java new file mode 100644 index 0000000..1de1523 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/IPropertiesProvider.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +public interface IPropertiesProvider { + public StorageProperties getStorageProperties(); + + public TransactionProperties getTransactionProperties(); + + public CompilerProperties getCompilerProperties(); + + public MetadataProperties getMetadataProperties(); + + public ExternalProperties getExternalProperties(); + + public FeedProperties getFeedProperties(); + + BuildProperties getBuildProperties(); + + public ReplicationProperties getReplicationProperties(); + + public MessagingProperties getMessagingProperties(); +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MessagingProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MessagingProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MessagingProperties.java index 6794c8d..2e98e1e 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MessagingProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MessagingProperties.java @@ -21,7 +21,7 @@ package org.apache.asterix.common.config; import org.apache.hyracks.util.StorageUtil; import org.apache.hyracks.util.StorageUtil.StorageUnit; -public class MessagingProperties extends AbstractAsterixProperties { +public class MessagingProperties extends AbstractProperties { private static final String MESSAGING_FRAME_SIZE_KEY = "messaging.frame.size"; private static final int MESSAGING_FRAME_SIZE_DEFAULT = StorageUtil.getSizeInBytes(4, StorageUnit.KILOBYTE); @@ -29,7 +29,7 @@ public class MessagingProperties extends AbstractAsterixProperties { private static final String MESSAGING_FRAME_COUNT_KEY = "messaging.frame.count"; private static final int MESSAGING_BUFFER_COUNTE_DEFAULT = 512; - public MessagingProperties(AsterixPropertiesAccessor accessor) { + public MessagingProperties(PropertiesAccessor accessor) { super(accessor); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java new file mode 100644 index 0000000..ab65b71 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +import java.util.List; +import java.util.Map; +import java.util.SortedMap; + +import org.apache.asterix.common.cluster.ClusterPartition; + +public class MetadataProperties extends AbstractProperties { + + private static final String METADATA_REGISTRATION_TIMEOUT_KEY = "metadata.registration.timeout.secs"; + private static final long METADATA_REGISTRATION_TIMEOUT_DEFAULT = 60; + + private static final String METADATA_PORT_KEY = "metadata.port"; + private static final int METADATA_PORT_DEFAULT = 0; + + private static final String METADATA_CALLBACK_PORT_KEY = "metadata.callback.port"; + private static final int METADATA_CALLBACK_PORT_DEFAULT = 0; + + public MetadataProperties(PropertiesAccessor accessor) { + super(accessor); + } + + @PropertyKey("instance.name") + public String getInstanceName() { + return accessor.getInstanceName(); + } + + @PropertyKey("metadata.node") + public String getMetadataNodeName() { + return accessor.getMetadataNodeName(); + } + + @PropertyKey("metadata.partition") + public ClusterPartition getMetadataPartition() { + return accessor.getMetadataPartition(); + } + + @PropertyKey("node.stores") + public Map getStores() { + return accessor.getStores(); + } + + public List getNodeNames() { + return accessor.getNodeNames(); + } + + public String getCoredumpPath(String nodeId) { + return accessor.getCoredumpPath(nodeId); + } + + @PropertyKey("core.dump.paths") + public Map getCoredumpPaths() { + return accessor.getCoredumpConfig(); + } + + @PropertyKey("node.partitions") + public Map getNodePartitions() { + return accessor.getNodePartitions(); + } + + @PropertyKey("cluster.partitions") + public SortedMap getClusterPartitions() { + return accessor.getClusterPartitions(); + } + + @PropertyKey("transaction.log.dirs") + public Map getTransactionLogDirs() { + return accessor.getTransactionLogDirs(); + } + + @PropertyKey(METADATA_REGISTRATION_TIMEOUT_KEY) + public long getRegistrationTimeoutSecs() { + return accessor.getProperty(METADATA_REGISTRATION_TIMEOUT_KEY, METADATA_REGISTRATION_TIMEOUT_DEFAULT, + PropertyInterpreters.getLongPropertyInterpreter()); + } + + @PropertyKey(METADATA_PORT_KEY) + public int getMetadataPort() { + return accessor.getProperty(METADATA_PORT_KEY, METADATA_PORT_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(METADATA_CALLBACK_PORT_KEY) + public int getMetadataCallbackPort() { + return accessor.getProperty(METADATA_CALLBACK_PORT_KEY, METADATA_CALLBACK_PORT_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java new file mode 100644 index 0000000..492282c --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicReference; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Unmarshaller; + +import org.apache.asterix.common.cluster.ClusterPartition; +import org.apache.asterix.common.configuration.AsterixConfiguration; +import org.apache.asterix.common.configuration.Coredump; +import org.apache.asterix.common.configuration.Extension; +import org.apache.asterix.common.configuration.Property; +import org.apache.asterix.common.configuration.Store; +import org.apache.asterix.common.configuration.TransactionLogDir; +import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.common.utils.ConfigUtil; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.hyracks.algebricks.common.utils.Pair; +import org.apache.hyracks.api.application.IApplicationConfig; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +public class PropertiesAccessor { + private static final Logger LOGGER = Logger.getLogger(PropertiesAccessor.class.getName()); + + private static final AtomicReference instanceHolder = new AtomicReference<>(); + private final String instanceName; + private final String metadataNodeName; + private final List nodeNames = new ArrayList<>();; + private final Map stores = new HashMap<>();; + private final Map coredumpConfig = new HashMap<>(); + + // This can be removed when asterix-configuration.xml is no longer required. + private final Map asterixConfigurationParams; + private final IApplicationConfig cfg; + private final Map transactionLogDirs = new HashMap<>(); + private final Map asterixBuildProperties = new HashMap<>(); + private final Map nodePartitionsMap; + private final SortedMap clusterPartitions = new TreeMap<>(); + // For extensions + private final List extensions; + + /** + * Constructor which reads asterix-configuration.xml, the old way. + * + * @throws AsterixException + * @throws IOException + */ + private PropertiesAccessor() throws AsterixException, IOException { + String fileName = System.getProperty(GlobalConfig.CONFIG_FILE_PROPERTY); + if (fileName == null) { + fileName = GlobalConfig.DEFAULT_CONFIG_FILE_NAME; + } + AsterixConfiguration asterixConfiguration = configure(fileName); + cfg = null; + instanceName = asterixConfiguration.getInstanceName(); + metadataNodeName = asterixConfiguration.getMetadataNode(); + List configuredStores = asterixConfiguration.getStore(); + nodePartitionsMap = new HashMap<>(); + int uniquePartitionId = 0; + // Here we iterate through all elements in asterix-configuration.xml. + // For each one, we create an array of ClusterPartitions and store this array + // in nodePartitionsMap, keyed by the node name. The array is the same length + // as the comma-separated child element, because Managix will have + // arranged for that element to be populated with the full paths to each + // partition directory (as formed by appending the subdirectory to + // each path from the user's original cluster.xml). + for (Store store : configuredStores) { + String trimmedStoreDirs = store.getStoreDirs().trim(); + String[] nodeStores = trimmedStoreDirs.split(","); + ClusterPartition[] nodePartitions = new ClusterPartition[nodeStores.length]; + for (int i = 0; i < nodePartitions.length; i++) { + ClusterPartition partition = new ClusterPartition(uniquePartitionId++, store.getNcId(), i); + clusterPartitions.put(partition.getPartitionId(), partition); + nodePartitions[i] = partition; + } + stores.put(store.getNcId(), nodeStores); + nodePartitionsMap.put(store.getNcId(), nodePartitions); + nodeNames.add(store.getNcId()); + } + + // Get extensions + extensions = new ArrayList<>(); + if (asterixConfiguration.getExtensions() != null) { + for (Extension ext : asterixConfiguration.getExtensions().getExtension()) { + extensions.add(ConfigUtil.toAsterixExtension(ext)); + } + } + + asterixConfigurationParams = new HashMap<>(); + for (Property p : asterixConfiguration.getProperty()) { + asterixConfigurationParams.put(p.getName(), p); + } + for (Coredump cd : asterixConfiguration.getCoredump()) { + coredumpConfig.put(cd.getNcId(), cd.getCoredumpPath()); + } + for (TransactionLogDir txnLogDir : asterixConfiguration.getTransactionLogDir()) { + transactionLogDirs.put(txnLogDir.getNcId(), txnLogDir.getTxnLogDirPath()); + } + loadAsterixBuildProperties(); + } + + private AsterixConfiguration configure(String fileName) throws IOException, AsterixException { + try (InputStream is = this.getClass().getClassLoader().getResourceAsStream(fileName)) { + if (is != null) { + return configure(is, fileName); + } + } + try (FileInputStream is = new FileInputStream(fileName)) { + return configure(is, fileName); + } catch (FileNotFoundException fnf1) { + LOGGER.warn("Failed to get configuration file " + fileName + " as FileInputStream. FileNotFoundException"); + LOGGER.warn("Attempting to get default configuration file " + GlobalConfig.DEFAULT_CONFIG_FILE_NAME + + " as FileInputStream"); + try (FileInputStream fis = new FileInputStream(GlobalConfig.DEFAULT_CONFIG_FILE_NAME)) { + return configure(fis, GlobalConfig.DEFAULT_CONFIG_FILE_NAME); + } catch (FileNotFoundException fnf2) { + fnf1.addSuppressed(fnf2); + throw new AsterixException("Could not find configuration file " + fileName, fnf1); + } + } + } + + private AsterixConfiguration configure(InputStream is, String fileName) throws AsterixException { + try { + JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class); + Unmarshaller unmarshaller = ctx.createUnmarshaller(); + return (AsterixConfiguration) unmarshaller.unmarshal(is); + } catch (JAXBException e) { + throw new AsterixException("Failed to read configuration file " + fileName, e); + } + } + + /** + * Constructor which wraps an IApplicationConfig. + */ + private PropertiesAccessor(IApplicationConfig cfg) throws AsterixException { + this.cfg = cfg; + instanceName = cfg.getString(AsterixProperties.SECTION_ASTERIX, AsterixProperties.PROPERTY_INSTANCE_NAME, + AsterixProperties.DEFAULT_INSTANCE_NAME); + nodePartitionsMap = new HashMap<>(); + MutableInt uniquePartitionId = new MutableInt(0); + extensions = new ArrayList<>(); + // Iterate through each configured NC. + for (String section : cfg.getSections()) { + if (section.startsWith(AsterixProperties.SECTION_PREFIX_NC)) { + configureNc(section, uniquePartitionId); + } else if (section.startsWith(AsterixProperties.SECTION_PREFIX_EXTENSION)) { + String className = AsterixProperties.getSectionId(AsterixProperties.SECTION_PREFIX_EXTENSION, section); + configureExtension(className, section); + } + } + metadataNodeName = getProperty(AsterixProperties.PROPERTY_METADATA_NODE, + nodeNames.isEmpty() ? "" : nodeNames.get(0), PropertyInterpreters.getStringPropertyInterpreter()); + asterixConfigurationParams = null; + loadAsterixBuildProperties(); + } + + private void configureExtension(String className, String section) { + Set keys = cfg.getKeys(section); + List> kvs = new ArrayList<>(); + for (String key : keys) { + String value = cfg.getString(section, key); + kvs.add(new Pair<>(key, value)); + } + extensions.add(new AsterixExtension(className, kvs)); + } + + private void configureNc(String section, MutableInt uniquePartitionId) { + String ncId = AsterixProperties.getSectionId(AsterixProperties.SECTION_PREFIX_NC, section); + + // Now we assign the coredump and txnlog directories for this node. + // QQQ Default values? Should they be specified here? Or should there + // be a default.ini? Certainly wherever they are, they should be platform-dependent. + coredumpConfig.put(ncId, cfg.getString(section, AsterixProperties.PROPERTY_COREDUMP_DIR, + AsterixProperties.DEFAULT_COREDUMP_DIR)); + transactionLogDirs.put(ncId, + cfg.getString(section, AsterixProperties.PROPERTY_TXN_LOG_DIR, AsterixProperties.DEFAULT_TXN_LOG_DIR)); + + // Now we create an array of ClusterPartitions for all the partitions + // on this NC. + String[] iodevices = cfg.getString(section, AsterixProperties.PROPERTY_IO_DEV, + AsterixProperties.DEFAULT_IO_DEV).split(","); + String storageSubdir = cfg.getString(section, AsterixProperties.PROPERTY_STORAGE_DIR, + AsterixProperties.DEFAULT_STORAGE_DIR); + String[] nodeStores = new String[iodevices.length]; + ClusterPartition[] nodePartitions = new ClusterPartition[iodevices.length]; + for (int i = 0; i < nodePartitions.length; i++) { + // Construct final storage path from iodevice dir + storage subdir.s + nodeStores[i] = iodevices[i] + File.separator + storageSubdir; + // Create ClusterPartition instances for this NC. + ClusterPartition partition = new ClusterPartition(uniquePartitionId.getValue(), ncId, i); + uniquePartitionId.increment(); + clusterPartitions.put(partition.getPartitionId(), partition); + nodePartitions[i] = partition; + } + stores.put(ncId, nodeStores); + nodePartitionsMap.put(ncId, nodePartitions); + nodeNames.add(ncId); + } + + private void loadAsterixBuildProperties() throws AsterixException { + Properties gitProperties = new Properties(); + try { + gitProperties.load(getClass().getClassLoader().getResourceAsStream("git.properties")); + for (final String name : gitProperties.stringPropertyNames()) { + asterixBuildProperties.put(name, gitProperties.getProperty(name)); + } + } catch (IOException e) { + throw new AsterixException(e); + } + } + + public String getMetadataNodeName() { + return metadataNodeName; + } + + public Map getStores() { + return stores; + } + + public List getNodeNames() { + return nodeNames; + } + + public String getCoredumpPath(String nodeId) { + return coredumpConfig.get(nodeId); + } + + public Map getTransactionLogDirs() { + return transactionLogDirs; + } + + public Map getCoredumpConfig() { + return coredumpConfig; + } + + public Map getBuildProperties() { + return asterixBuildProperties; + } + + public T getProperty(String property, T defaultValue, IPropertyInterpreter interpreter) { + String value; + Property p = null; + if (asterixConfigurationParams != null) { + p = asterixConfigurationParams.get(property); + value = (p == null) ? null : p.getValue(); + } else { + value = cfg.getString("app", property); + if (value == null) { + value = cfg.getString("asterix", property); + if (value != null) { + LOGGER.warn("[asterix] config section deprecated and will be removed in a future release;" + + " please update to [app] (found: " + property + ')'); + } + } + } + if (value == null) { + return defaultValue; + } + try { + return interpreter.interpret(value); + } catch (IllegalArgumentException e) { + if (LOGGER.isEnabledFor(Level.ERROR)) { + StringBuilder msg = new StringBuilder( + "Invalid property value '" + value + "' for property '" + property + "'.\n"); + if (p != null) { + msg.append("See the description: \n" + p.getDescription() + "\n"); + } + msg.append("Default = " + defaultValue); + LOGGER.error(msg.toString()); + } + throw e; + } + } + + public String getInstanceName() { + return instanceName; + } + + public ClusterPartition getMetadataPartition() { + // metadata partition is always the first partition on the metadata node + return nodePartitionsMap.get(metadataNodeName)[0]; + } + + public Map getNodePartitions() { + return nodePartitionsMap; + } + + public SortedMap getClusterPartitions() { + return clusterPartitions; + } + + public List getExtensions() { + return extensions; + } + + public static PropertiesAccessor getInstance(IApplicationConfig cfg) throws IOException, AsterixException { + // Determine whether to use old-style asterix-configuration.xml or new-style configuration. + // QQQ strip this out eventually + // QQQ this is NOT a good way to determine whether the config is valid + PropertiesAccessor propertiesAccessor; + if (cfg != null && cfg.getString("cc", "cluster.address") != null) { + propertiesAccessor = new PropertiesAccessor(cfg); + } else { + propertiesAccessor = new PropertiesAccessor(); + } + if (!instanceHolder.compareAndSet(null, propertiesAccessor)) { + propertiesAccessor = instanceHolder.get(); + } + return propertiesAccessor; + } + + public static PropertiesAccessor getInstance() throws IOException, AsterixException { + return getInstance(null); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ReplicationProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ReplicationProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ReplicationProperties.java new file mode 100644 index 0000000..164a525 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/ReplicationProperties.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.asterix.common.replication.Replica; +import org.apache.asterix.event.schema.cluster.Cluster; +import org.apache.asterix.event.schema.cluster.Node; +import org.apache.hyracks.util.StorageUtil; +import org.apache.hyracks.util.StorageUtil.StorageUnit; + +public class ReplicationProperties extends AbstractProperties { + + private static final Logger LOGGER = Logger.getLogger(ReplicationProperties.class.getName()); + + + private static final int REPLICATION_DATAPORT_DEFAULT = 2000; + + private static final String REPLICATION_ENABLED_KEY = "replication.enabled"; + + private static final String REPLICATION_FACTOR_KEY = "replication.factor"; + private static final int REPLICATION_FACTOR_DEFAULT = 1; + + private static final String REPLICATION_TIMEOUT_KEY = "replication.timeout"; + private static final int REPLICATION_TIME_OUT_DEFAULT = 15; + + private static final String REPLICATION_MAX_REMOTE_RECOVERY_ATTEMPTS_KEY = + "replication.max.remote.recovery.attempts"; + private static final int MAX_REMOTE_RECOVERY_ATTEMPTS = 5; + + private static final String NODE_IP_ADDRESS_DEFAULT = "127.0.0.1"; + + private static final String REPLICATION_LOG_BATCH_SIZE_KEY = "replication.log.batchsize"; + private static final int REPLICATION_LOG_BATCH_SIZE_DEFAULT = StorageUtil.getSizeInBytes(4, StorageUnit.KILOBYTE); + + private static final String REPLICATION_LOG_BUFFER_NUM_PAGES_KEY = "replication.log.buffer.numpages"; + private static final int REPLICATION_LOG_BUFFER_NUM_PAGES_DEFAULT = 8; + + private static final String REPLICATION_LOG_BUFFER_PAGE_SIZE_KEY = "replication.log.buffer.pagesize"; + private static final int REPLICATION_LOG_BUFFER_PAGE_SIZE_DEFAULT = StorageUtil.getSizeInBytes(128, + StorageUnit.KILOBYTE); + + private final String nodeNamePrefix; + private final Cluster cluster; + + public ReplicationProperties(PropertiesAccessor accessor) { + super(accessor); + this.cluster = ClusterProperties.INSTANCE.getCluster(); + + if (cluster != null) { + nodeNamePrefix = cluster.getInstanceName() + "_"; + } else { + nodeNamePrefix = ""; + } + } + + @PropertyKey(REPLICATION_ENABLED_KEY) + public boolean isReplicationEnabled() { + return ClusterProperties.INSTANCE.isReplicationEnabled(); + } + + public String getReplicaIPAddress(String nodeId) { + if (cluster != null) { + for (int i = 0; i < cluster.getNode().size(); i++) { + Node node = cluster.getNode().get(i); + if (getRealCluserNodeID(node.getId()).equals(nodeId)) { + return node.getClusterIp(); + } + } + } + return NODE_IP_ADDRESS_DEFAULT; + } + + public int getDataReplicationPort(String nodeId) { + if (cluster != null && cluster.getDataReplication() != null) { + for (int i = 0; i < cluster.getNode().size(); i++) { + Node node = cluster.getNode().get(i); + if (getRealCluserNodeID(node.getId()).equals(nodeId)) { + return node.getReplicationPort() != null ? node.getReplicationPort().intValue() + : cluster.getDataReplication().getReplicationPort().intValue(); + } + } + } + return REPLICATION_DATAPORT_DEFAULT; + } + + public Set getRemoteReplicas(String nodeId) { + Set remoteReplicas = new HashSet<>();; + + int numberOfRemoteReplicas = getReplicationFactor() - 1; + //Using chained-declustering + if (cluster != null) { + int nodeIndex = -1; + //find the node index in the cluster config + for (int i = 0; i < cluster.getNode().size(); i++) { + Node node = cluster.getNode().get(i); + if (getRealCluserNodeID(node.getId()).equals(nodeId)) { + nodeIndex = i; + break; + } + } + + if (nodeIndex == -1) { + LOGGER.log(Level.WARNING, + "Could not find node " + getRealCluserNodeID(nodeId) + " in cluster configurations"); + return null; + } + + //find nodes to the right of this node + for (int i = nodeIndex + 1; i < cluster.getNode().size(); i++) { + remoteReplicas.add(getReplicaByNodeIndex(i)); + if (remoteReplicas.size() == numberOfRemoteReplicas) { + break; + } + } + + //if not all remote replicas have been found, start from the beginning + if (remoteReplicas.size() != numberOfRemoteReplicas) { + for (int i = 0; i < cluster.getNode().size(); i++) { + remoteReplicas.add(getReplicaByNodeIndex(i)); + if (remoteReplicas.size() == numberOfRemoteReplicas) { + break; + } + } + } + } + return remoteReplicas; + } + + private Replica getReplicaByNodeIndex(int nodeIndex) { + Node node = cluster.getNode().get(nodeIndex); + Node replicaNode = new Node(); + replicaNode.setId(getRealCluserNodeID(node.getId())); + replicaNode.setClusterIp(node.getClusterIp()); + return new Replica(replicaNode); + } + + public Replica getReplicaById(String nodeId) { + int nodeIndex = -1; + if (cluster != null) { + for (int i = 0; i < cluster.getNode().size(); i++) { + Node node = cluster.getNode().get(i); + + if (getRealCluserNodeID(node.getId()).equals(nodeId)) { + nodeIndex = i; + break; + } + } + } + + if (nodeIndex < 0) { + return null; + } + + return getReplicaByNodeIndex(nodeIndex); + } + + public Set getRemoteReplicasIds(String nodeId) { + Set remoteReplicasIds = new HashSet<>(); + Set remoteReplicas = getRemoteReplicas(nodeId); + + for (Replica replica : remoteReplicas) { + remoteReplicasIds.add(replica.getId()); + } + + return remoteReplicasIds; + } + + public String getRealCluserNodeID(String nodeId) { + return nodeNamePrefix + nodeId; + } + + public Set getNodeReplicasIds(String nodeId) { + Set replicaIds = new HashSet<>(); + replicaIds.add(nodeId); + replicaIds.addAll(getRemoteReplicasIds(nodeId)); + return replicaIds; + } + + @PropertyKey(REPLICATION_FACTOR_KEY) + public int getReplicationFactor() { + if (cluster != null) { + if (cluster.getDataReplication() == null || cluster.getDataReplication().getReplicationFactor() == null) { + return REPLICATION_FACTOR_DEFAULT; + } + return cluster.getDataReplication().getReplicationFactor().intValue(); + } + return REPLICATION_FACTOR_DEFAULT; + } + + @PropertyKey(REPLICATION_TIMEOUT_KEY) + public int getReplicationTimeOut() { + if (cluster != null) { + return cluster.getDataReplication().getReplicationTimeOut().intValue(); + } + return REPLICATION_TIME_OUT_DEFAULT; + } + + /** + * @param nodeId + * @return The set of nodes which replicate to this node, including the node itself + */ + public Set getNodeReplicationClients(String nodeId) { + Set clientReplicas = new HashSet<>(); + clientReplicas.add(nodeId); + + int clientsCount = getReplicationFactor(); + + //Using chained-declustering backwards + if (cluster != null) { + int nodeIndex = -1; + //find the node index in the cluster config + for (int i = 0; i < cluster.getNode().size(); i++) { + Node node = cluster.getNode().get(i); + if (getRealCluserNodeID(node.getId()).equals(nodeId)) { + nodeIndex = i; + break; + } + } + + //find nodes to the left of this node + for (int i = nodeIndex - 1; i >= 0; i--) { + clientReplicas.add(getReplicaByNodeIndex(i).getId()); + if (clientReplicas.size() == clientsCount) { + break; + } + } + + //if not all client replicas have been found, start from the end + if (clientReplicas.size() != clientsCount) { + for (int i = cluster.getNode().size() - 1; i >= 0; i--) { + clientReplicas.add(getReplicaByNodeIndex(i).getId()); + if (clientReplicas.size() == clientsCount) { + break; + } + } + } + } + return clientReplicas; + } + + @PropertyKey(REPLICATION_MAX_REMOTE_RECOVERY_ATTEMPTS_KEY) + public int getMaxRemoteRecoveryAttempts() { + return MAX_REMOTE_RECOVERY_ATTEMPTS; + } + + @PropertyKey(REPLICATION_LOG_BUFFER_PAGE_SIZE_KEY) + public int getLogBufferPageSize() { + return accessor.getProperty(REPLICATION_LOG_BUFFER_PAGE_SIZE_KEY, REPLICATION_LOG_BUFFER_PAGE_SIZE_DEFAULT, + PropertyInterpreters.getIntegerBytePropertyInterpreter()); + } + + @PropertyKey(REPLICATION_LOG_BUFFER_NUM_PAGES_KEY) + public int getLogBufferNumOfPages() { + return accessor.getProperty(REPLICATION_LOG_BUFFER_NUM_PAGES_KEY, REPLICATION_LOG_BUFFER_NUM_PAGES_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(REPLICATION_LOG_BATCH_SIZE_KEY) + public int getLogBatchSize() { + return accessor.getProperty(REPLICATION_LOG_BATCH_SIZE_KEY, REPLICATION_LOG_BATCH_SIZE_DEFAULT, + PropertyInterpreters.getIntegerBytePropertyInterpreter()); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java new file mode 100644 index 0000000..b86d961 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.util.StorageUtil; + +import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE; +import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE; + +public class StorageProperties extends AbstractProperties { + + private static final String STORAGE_BUFFERCACHE_PAGESIZE_KEY = "storage.buffercache.pagesize"; + private static final int STORAGE_BUFFERCACHE_PAGESIZE_DEFAULT = StorageUtil.getSizeInBytes(128, KILOBYTE); + + private static final String STORAGE_BUFFERCACHE_SIZE_KEY = "storage.buffercache.size"; + private static final long STORAGE_BUFFERCACHE_SIZE_DEFAULT = StorageUtil.getSizeInBytes(512, MEGABYTE); + + private static final String STORAGE_BUFFERCACHE_MAXOPENFILES_KEY = "storage.buffercache.maxopenfiles"; + private static final int STORAGE_BUFFERCACHE_MAXOPENFILES_DEFAULT = Integer.MAX_VALUE; + + private static final String STORAGE_MEMORYCOMPONENT_PAGESIZE_KEY = "storage.memorycomponent.pagesize"; + private static final int STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT = StorageUtil.getSizeInBytes(128, KILOBYTE); + + private static final String STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY = "storage.memorycomponent.numpages"; + private static final int STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 256; // ... so 32MB components + + private static final String STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY = + "storage.metadata.memorycomponent.numpages"; + private static final int STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 256; // ... so 32MB components + + private static final String STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY = "storage.memorycomponent.numcomponents"; + private static final int STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_DEFAULT = 2; // 2 components + + private static final String STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY = "storage.memorycomponent.globalbudget"; + private static final long STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT = StorageUtil.getSizeInBytes(512, MEGABYTE); + + private static final String STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_KEY = + "storage.lsm.bloomfilter.falsepositiverate"; + private static final double STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_DEFAULT = 0.01; + + public StorageProperties(PropertiesAccessor accessor) { + super(accessor); + } + + @PropertyKey(STORAGE_BUFFERCACHE_PAGESIZE_KEY) + public int getBufferCachePageSize() { + return accessor.getProperty(STORAGE_BUFFERCACHE_PAGESIZE_KEY, STORAGE_BUFFERCACHE_PAGESIZE_DEFAULT, + PropertyInterpreters.getIntegerBytePropertyInterpreter()); + } + + @PropertyKey(STORAGE_BUFFERCACHE_SIZE_KEY) + public long getBufferCacheSize() { + return accessor.getProperty(STORAGE_BUFFERCACHE_SIZE_KEY, STORAGE_BUFFERCACHE_SIZE_DEFAULT, + PropertyInterpreters.getLongBytePropertyInterpreter()); + } + + public int getBufferCacheNumPages() { + return (int) (getBufferCacheSize() / (getBufferCachePageSize() + IBufferCache.RESERVED_HEADER_BYTES)); + } + + @PropertyKey(STORAGE_BUFFERCACHE_MAXOPENFILES_KEY) + public int getBufferCacheMaxOpenFiles() { + return accessor.getProperty(STORAGE_BUFFERCACHE_MAXOPENFILES_KEY, STORAGE_BUFFERCACHE_MAXOPENFILES_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(STORAGE_MEMORYCOMPONENT_PAGESIZE_KEY) + public int getMemoryComponentPageSize() { + return accessor.getProperty(STORAGE_MEMORYCOMPONENT_PAGESIZE_KEY, STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT, + PropertyInterpreters.getIntegerBytePropertyInterpreter()); + } + + @PropertyKey(STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY) + public int getMemoryComponentNumPages() { + return accessor.getProperty(STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY, STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY) + public int getMetadataMemoryComponentNumPages() { + return accessor.getProperty(STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY, + STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY) + public int getMemoryComponentsNum() { + return accessor.getProperty(STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY, + STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_DEFAULT, PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY) + public long getMemoryComponentGlobalBudget() { + return accessor.getProperty(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY, + STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT, PropertyInterpreters.getLongBytePropertyInterpreter()); + } + + @PropertyKey(STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_KEY) + public double getBloomFilterFalsePositiveRate() { + return accessor.getProperty(STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_KEY, + STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_DEFAULT, PropertyInterpreters.getDoublePropertyInterpreter()); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java new file mode 100644 index 0000000..e4fd71d --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/TransactionProperties.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.config; + +import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE; +import static org.apache.hyracks.util.StorageUtil.StorageUnit.MEGABYTE; + +import java.util.Map; + +import org.apache.hyracks.util.StorageUtil; + +public class TransactionProperties extends AbstractProperties { + + private static final String TXN_LOG_BUFFER_NUMPAGES_KEY = "txn.log.buffer.numpages"; + private static final int TXN_LOG_BUFFER_NUMPAGES_DEFAULT = 8; + + private static final String TXN_LOG_BUFFER_PAGESIZE_KEY = "txn.log.buffer.pagesize"; + private static final int TXN_LOG_BUFFER_PAGESIZE_DEFAULT = StorageUtil.getSizeInBytes(128, KILOBYTE); + + public static final String TXN_LOG_PARTITIONSIZE_KEY = "txn.log.partitionsize"; + private static final long TXN_LOG_PARTITIONSIZE_DEFAULT = StorageUtil.getSizeInBytes(256L, MEGABYTE); + + private static final String TXN_LOG_CHECKPOINT_LSNTHRESHOLD_KEY = "txn.log.checkpoint.lsnthreshold"; + private static final int TXN_LOG_CHECKPOINT_LSNTHRESHOLD_DEFAULT = StorageUtil.getSizeInBytes(64, MEGABYTE); + + public static final String TXN_LOG_CHECKPOINT_POLLFREQUENCY_KEY = "txn.log.checkpoint.pollfrequency"; + private static final int TXN_LOG_CHECKPOINT_POLLFREQUENCY_DEFAULT = 120; // 120s + + private static final String TXN_LOG_CHECKPOINT_HISTORY_KEY = "txn.log.checkpoint.history"; + private static final int TXN_LOG_CHECKPOINT_HISTORY_DEFAULT = 0; + + private static final String TXN_LOCK_ESCALATIONTHRESHOLD_KEY = "txn.lock.escalationthreshold"; + private static final int TXN_LOCK_ESCALATIONTHRESHOLD_DEFAULT = 1000; + + private static final String TXN_LOCK_SHRINKTIMER_KEY = "txn.lock.shrinktimer"; + private static final int TXN_LOCK_SHRINKTIMER_DEFAULT = 5000; // 5s + + private static final String TXN_LOCK_TIMEOUT_WAITTHRESHOLD_KEY = "txn.lock.timeout.waitthreshold"; + private static final int TXN_LOCK_TIMEOUT_WAITTHRESHOLD_DEFAULT = 60000; // 60s + + private static final String TXN_LOCK_TIMEOUT_SWEEPTHRESHOLD_KEY = "txn.lock.timeout.sweepthreshold"; + private static final int TXN_LOCK_TIMEOUT_SWEEPTHRESHOLD_DEFAULT = 10000; // 10s + + private static final String TXN_COMMIT_PROFILER_REPORT_INTERVAL_KEY = "txn.commitprofiler.reportinterval"; + private static final int TXN_COMMIT_PROFILER_REPORT_INTERVAL_DEFAULT = 5; // 5 seconds + + private static final String TXN_JOB_RECOVERY_MEMORY_SIZE_KEY = "txn.job.recovery.memorysize"; + private static final long TXN_JOB_RECOVERY_MEMORY_SIZE_DEFAULT = StorageUtil.getSizeInBytes(64L, MEGABYTE); + + public TransactionProperties(PropertiesAccessor accessor) { + super(accessor); + } + + public String getLogDirectory(String nodeId) { + return accessor.getTransactionLogDirs().get(nodeId); + } + + public Map getLogDirectories() { + return accessor.getTransactionLogDirs(); + } + + @PropertyKey(TXN_LOG_BUFFER_NUMPAGES_KEY) + public int getLogBufferNumPages() { + return accessor.getProperty(TXN_LOG_BUFFER_NUMPAGES_KEY, TXN_LOG_BUFFER_NUMPAGES_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(TXN_LOG_BUFFER_PAGESIZE_KEY) + public int getLogBufferPageSize() { + return accessor.getProperty(TXN_LOG_BUFFER_PAGESIZE_KEY, TXN_LOG_BUFFER_PAGESIZE_DEFAULT, + PropertyInterpreters.getIntegerBytePropertyInterpreter()); + } + + @PropertyKey(TXN_LOG_PARTITIONSIZE_KEY) + public long getLogPartitionSize() { + return accessor.getProperty(TXN_LOG_PARTITIONSIZE_KEY, TXN_LOG_PARTITIONSIZE_DEFAULT, + PropertyInterpreters.getLongBytePropertyInterpreter()); + } + + @PropertyKey(TXN_LOG_CHECKPOINT_LSNTHRESHOLD_KEY) + public int getCheckpointLSNThreshold() { + return accessor.getProperty(TXN_LOG_CHECKPOINT_LSNTHRESHOLD_KEY, TXN_LOG_CHECKPOINT_LSNTHRESHOLD_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(TXN_LOG_CHECKPOINT_POLLFREQUENCY_KEY) + public int getCheckpointPollFrequency() { + return accessor.getProperty(TXN_LOG_CHECKPOINT_POLLFREQUENCY_KEY, TXN_LOG_CHECKPOINT_POLLFREQUENCY_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(TXN_LOG_CHECKPOINT_HISTORY_KEY) + public int getCheckpointHistory() { + return accessor.getProperty(TXN_LOG_CHECKPOINT_HISTORY_KEY, TXN_LOG_CHECKPOINT_HISTORY_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(TXN_LOCK_ESCALATIONTHRESHOLD_KEY) + public int getEntityToDatasetLockEscalationThreshold() { + return accessor.getProperty(TXN_LOCK_ESCALATIONTHRESHOLD_KEY, TXN_LOCK_ESCALATIONTHRESHOLD_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(TXN_LOCK_SHRINKTIMER_KEY) + public int getLockManagerShrinkTimer() { + return accessor.getProperty(TXN_LOCK_SHRINKTIMER_KEY, TXN_LOCK_SHRINKTIMER_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(TXN_LOCK_TIMEOUT_WAITTHRESHOLD_KEY) + public int getTimeoutWaitThreshold() { + return accessor.getProperty(TXN_LOCK_TIMEOUT_WAITTHRESHOLD_KEY, TXN_LOCK_TIMEOUT_WAITTHRESHOLD_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(TXN_LOCK_TIMEOUT_SWEEPTHRESHOLD_KEY) + public int getTimeoutSweepThreshold() { + return accessor.getProperty(TXN_LOCK_TIMEOUT_SWEEPTHRESHOLD_KEY, TXN_LOCK_TIMEOUT_SWEEPTHRESHOLD_DEFAULT, + PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(TXN_COMMIT_PROFILER_REPORT_INTERVAL_KEY) + public int getCommitProfilerReportInterval() { + return accessor.getProperty(TXN_COMMIT_PROFILER_REPORT_INTERVAL_KEY, + TXN_COMMIT_PROFILER_REPORT_INTERVAL_DEFAULT, PropertyInterpreters.getIntegerPropertyInterpreter()); + } + + @PropertyKey(TXN_JOB_RECOVERY_MEMORY_SIZE_KEY) + public long getJobRecoveryMemorySize() { + return accessor.getProperty(TXN_JOB_RECOVERY_MEMORY_SIZE_KEY, TXN_JOB_RECOVERY_MEMORY_SIZE_DEFAULT, + PropertyInterpreters.getLongBytePropertyInterpreter()); + } +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java deleted file mode 100644 index e74659d..0000000 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixFileMapManager.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.common.context; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.api.io.FileReference; -import org.apache.hyracks.api.io.IIOManager; -import org.apache.hyracks.storage.common.file.IFileMapManager; - -public class AsterixFileMapManager implements IFileMapManager { - - private static final long serialVersionUID = 1L; - private final transient IIOManager ioManager; - private final Map id2nameMap = new HashMap<>(); - private final Map name2IdMap = new HashMap<>(); - private int idCounter = 0; - - public AsterixFileMapManager(IIOManager ioManager) { - this.ioManager = ioManager; - } - - @Override - public FileReference lookupFileName(int fileId) throws HyracksDataException { - String fName = id2nameMap.get(fileId); - if (fName == null) { - throw new HyracksDataException("No mapping found for id: " + fileId); - } - return ioManager.resolveAbsolutePath(fName); - } - - @Override - public int lookupFileId(FileReference fileRef) throws HyracksDataException { - String fileName = fileRef.getFile().getAbsolutePath(); - Integer fileId = name2IdMap.get(fileName); - if (fileId == null) { - throw new HyracksDataException("No mapping found for name: " + fileName); - } - return fileId; - } - - @Override - public boolean isMapped(FileReference fileRef) { - String fileName = fileRef.getFile().getAbsolutePath(); - return name2IdMap.containsKey(fileName); - } - - @Override - public boolean isMapped(int fileId) { - return id2nameMap.containsKey(fileId); - } - - @Override - public void unregisterFile(int fileId) throws HyracksDataException { - String fileName = id2nameMap.remove(fileId); - name2IdMap.remove(fileName); - } - - @Override - public void registerFile(FileReference fileRef) throws HyracksDataException { - Integer fileId = idCounter++; - String fileName = fileRef.getFile().getAbsolutePath(); - id2nameMap.put(fileId, fileName); - name2IdMap.put(fileName, fileId); - } - - @Override - public int registerMemoryFile() { - Integer fileId = idCounter++; - id2nameMap.put(fileId, null); - return fileId; - } - - @Override - public void unregisterMemFile(int fileId) throws HyracksDataException { - id2nameMap.remove(fileId); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java index 6fbbfe4..af972b6 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/AsterixVirtualBufferCacheProvider.java @@ -20,7 +20,7 @@ package org.apache.asterix.common.context; import java.util.List; -import org.apache.asterix.common.api.IAsterixAppRuntimeContext; +import org.apache.asterix.common.api.IAppRuntimeContext; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; @@ -56,7 +56,7 @@ public class AsterixVirtualBufferCacheProvider implements IVirtualBufferCachePro deviceId = i; } } - return ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject()) + return ((IAppRuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject()) .getDatasetLifecycleManager().getVirtualBufferCaches(datasetID, deviceId); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicyFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicyFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicyFactory.java index 3b65123..b2bcf17 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicyFactory.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/CorrelatedPrefixMergePolicyFactory.java @@ -24,7 +24,7 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import org.apache.asterix.common.api.IAsterixAppRuntimeContext; +import org.apache.asterix.common.api.IAppRuntimeContext; import org.apache.asterix.common.api.IDatasetLifecycleManager; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.storage.am.common.api.IResourceLifecycleManager; @@ -43,7 +43,7 @@ public class CorrelatedPrefixMergePolicyFactory implements ILSMMergePolicyFactor @Override public ILSMMergePolicy createMergePolicy(Map properties, IHyracksTaskContext ctx) { - IDatasetLifecycleManager dslcManager = ((IAsterixAppRuntimeContext) ctx.getJobletContext() + IDatasetLifecycleManager dslcManager = ((IAppRuntimeContext) ctx.getJobletContext() .getApplicationContext().getApplicationObject()).getDatasetLifecycleManager(); ILSMMergePolicy policy = new CorrelatedPrefixMergePolicy(dslcManager, datasetID); policy.configure(properties); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java index fd4a9b5..f49b07a 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java @@ -27,7 +27,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.asterix.common.api.IDatasetLifecycleManager; -import org.apache.asterix.common.config.AsterixStorageProperties; +import org.apache.asterix.common.config.StorageProperties; import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.common.ioopcallbacks.AbstractLSMIOOperationCallback; import org.apache.asterix.common.transactions.ILogManager; @@ -47,7 +47,7 @@ import org.apache.hyracks.storage.common.file.LocalResource; public class DatasetLifecycleManager implements IDatasetLifecycleManager, ILifeCycleComponent { private final Map datasets = new ConcurrentHashMap<>(); - private final AsterixStorageProperties storageProperties; + private final StorageProperties storageProperties; private final ILocalResourceRepository resourceRepository; private final int firstAvilableUserDatasetID; private final long capacity; @@ -57,7 +57,7 @@ public class DatasetLifecycleManager implements IDatasetLifecycleManager, ILifeC private final int numPartitions; private volatile boolean stopped = false; - public DatasetLifecycleManager(AsterixStorageProperties storageProperties, + public DatasetLifecycleManager(StorageProperties storageProperties, ILocalResourceRepository resourceRepository, int firstAvilableUserDatasetID, ILogManager logManager, int numPartitions) { this.logManager = logManager; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetVirtualBufferCaches.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetVirtualBufferCaches.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetVirtualBufferCaches.java index df85509..c7eda4d 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetVirtualBufferCaches.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetVirtualBufferCaches.java @@ -23,7 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.asterix.common.config.AsterixStorageProperties; +import org.apache.asterix.common.config.StorageProperties; import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache; import org.apache.hyracks.storage.am.lsm.common.impls.MultitenantVirtualBufferCache; import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache; @@ -32,12 +32,12 @@ import org.apache.hyracks.storage.common.buffercache.ResourceHeapBufferAllocator public class DatasetVirtualBufferCaches { private final int datasetID; - private final AsterixStorageProperties storageProperties; + private final StorageProperties storageProperties; private final int firstAvilableUserDatasetID; private final int numPartitions; private final Map> ioDeviceVirtualBufferCaches = new HashMap<>(); - public DatasetVirtualBufferCaches(int datasetID, AsterixStorageProperties storageProperties, + public DatasetVirtualBufferCaches(int datasetID, StorageProperties storageProperties, int firstAvilableUserDatasetID, int numPartitions) { this.datasetID = datasetID; this.storageProperties = storageProperties; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f76d9c37/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/FileMapManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/FileMapManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/FileMapManager.java new file mode 100644 index 0000000..c492661 --- /dev/null +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/FileMapManager.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.common.context; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.io.IIOManager; +import org.apache.hyracks.storage.common.file.IFileMapManager; + +public class FileMapManager implements IFileMapManager { + + private static final long serialVersionUID = 1L; + private final transient IIOManager ioManager; + private final Map id2nameMap = new HashMap<>(); + private final Map name2IdMap = new HashMap<>(); + private int idCounter = 0; + + public FileMapManager(IIOManager ioManager) { + this.ioManager = ioManager; + } + + @Override + public FileReference lookupFileName(int fileId) throws HyracksDataException { + String fName = id2nameMap.get(fileId); + if (fName == null) { + throw new HyracksDataException("No mapping found for id: " + fileId); + } + return ioManager.resolveAbsolutePath(fName); + } + + @Override + public int lookupFileId(FileReference fileRef) throws HyracksDataException { + String fileName = fileRef.getFile().getAbsolutePath(); + Integer fileId = name2IdMap.get(fileName); + if (fileId == null) { + throw new HyracksDataException("No mapping found for name: " + fileName); + } + return fileId; + } + + @Override + public boolean isMapped(FileReference fileRef) { + String fileName = fileRef.getFile().getAbsolutePath(); + return name2IdMap.containsKey(fileName); + } + + @Override + public boolean isMapped(int fileId) { + return id2nameMap.containsKey(fileId); + } + + @Override + public void unregisterFile(int fileId) throws HyracksDataException { + String fileName = id2nameMap.remove(fileId); + name2IdMap.remove(fileName); + } + + @Override + public void registerFile(FileReference fileRef) throws HyracksDataException { + Integer fileId = idCounter++; + String fileName = fileRef.getFile().getAbsolutePath(); + id2nameMap.put(fileId, fileName); + name2IdMap.put(fileName, fileId); + } + + @Override + public int registerMemoryFile() { + Integer fileId = idCounter++; + id2nameMap.put(fileId, null); + return fileId; + } + + @Override + public void unregisterMemFile(int fileId) throws HyracksDataException { + id2nameMap.remove(fileId); + } +}