asterixdb-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Chris Hillery (Code Review)" <do-not-re...@asterix-gerrit.ics.uci.edu>
Subject Change in asterixdb[master]: WIP for Asterix configuration changes.
Date Fri, 31 Jul 2015 11:09:09 GMT
Chris Hillery has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/336

Change subject: WIP for Asterix configuration changes.
......................................................................

WIP for Asterix configuration changes.

Change-Id: Ie3027c8c839f25ea858790bd3340187f4b11f212
---
M asterix-app/pom.xml
M asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
M asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
M asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
M asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java
M asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java
M asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
M asterix-server/pom.xml
M pom.xml
9 files changed, 135 insertions(+), 55 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/36/336/1

diff --git a/asterix-app/pom.xml b/asterix-app/pom.xml
index 25f1ad9..8ee5644 100644
--- a/asterix-app/pom.xml
+++ b/asterix-app/pom.xml
@@ -128,6 +128,10 @@
             <artifactId>hyracks-client</artifactId>
         </dependency>
         <dependency>
+            <groupId>edu.uci.ics.hyracks</groupId>
+            <artifactId>hyracks-api</artifactId>
+        </dependency>
+        <dependency>
             <groupId>edu.uci.ics.asterix</groupId>
             <artifactId>asterix-algebra</artifactId>
             <version>0.8.7-SNAPSHOT</version>
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
index 6d7f2a4..ef6267f 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
@@ -41,6 +41,7 @@
 import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceRepositoryFactory;
 import edu.uci.ics.asterix.transaction.management.service.logging.LogManager;
 import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionSubsystem;
+import edu.uci.ics.hyracks.api.application.IApplicationConfig;
 import edu.uci.ics.hyracks.api.application.INCApplicationContext;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 import edu.uci.ics.hyracks.api.io.IIOManager;
@@ -69,16 +70,6 @@
 import edu.uci.ics.hyracks.storage.common.file.ResourceIdFactoryProvider;
 
 public class AsterixAppRuntimeContext implements IAsterixAppRuntimeContext, IAsterixPropertiesProvider
{
-
-    private static final AsterixPropertiesAccessor ASTERIX_PROPERTIES_ACCESSOR;
-
-    static {
-        try {
-            ASTERIX_PROPERTIES_ACCESSOR = new AsterixPropertiesAccessor();
-        } catch (AsterixException e) {
-            throw new ExceptionInInitializerError(e);
-        }
-    }
 
     private static final int METADATA_IO_DEVICE_ID = 0;
 
@@ -109,12 +100,22 @@
 
     public AsterixAppRuntimeContext(INCApplicationContext ncApplicationContext) throws AsterixException
{
         this.ncApplicationContext = ncApplicationContext;
-        compilerProperties = new AsterixCompilerProperties(ASTERIX_PROPERTIES_ACCESSOR);
-        externalProperties = new AsterixExternalProperties(ASTERIX_PROPERTIES_ACCESSOR);
-        metadataProperties = new AsterixMetadataProperties(ASTERIX_PROPERTIES_ACCESSOR);
-        storageProperties = new AsterixStorageProperties(ASTERIX_PROPERTIES_ACCESSOR);
-        txnProperties = new AsterixTransactionProperties(ASTERIX_PROPERTIES_ACCESSOR);
-        feedProperties = new AsterixFeedProperties(ASTERIX_PROPERTIES_ACCESSOR);
+        // Determine whether to use old-style asterix-configuration.xml or new-style configuration.
+        // QQQ strip this out eventually
+        AsterixPropertiesAccessor propertiesAccessor;
+        IApplicationConfig cfg = ncApplicationContext.getAppConfig();
+        // QQQ this is NOT a good way to determine whether the config is valid
+        if (cfg.getString("cc", "cluster.address") != null) {
+            propertiesAccessor = new AsterixPropertiesAccessor(cfg);
+        } else {
+            propertiesAccessor = new AsterixPropertiesAccessor();;
+        }
+        compilerProperties = new AsterixCompilerProperties(propertiesAccessor);
+        externalProperties = new AsterixExternalProperties(propertiesAccessor);
+        metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
+        storageProperties = new AsterixStorageProperties(propertiesAccessor);
+        txnProperties = new AsterixTransactionProperties(propertiesAccessor);
+        feedProperties = new AsterixFeedProperties(propertiesAccessor);
     }
 
     public void initialize() throws IOException, ACIDException, AsterixException {
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 956b447..399a993 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -21,6 +21,7 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import edu.uci.ics.hyracks.api.application.IApplicationConfig;
 import org.kohsuke.args4j.CmdLineException;
 import org.kohsuke.args4j.CmdLineParser;
 import org.kohsuke.args4j.Option;
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
index fc8b3df..d800009 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
@@ -19,6 +19,7 @@
 import java.io.InputStream;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -35,18 +36,24 @@
 import edu.uci.ics.asterix.common.configuration.Store;
 import edu.uci.ics.asterix.common.configuration.TransactionLogDir;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.hyracks.api.application.IApplicationConfig;
 
 public class AsterixPropertiesAccessor {
     private static final Logger LOGGER = Logger.getLogger(AsterixPropertiesAccessor.class.getName());
 
     private final String instanceName;
     private final String metadataNodeName;
-    private final Set<String> nodeNames;
-    private final Map<String, String[]> stores;
-    private final Map<String, String> coredumpConfig;
+    private final Set<String> nodeNames = new HashSet<>();
+    private final Map<String, String[]> stores = new HashMap<>();
+    private final Map<String, String> coredumpConfig = new HashMap<>();
+    private final Map<String, String> transactionLogDirs = new HashMap<>();
     private final Map<String, Property> asterixConfigurationParams;
-    private final Map<String, String> transactionLogDirs;
+    private final IApplicationConfig cfg;
 
+    /**
+     * Constructor which reads asterix-configuration.xml, the old way.
+     * @throws AsterixException
+     */
     public AsterixPropertiesAccessor() throws AsterixException {
         String fileName = System.getProperty(GlobalConfig.CONFIG_FILE_PROPERTY);
         if (fileName == null) {
@@ -72,9 +79,7 @@
         }
         instanceName = asterixConfiguration.getInstanceName();
         metadataNodeName = asterixConfiguration.getMetadataNode();
-        stores = new HashMap<String, String[]>();
         List<Store> configuredStores = asterixConfiguration.getStore();
-        nodeNames = new HashSet<String>();
         for (Store store : configuredStores) {
             String trimmedStoreDirs = store.getStoreDirs().trim();
             stores.put(store.getNcId(), trimmedStoreDirs.split(","));
@@ -84,15 +89,50 @@
         for (Property p : asterixConfiguration.getProperty()) {
             asterixConfigurationParams.put(p.getName(), p);
         }
-        coredumpConfig = new HashMap<String, String>();
         for (Coredump cd : asterixConfiguration.getCoredump()) {
             coredumpConfig.put(cd.getNcId(), cd.getCoredumpPath());
         }
-        transactionLogDirs = new HashMap<String, String>();
         for (TransactionLogDir txnLogDir : asterixConfiguration.getTransactionLogDir()) {
             transactionLogDirs.put(txnLogDir.getNcId(), txnLogDir.getTxnLogDirPath());
         }
 
+        cfg = null;
+    }
+
+    /**
+     * Constructor which wraps an IApplicationConfig.
+     */
+    public AsterixPropertiesAccessor (IApplicationConfig cfg) {
+        this.cfg = cfg;
+        instanceName = cfg.getString("asterix", "instance", "DEFAULT_INSTANCE");
+        String mdNode = null;
+        for (String section : cfg.getSections()) {
+            if (!section.startsWith("nc/")) {
+                continue;
+            }
+            String ncId = section.substring(3);
+            nodeNames.add(ncId);
+
+            if (mdNode == null) {
+                // Default is first node == metadata node
+                mdNode = ncId;
+            }
+            if (cfg.getString(section, "metadata.port") != null) {
+                // QQQ But we don't actually *honor* metadata.port yet!
+                mdNode = ncId;
+            }
+
+            // QQQ Default values? Should they be specified here? Or should there
+            // be a default.ini? They can't be inserted by TriggerNCWork except
+            // possibly for hyracks-specified values. Certainly wherever they are,
+            // they should be platform-dependent.
+            stores.put(ncId, cfg.getString(section, "iodevices", "/var/lib/asterixdb/data").split(","));
+            coredumpConfig.put(ncId, cfg.getString(section, "coredumpdir", "/var/lib/asterixdb/coredump"));
+            transactionLogDirs.put(ncId, cfg.getString(section, "txnlogdir", "/var/lib/asterixdb/txn-log"));
+        }
+
+        metadataNodeName = mdNode;
+        asterixConfigurationParams = null;
     }
 
     public String getMetadataNodeName() {
@@ -138,24 +178,33 @@
     }
 
     public <T> T getProperty(String property, T defaultValue, IPropertyInterpreter<T>
interpreter) {
-        Property p = asterixConfigurationParams.get(property);
-        if (p == null) {
+        String value;
+        Property p = null;
+        if (asterixConfigurationParams != null) {
+            p = asterixConfigurationParams.get(property);
+            value = (p == null) ? null : p.getValue();
+        } else {
+            value = cfg.getString("asterix", property);
+        }
+        if (value == null) {
             return defaultValue;
         }
-
         try {
-            return interpreter.interpret(p);
+            return interpreter.interpret(value);
         } catch (IllegalArgumentException e) {
-            logConfigurationError(p, defaultValue);
+            if (LOGGER.isLoggable(Level.SEVERE)) {
+                StringBuilder msg = new StringBuilder("Invalid property value '" + value
+ "' for property '" + property + "'.\n");
+                if (p != null) {
+                    msg.append("See the description: \n" + p.getDescription() + "\n");
+                }
+                msg.append("Default = " + defaultValue);
+                LOGGER.severe(msg.toString());
+            }
             throw e;
         }
     }
 
     private <T> void logConfigurationError(Property p, T defaultValue) {
-        if (LOGGER.isLoggable(Level.SEVERE)) {
-            LOGGER.severe("Invalid property value '" + p.getValue() + "' for property '"
+ p.getName()
-                    + "'.\n See the description: \n" + p.getDescription() + "\nDefault =
" + defaultValue);
-        }
     }
 
     public String getInstanceName() {
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java
b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java
index 67acf4b..c4acdd9 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java
@@ -17,5 +17,5 @@
 import edu.uci.ics.asterix.common.configuration.Property;
 
 public interface IPropertyInterpreter<T> {
-    public T interpret(Property p) throws IllegalArgumentException;
+    public T interpret(String s) throws IllegalArgumentException;
 }
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java
b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java
index 465d7c5..1c7256b 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java
@@ -22,11 +22,10 @@
 
     public static IPropertyInterpreter<Integer> getIntegerPropertyInterpreter() {
         return new IPropertyInterpreter<Integer>() {
-
             @Override
-            public Integer interpret(Property p) throws IllegalArgumentException {
+            public Integer interpret(String s) throws IllegalArgumentException {
                 try {
-                    return Integer.parseInt(p.getValue());
+                    return Integer.parseInt(s);
                 } catch (NumberFormatException e) {
                     throw new IllegalArgumentException(e);
                 }
@@ -36,20 +35,19 @@
 
     public static IPropertyInterpreter<Boolean> getBooleanPropertyInterpreter() {
         return new IPropertyInterpreter<Boolean>() {
-
-            public Boolean interpret(Property p) throws IllegalArgumentException {
-                return Boolean.parseBoolean(p.getValue());
+            @Override
+            public Boolean interpret(String s) throws IllegalArgumentException {
+                return Boolean.parseBoolean(s);
             }
         };
     }
 
     public static IPropertyInterpreter<Long> getLongPropertyInterpreter() {
         return new IPropertyInterpreter<Long>() {
-
             @Override
-            public Long interpret(Property p) throws IllegalArgumentException {
+            public Long interpret(String s) throws IllegalArgumentException {
                 try {
-                    return Long.parseLong(p.getValue());
+                    return Long.parseLong(s);
                 } catch (NumberFormatException e) {
                     throw new IllegalArgumentException(e);
                 }
@@ -59,31 +57,28 @@
 
     public static IPropertyInterpreter<Level> getLevelPropertyInterpreter() {
         return new IPropertyInterpreter<Level>() {
-
             @Override
-            public Level interpret(Property p) throws IllegalArgumentException {
-                return Level.parse(p.getValue());
+            public Level interpret(String s) throws IllegalArgumentException {
+                return Level.parse(s);
             }
         };
     }
 
     public static IPropertyInterpreter<String> getStringPropertyInterpreter() {
         return new IPropertyInterpreter<String>() {
-
             @Override
-            public String interpret(Property p) throws IllegalArgumentException {
-                return p.getValue();
+            public String interpret(String s) throws IllegalArgumentException {
+                return s;
             }
         };
     }
 
     public static IPropertyInterpreter<Double> getDoublePropertyInterpreter() {
         return new IPropertyInterpreter<Double>() {
-
             @Override
-            public Double interpret(Property p) throws IllegalArgumentException {
+            public Double interpret(String s) throws IllegalArgumentException {
                 try {
-                    return Double.parseDouble(p.getValue());
+                    return Double.parseDouble(s);
                 } catch (NumberFormatException e) {
                     throw new IllegalArgumentException(e);
                 }
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
index fece403..2ff248b 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
@@ -27,6 +27,7 @@
 import edu.uci.ics.asterix.common.dataflow.IAsterixApplicationContextInfo;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
+import edu.uci.ics.hyracks.api.application.IApplicationConfig;
 import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
@@ -52,10 +53,21 @@
     private IHyracksClientConnection hcc;
 
     public static void initialize(ICCApplicationContext ccAppCtx, IHyracksClientConnection
hcc) throws AsterixException {
-        if (INSTANCE == null) {
-            INSTANCE = new AsterixAppContextInfo(ccAppCtx, hcc);
+        if (INSTANCE != null) {
+            return;
         }
-        AsterixPropertiesAccessor propertiesAccessor = new AsterixPropertiesAccessor();
+        INSTANCE = new AsterixAppContextInfo(ccAppCtx, hcc);
+
+        // Determine whether to use old-style asterix-configuration.xml or new-style configuration.
+        // QQQ strip this out eventually
+        AsterixPropertiesAccessor propertiesAccessor;
+        IApplicationConfig cfg = ccAppCtx.getAppConfig();
+        // QQQ this is NOT a good way to determine whether the config is valid
+        if (cfg.getString("cc", "cluster.address") != null) {
+            propertiesAccessor = new AsterixPropertiesAccessor(cfg);
+        } else {
+            propertiesAccessor = new AsterixPropertiesAccessor();
+        }
         INSTANCE.compilerProperties = new AsterixCompilerProperties(propertiesAccessor);
         INSTANCE.externalProperties = new AsterixExternalProperties(propertiesAccessor);
         INSTANCE.metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
diff --git a/asterix-server/pom.xml b/asterix-server/pom.xml
index 203e5d2..a542212 100644
--- a/asterix-server/pom.xml
+++ b/asterix-server/pom.xml
@@ -65,6 +65,13 @@
                 <commandLineArgument>edu.uci.ics.asterix.hyracks.bootstrap.NCApplicationEntryPoint</commandLineArgument>
               </commandLineArguments>
             </program>
+            <program>
+              <platforms>
+                <platform>unix</platform>
+              </platforms>
+              <name>asterixncservice</name>
+              <mainClass>edu.uci.ics.hyracks.control.nc.service.NCService</mainClass>
+            </program>
           </programs>
           <daemons>
             <daemon>
@@ -134,6 +141,12 @@
   		<scope>compile</scope>
   	</dependency>
   	<dependency>
+  		<groupId>edu.uci.ics.hyracks</groupId>
+  		<artifactId>hyracks-nc-service</artifactId>
+  		<type>jar</type>
+  		<scope>compile</scope>
+  	</dependency>
+  	<dependency>
   		<groupId>edu.uci.ics.asterix</groupId>
   		<artifactId>asterix-app</artifactId>
   		<version>0.8.7-SNAPSHOT</version>
diff --git a/pom.xml b/pom.xml
index 6f4217e..caa7d3b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -357,6 +357,11 @@
             </dependency>
             <dependency>
                 <groupId>edu.uci.ics.hyracks</groupId>
+                <artifactId>hyracks-nc-service</artifactId>
+                <version>${hyracks.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>edu.uci.ics.hyracks</groupId>
                 <artifactId>hyracks-server</artifactId>
                 <version>${hyracks.version}</version>
             </dependency>

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/336
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie3027c8c839f25ea858790bd3340187f4b11f212
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Chris Hillery <ceej@lambda.nu>

Mime
View raw message