asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Murtadha Hubail (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: [ASTERIXDB-2276][CONF] Introduce Max Active Writable Datasets
Date Wed, 07 Feb 2018 16:43:10 GMT
Murtadha Hubail has submitted this change and it was merged.

Change subject: [ASTERIXDB-2276][CONF] Introduce Max Active Writable Datasets
......................................................................


[ASTERIXDB-2276][CONF] Introduce Max Active Writable Datasets

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Introduce number of metadata datasets parameter
  and use it to reserve memory for all metadata
  datasets.
- Default metadata dataset memory component number
  of pages to 32 pages.
- Introduce max active writable datasets and use
  it to automatically calculate the number of pages
  of a user dataset memory component.
- Remve the assumpation of reserving an NC core
  for heartbeats.
- Fix for [ASTERIXDB-2279] by making the expected
  result order deterministic.

Change-Id: I9909c26b1e12b431f913e201d2c3d83769be7269
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2362
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
---
M asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
M asterixdb/asterix-app/src/main/resources/cc.conf
M asterixdb/asterix-app/src/main/resources/cc2.conf
M asterixdb/asterix-app/src/main/resources/cc3.conf
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
M asterixdb/asterix-app/src/test/resources/cc-multipart.conf
M asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
M asterixdb/asterix-app/src/test/resources/cc.conf
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
M asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
14 files changed, 56 insertions(+), 56 deletions(-)

Approvals:
  Anon. E. Moose #1000171: 
  Jenkins: Verified; No violations found; ; Verified
  Michael Blow: Looks good to me, approved



diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 1220462..b0382f7 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -222,19 +222,9 @@
     @Override
     public NodeCapacity getCapacity() {
         StorageProperties storageProperties = runtimeContext.getStorageProperties();
-        // Deducts the reserved buffer cache size and memory component size from the maxium
heap size,
-        // and deducts one core for processing heartbeats.
-        long memorySize = Runtime.getRuntime().maxMemory() - storageProperties.getBufferCacheSize()
-                - storageProperties.getMemoryComponentGlobalBudget();
-        if (memorySize <= 0) {
-            throw new IllegalStateException("Invalid node memory configuration, more memory
budgeted than available "
-                    + "in JVM. Runtime max memory: " + Runtime.getRuntime().maxMemory() +
" Buffer cache size: "
-                    + storageProperties.getBufferCacheSize() + " Memory component global
budget: "
-                    + storageProperties.getMemoryComponentGlobalBudget());
-        }
+        final long memorySize = storageProperties.getJobExecutionMemoryBudget();
         int allCores = Runtime.getRuntime().availableProcessors();
-        int maximumCoresForComputation = allCores > 1 ? allCores - 1 : allCores;
-        return new NodeCapacity(memorySize, maximumCoresForComputation);
+        return new NodeCapacity(memorySize, allCores);
     }
 
     private void performLocalCleanUp() throws HyracksDataException {
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf
index 914b7b6..fe6cf64 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -37,7 +37,6 @@
 jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
 storage.buffercache.pagesize=32KB
 storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
 storage.memorycomponent.globalbudget=512MB
 
 [cc]
diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf
index 6c01386..113497d 100644
--- a/asterixdb/asterix-app/src/main/resources/cc2.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc2.conf
@@ -37,7 +37,6 @@
 jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
 storage.buffercache.pagesize=32KB
 storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
 storage.memorycomponent.globalbudget=512MB
 
 [cc]
diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf b/asterixdb/asterix-app/src/main/resources/cc3.conf
index 933e4af..e210bf62 100644
--- a/asterixdb/asterix-app/src/main/resources/cc3.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc3.conf
@@ -37,7 +37,6 @@
 jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
 storage.buffercache.pagesize=32KB
 storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
 storage.memorycomponent.globalbudget=512MB
 
 [cc]
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
index f029669..e752493 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateDefaultParameterTest.java
@@ -87,8 +87,8 @@
                 Assert.assertTrue(getValue(row) == maxHeap / 4);
                 matchCount++;
             }
-            if (row.contains("storage.memorycomponent.numpages")) {
-                Assert.assertTrue(getValue(row) == maxHeap / (131072 * 64));
+            if (row.contains("storage.max.active.writable.datasets")) {
+                Assert.assertTrue(getValue(row) == 8);
                 matchCount++;
             }
         }
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
index 92e403c..05e5aad 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
@@ -48,10 +48,7 @@
 
     @Before
     public void setUp() throws Exception {
-        // Read default test configurations
-        // Write test config file
-        integrationUtil.addOption(StorageProperties.Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET,
"128MB");
-        integrationUtil.addOption(StorageProperties.Option.STORAGE_MEMORYCOMPONENT_NUMPAGES,
32);
+        integrationUtil.addOption(StorageProperties.Option.STORAGE_MAX_ACTIVE_WRITABLE_DATASETS,
20);
         integrationUtil.setGracefulShutdown(false);
         integrationUtil.init(true, TEST_CONFIG_FILE_PATH);
     }
diff --git a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
index f28545d..5cf1bbe 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
@@ -38,7 +38,6 @@
 storage.subdir=test_storage
 storage.buffercache.pagesize=32KB
 storage.buffercache.size=48MB
-storage.memorycomponent.numpages=32
 storage.memorycomponent.globalbudget=512MB
 
 [cc]
diff --git a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
index 0b66cfe..6cd3b87 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
@@ -35,8 +35,7 @@
 jvm.args=-Xmx4096m -Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
 storage.buffercache.pagesize=32KB
 storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
-storage.memorycomponent.globalbudget=512MB
+storage.memorycomponent.globalbudget=128MB
 
 [cc]
 address = 127.0.0.1
@@ -53,4 +52,5 @@
 messaging.frame.size=4096
 messaging.frame.count=512
 txn.log.partitionsize=2MB
-txn.log.checkpoint.pollfrequency=2147483647
\ No newline at end of file
+txn.log.checkpoint.pollfrequency=2147483647
+storage.max.active.writable.datasets=50
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf
index 3e4ecd9..5cf1bbe 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -38,7 +38,6 @@
 storage.subdir=test_storage
 storage.buffercache.pagesize=32KB
 storage.buffercache.size=48MB
-storage.memorycomponent.numpages=16
 storage.memorycomponent.globalbudget=512MB
 
 [cc]
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp
index 6856726..cd1bcfc 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-971/query-ASTERIXDB-971.3.query.sqlpp
@@ -23,4 +23,5 @@
 SELECT ELEMENT (
     WITH z AS (FROM Accounts AS y WHERE y.id=x SELECT VALUE y)
     SELECT x, z
-);
+)
+ORDER BY x;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 85b121e..59057b1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -28,6 +28,7 @@
     "replication\.log\.buffer\.pagesize" : 131072,
     "replication\.strategy" : "none",
     "replication\.timeout" : 30,
+    "storage.max.active.writable.datasets" : 8,
     "txn\.commitprofiler\.enabled" : false,
     "txn\.commitprofiler\.reportinterval" : 5,
     "txn\.job\.recovery\.memorysize" : 67108864,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 7017b06..13dacb8 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -28,6 +28,7 @@
     "replication\.log\.buffer\.pagesize" : 131072,
     "replication\.strategy" : "none",
     "replication\.timeout" : 30,
+    "storage.max.active.writable.datasets" : 8,
     "txn\.commitprofiler\.enabled" : false,
     "txn\.commitprofiler\.reportinterval" : 5,
     "txn\.job\.recovery\.memorysize" : 67108864,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 00e4d67..9f18638 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -28,6 +28,7 @@
     "replication\.log\.buffer\.pagesize" : 131072,
     "replication\.strategy" : "none",
     "replication\.timeout" : 30,
+    "storage.max.active.writable.datasets" : 8,
     "txn\.commitprofiler\.enabled" : false,
     "txn\.commitprofiler\.reportinterval" : 5,
     "txn\.job\.recovery\.memorysize" : 67108864,
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index 963f0ca..e01198f 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -42,16 +42,11 @@
         STORAGE_BUFFERCACHE_MAXOPENFILES(INTEGER, Integer.MAX_VALUE),
         STORAGE_MEMORYCOMPONENT_GLOBALBUDGET(LONG_BYTE_UNIT, Runtime.getRuntime().maxMemory()
/ 4),
         STORAGE_MEMORYCOMPONENT_PAGESIZE(INTEGER_BYTE_UNIT, StorageUtil.getIntSizeInBytes(128,
KILOBYTE)),
-        STORAGE_MEMORYCOMPONENT_NUMPAGES(INTEGER, (Function<IApplicationConfig, Integer>)
accessor ->
-        // By default, uses 1/16 of the STORAGE_MEMORYCOMPONENT_GLOBALBUDGET for the write
buffer
-        // budget for a dataset, including data and indexes.
-        (int) (accessor.getLong(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) / (16 * accessor.getInt(STORAGE_MEMORYCOMPONENT_PAGESIZE)))),
         STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS(INTEGER, 2),
-        STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES(INTEGER, (Function<IApplicationConfig,
Integer>) accessor ->
-        // By default, uses the min of 1/64 of the STORAGE_MEMORYCOMPONENT_GLOBALBUDGET and
256 pages
-        // for the write buffer budget for a metadata dataset, including data and indexes.
-        Math.min((int) (accessor.getLong(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET) / (64 * accessor.getInt(STORAGE_MEMORYCOMPONENT_PAGESIZE))),
256)),
-        STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d);
+        STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES(INTEGER, 32),
+        STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d),
+        STORAGE_METADATA_DATASETS(INTEGER, 14),
+        STORAGE_MAX_ACTIVE_WRITABLE_DATASETS(INTEGER, 8);
 
         private final IOptionType interpreter;
         private final Object defaultValue;
@@ -61,13 +56,11 @@
             this.defaultValue = defaultValue;
         }
 
-        <T> Option(IOptionType<T> interpreter, Function<IApplicationConfig,
T> defaultValueFunction) {
-            this.interpreter = interpreter;
-            this.defaultValue = defaultValueFunction;
-        }
-
         @Override
         public Section section() {
+            if (this == STORAGE_MAX_ACTIVE_WRITABLE_DATASETS) {
+                return Section.COMMON;
+            }
             return Section.NC;
         }
 
@@ -86,17 +79,16 @@
                             + "of the memory component page size";
                 case STORAGE_MEMORYCOMPONENT_PAGESIZE:
                     return "The page size in bytes for pages allocated to memory components";
-                case STORAGE_MEMORYCOMPONENT_NUMPAGES:
-                    return "The number of pages to allocate for a memory component.  This
budget is shared by all "
-                            + "the memory components of the primary index and all its secondary
indexes across all I/O "
-                            + "devices on a node.  Note: in-memory components usually has
fill factor of 75% since "
-                            + "the pages are 75% full and the remaining 25% is un-utilized";
                 case STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS:
                     return "The number of memory components to be used per lsm index";
                 case STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES:
                     return "The number of pages to allocate for a metadata memory component";
                 case STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE:
                     return "The maximum acceptable false positive rate for bloom filters
associated with LSM indexes";
+                case STORAGE_METADATA_DATASETS:
+                    return "The number of metadata datasets";
+                case STORAGE_MAX_ACTIVE_WRITABLE_DATASETS:
+                    return "The maximum number of datasets that can be concurrently modified";
                 default:
                     throw new IllegalStateException("NYI: " + this);
             }
@@ -114,16 +106,10 @@
 
         @Override
         public String usageDefaultOverride(IApplicationConfig accessor, Function<IOption,
String> optionPrinter) {
-            switch (this) {
-                case STORAGE_MEMORYCOMPONENT_NUMPAGES:
-                    return "1/16th of the " + optionPrinter.apply(Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET)
-                            + " value";
-                case STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES:
-                    return "1/64th of the " + optionPrinter.apply(Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET)
-                            + " value or 256, whichever is larger";
-                default:
-                    return null;
+            if (this == STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES) {
+                return "32 pages";
             }
+            return null;
         }
     }
 
@@ -148,7 +134,10 @@
     }
 
     public int getMemoryComponentNumPages() {
-        return accessor.getInt(Option.STORAGE_MEMORYCOMPONENT_NUMPAGES);
+        final long metadataReservedMem = getMetadataReservedMemory();
+        final long globalUserDatasetMem = getMemoryComponentGlobalBudget() - metadataReservedMem;
+        final long userDatasetMem = globalUserDatasetMem / getMaxActiveWritableDatasets();
+        return (int) (userDatasetMem / getMemoryComponentPageSize());
     }
 
     public int getMetadataMemoryComponentNumPages() {
@@ -170,4 +159,29 @@
     public int getBufferCacheNumPages() {
         return (int) (getBufferCacheSize() / (getBufferCachePageSize() + IBufferCache.RESERVED_HEADER_BYTES));
     }
+
+    public long getJobExecutionMemoryBudget() {
+        final long jobExecutionMemory =
+                Runtime.getRuntime().maxMemory() - getBufferCacheSize() - getMemoryComponentGlobalBudget();
+        if (jobExecutionMemory <= 0) {
+            final String msg = String.format(
+                    "Invalid node memory configuration, more memory budgeted than available
in JVM. Runtime max memory:"
+                            + " (%d), Buffer cache memory (%d), memory component global budget
(%d)",
+                    Runtime.getRuntime().maxMemory(), getBufferCacheSize(), getMemoryComponentGlobalBudget());
+            throw new IllegalStateException(msg);
+        }
+        return jobExecutionMemory;
+    }
+
+    public int getMaxActiveWritableDatasets() {
+        return accessor.getInt(Option.STORAGE_MAX_ACTIVE_WRITABLE_DATASETS);
+    }
+
+    private int getMetadataDatasets() {
+        return accessor.getInt(Option.STORAGE_METADATA_DATASETS);
+    }
+
+    private long getMetadataReservedMemory() {
+        return (getMetadataMemoryComponentNumPages() * (long) getMemoryComponentPageSize())
* getMetadataDatasets();
+    }
 }

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/2362
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I9909c26b1e12b431f913e201d2c3d83769be7269
Gerrit-PatchSet: 6
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <mhubail@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Luo Chen <cluo8@uci.edu>
Gerrit-Reviewer: Michael Blow <mblow@apache.org>
Gerrit-Reviewer: Murtadha Hubail <mhubail@apache.org>
Gerrit-Reviewer: Till Westmann <tillw@apache.org>
Gerrit-Reviewer: abdullah alamoudi <bamousaa@gmail.com>

Mime
View raw message