incubator-blur-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amccu...@apache.org
Subject git commit: Changing csd to register memory used with cloudera manager.
Date Thu, 15 Oct 2015 12:49:56 GMT
Repository: incubator-blur
Updated Branches:
  refs/heads/master 317245f4c -> 8b7eb8f09


Changing csd to register memory used with cloudera manager.


Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/8b7eb8f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/8b7eb8f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/8b7eb8f0

Branch: refs/heads/master
Commit: 8b7eb8f096ef260629f06e890c68f747f5c7b6f6
Parents: 317245f
Author: Aaron McCurry <amccurry@gmail.com>
Authored: Thu Oct 15 08:49:50 2015 -0400
Committer: Aaron McCurry <amccurry@gmail.com>
Committed: Thu Oct 15 08:49:50 2015 -0400

----------------------------------------------------------------------
 .../resources/descriptor/service.sdl.template   | 50 ++++++++++++++++++--
 .../apache/blur/doc/CreateCSDDescriptor.java    | 29 +++++++++++-
 2 files changed, 74 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8b7eb8f0/cdh-csd/src/main/resources/descriptor/service.sdl.template
----------------------------------------------------------------------
diff --git a/cdh-csd/src/main/resources/descriptor/service.sdl.template b/cdh-csd/src/main/resources/descriptor/service.sdl.template
index 2f66d28..17b2068 100644
--- a/cdh-csd/src/main/resources/descriptor/service.sdl.template
+++ b/cdh-csd/src/main/resources/descriptor/service.sdl.template
@@ -72,7 +72,7 @@
             "BLUR_DEFAULT_TABLE_PATH": "/user/${user}/${blur_cluster_name}/tables",
             "BLUR_COMMAND_PATH":"/user/${user}/${blur_cluster_name}/commands",
             "BLUR_CLUSTER_NAME":"${blur_cluster_name}",
-            "BLUR_CONTROLLER_JVM_OPTIONS":"${blur_controller_jvm_options}"
+            "BLUR_CONTROLLER_JVM_OPTIONS":"-Xmx${blur_controller_jvm_heap} -Xms${blur_controller_jvm_heap}
${blur_controller_jvm_options}"
         }
       },
       "externalLink" : {
@@ -102,9 +102,23 @@
           "configName":"blur.controller.jvm.options",
 	      "required":"false",
 	      "type":"string",
-	      "default":"-Xmx1024m -Djava.net.preferIPv4Stack=true",
+	      "default":"-Djava.net.preferIPv4Stack=true",
 	      "configurableInWizard":true
       },
+    {
+      "name" : "blur_controller_jvm_heap",
+      "configName": "blur.controller.jvm.heap",
+      "label" : "Total Java Heap Sizes of Controller in Bytes",
+      "description" : "Memory available to the Controller. Passed to Java -Xmx. Measured
in bytes.",
+      "required" : "true",
+      "type" : "memory",
+      "unit" : "bytes",
+      "min" : 2147483648,
+      "default" : 8589934592,
+      "scaleFactor" : 1.3,
+      "autoConfigShare" : 100
+    }
+      ,
 |||Controller-Server-Properties|||
                  ]
    },
@@ -120,7 +134,7 @@
             "BLUR_DEFAULT_TABLE_PATH": "/user/${user}/${blur_cluster_name}/tables",
             "BLUR_COMMAND_PATH":"/user/${user}/${blur_cluster_name}/commands",
             "BLUR_CLUSTER_NAME":"${blur_cluster_name}",
-            "BLUR_SHARD_JVM_OPTIONS":"${blur_shard_jvm_options}"
+            "BLUR_SHARD_JVM_OPTIONS":"-Xmx${blur_shard_jvm_heap} -Xms${blur_shard_jvm_heap}
${blur_shard_jvm_options}"
           }
 	    },
       "additionalExternalLinks" : [{
@@ -150,9 +164,37 @@
 	          "configName":"blur.shard.jvm.options",
 		      "required":"false",
 		      "type":"string",
-		      "default":"-Xmx1024m -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=256m",
+		      "default":"-Djava.net.preferIPv4Stack=true",
 		      "configurableInWizard":true
 	      	},
+    {
+      "name" : "blur_shard_jvm_heap",
+      "configName": "blur.shard.jvm.heap",
+      "label" : "Total Java Heap Sizes of Shard in Bytes",
+      "description" : "Memory available to the Shard. Passed to Java -Xmx. Measured in bytes.",
+      "required" : "true",
+      "type" : "memory",
+      "unit" : "bytes",
+      "min" : 2147483648,
+      "default" : 8589934592,
+      "scaleFactor" : 1.3,
+      "autoConfigShare" : 100
+    },
+    
+    {
+      "name" : "blur_shard_block_cache_total_size",
+      "configName": "blur.shard.block.cache.total.size",
+      "label" : "Total Block Cache Sizes of Shard in Bytes",
+      "description" : "Block Cache Memory available to the Shard. Measured in bytes.",
+      "required" : "true",
+      "type" : "memory",
+      "unit" : "bytes",
+      "min" : 134217728,
+      "default" : 8589934592,
+      "scaleFactor" : 1.3,
+      "autoConfigShare" : 100
+    },
+	      	
 |||Shard-Server-Properties|||
                  ]
 	}   

http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/8b7eb8f0/cdh-csd/src/test/java/org/apache/blur/doc/CreateCSDDescriptor.java
----------------------------------------------------------------------
diff --git a/cdh-csd/src/test/java/org/apache/blur/doc/CreateCSDDescriptor.java b/cdh-csd/src/test/java/org/apache/blur/doc/CreateCSDDescriptor.java
index eacc7dc..321631d 100644
--- a/cdh-csd/src/test/java/org/apache/blur/doc/CreateCSDDescriptor.java
+++ b/cdh-csd/src/test/java/org/apache/blur/doc/CreateCSDDescriptor.java
@@ -22,19 +22,25 @@ import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.PrintWriter;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 
 import org.apache.blur.doc.BlurPropertyParser.BlurProp;
+import org.apache.commons.io.IOUtils;
+import org.json.JSONException;
+import org.json.JSONObject;
 
 public class CreateCSDDescriptor {
 
   private static final String BLUR_VERSION = "|||BLUR-VERSION|||";
 
-  public static void main(String[] args) throws IOException {
+  public static void main(String[] args) throws IOException, JSONException {
     BlurPropertyParser parser = new BlurPropertyParser();
     Map<String, List<BlurProp>> props = parser.parse();
+    removeProps("blur.shard.block.cache.total.size", props);
     Map<String, StringBuffer> map = new HashMap<String, StringBuffer>();
 
     JsonPropertyFormatter formatter = new JsonPropertyFormatter();
@@ -62,6 +68,27 @@ public class CreateCSDDescriptor {
     String blurVersion = args[2];
 
     replaceValuesInFile(source, dest, map, formatVersion(blurVersion));
+
+    FileInputStream input = new FileInputStream(dest);
+    try {
+      // Check if the csd is a good json file.
+      new JSONObject(IOUtils.toString(input));
+    } finally {
+      input.close();
+    }
+  }
+
+  private static void removeProps(String propertyName, Map<String, List<BlurProp>>
props) {
+    for (Entry<String, List<BlurProp>> e : props.entrySet()) {
+      List<BlurProp> value = e.getValue();
+      List<BlurProp> remove = new ArrayList<BlurProp>();
+      for (BlurProp blurProp : value) {
+        if (propertyName.equals(blurProp.getName())) {
+          remove.add(blurProp);
+        }
+      }
+      value.removeAll(remove);
+    }
   }
 
   public static String formatVersion(String blurVersion) {


Mime
View raw message