bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwmon...@apache.org
Subject bigtop git commit: BIGTOP-2844: zeppelin charm: consistent spark config
Date Thu, 03 Aug 2017 14:51:02 GMT
Repository: bigtop
Updated Branches:
  refs/heads/master c29e841d7 -> c1a210371


BIGTOP-2844: zeppelin charm: consistent spark config

Closes #257


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c1a21037
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c1a21037
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c1a21037

Branch: refs/heads/master
Commit: c1a21037165334716ec2a22debcd011d89f1074d
Parents: c29e841
Author: Kevin W Monroe <kevin.monroe@canonical.com>
Authored: Thu Jul 13 20:27:35 2017 +0000
Committer: Kevin W Monroe <kevin.monroe@canonical.com>
Committed: Thu Aug 3 09:50:31 2017 -0500

----------------------------------------------------------------------
 .../lib/charms/layer/bigtop_zeppelin.py             | 10 ++++++----
 .../layer-zeppelin/tests/03-zeppelin-spark-smoke.py | 16 ++++++++++++++--
 2 files changed, 20 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1a21037/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py
b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py
index 44512d2..812a644 100644
--- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py
+++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/lib/charms/layer/bigtop_zeppelin.py
@@ -111,7 +111,7 @@ class Zeppelin(object):
         overrides = unitdata.kv().getrange('zeppelin.bigtop.overrides.',
                                            strip=True)
 
-        # The zep deb depends on spark-core, spark-python, and unfortunately,
+        # The zep deb depends on spark-core which unfortunately brings in
         # most of hadoop. Include appropriate roles here to ensure these
         # packages are configured in the same way as our other Bigtop
         # software deployed with puppet.
@@ -156,9 +156,10 @@ class Zeppelin(object):
         utils.run_as('hdfs', 'hdfs', 'dfs', '-mkdir', '-p', '/user/zeppelin')
         utils.run_as('hdfs', 'hdfs', 'dfs', '-chown', 'zeppelin', '/user/zeppelin')
 
-        # If spark is ready, let it handle the spark_master_url. Otherwise,
-        # zepp is in local mode; set it to yarn-client since hadoop is here.
+        # If spark is ready, let configure_spark() trigger bigtop. Otherwise,
+        # put our spark in yarn-client mode since hadoop is here.
         if not is_state('spark.ready'):
+            self._add_override('spark::common::master_url', 'yarn-client')
             self._add_override('zeppelin::server::spark_master_url', 'yarn-client')
             self.trigger_bigtop()
 
@@ -166,7 +167,8 @@ class Zeppelin(object):
         '''
         Configure the zeppelin spark interpreter
         '''
-        # TODO: Need Puppet params created to set Spark driver and executor memory
+        # TODO: Add config for Spark driver and executor memory overrides
+        self._add_override('spark::common::master_url', master_url)
         self._add_override('zeppelin::server::spark_master_url', master_url)
         self.trigger_bigtop()
 

http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1a21037/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py
b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py
index daddc51..56f5f96 100755
--- a/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py
+++ b/bigtop-packages/src/charm/zeppelin/layer-zeppelin/tests/03-zeppelin-spark-smoke.py
@@ -28,14 +28,26 @@ class TestDeploy(unittest.TestCase):
     def setUpClass(cls):
         cls.d = amulet.Deployment(series='xenial')
         cls.d.add('zeppelin')
-        cls.d.add('spark')
+        cls.d.add('zeppelin-test-spark', charm='spark')
 
-        cls.d.relate('zeppelin:spark', 'spark:client')
+        cls.d.relate('zeppelin-test-spark:client', 'zeppelin:spark')
 
         cls.d.setup(timeout=1800)
         cls.d.sentry.wait_for_messages({'zeppelin': re.compile('ready with')}, timeout=1800)
         cls.zeppelin = cls.d.sentry['zeppelin'][0]
 
+    @classmethod
+    def tearDownClass(cls):
+        # NB: seems to be a remove_service issue with amulet. However, the
+        # unit does still get removed. Pass OSError for now:
+        #  OSError: juju command failed ['remove-application', ...]:
+        #  ERROR allocation for service ... owned by ... not found
+        try:
+            cls.d.remove_service('zeppelin-test-spark')
+        except OSError as e:
+            print("IGNORE: Amulet remove_service failed: {}".format(e))
+            pass
+
     def test_zeppelin(self):
         """
         Validate Zeppelin by running the smoke-test action.


Mime
View raw message