ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From oleew...@apache.org
Subject [25/50] [abbrv] ambari git commit: AMBARI-18279. Use the PropertiesFile python command to create property files instead of jinja templates (Miklos Gergely via oleewere)
Date Wed, 07 Sep 2016 23:38:15 GMT
AMBARI-18279. Use the PropertiesFile python command to create property files instead of jinja
templates (Miklos Gergely via oleewere)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3f790c8f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3f790c8f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3f790c8f

Branch: refs/heads/branch-dev-logsearch
Commit: 3f790c8f2f384a8a4cd4629f5da62f814fcaf65d
Parents: e9e834b
Author: Miklos Gergely <mgergely@hortonworks.com>
Authored: Wed Sep 7 22:17:05 2016 +0200
Committer: oleewere <oleewere@gmail.com>
Committed: Thu Sep 8 01:34:00 2016 +0200

----------------------------------------------------------------------
 .../configuration/logsearch-properties.xml      |   6 +-
 .../LOGSEARCH/0.5.0/package/scripts/params.py   | 125 +++++++++----------
 .../0.5.0/package/scripts/setup_logfeeder.py    |  10 +-
 .../0.5.0/package/scripts/setup_logsearch.py    |   7 +-
 .../package/templates/logfeeder.properties.j2   |  32 -----
 .../package/templates/logsearch.properties.j2   |  72 -----------
 .../stacks/2.4/LOGSEARCH/test_logfeeder.py      |  18 ++-
 .../stacks/2.4/LOGSEARCH/test_logsearch.py      |  30 ++++-
 .../test/python/stacks/2.4/configs/default.json |  13 +-
 ambari-web/app/data/HDP2/site_properties.js     |   6 +-
 10 files changed, 120 insertions(+), 199 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/3f790c8f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
index 56ff2d1..e521c8e 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
@@ -133,7 +133,7 @@
     <on-ambari-upgrade add="true"/>
   </property>
   <property>
-    <name>logsearch.external.auth.enabled</name>
+    <name>logsearch.auth.external_auth.enable</name>
     <value>true</value>
     <display-name>External authentication</display-name>
     <description>Enable external authentication</description>
@@ -143,14 +143,14 @@
     <on-ambari-upgrade add="true"/>
   </property>
   <property>
-    <name>logsearch.external.auth.host_url</name>
+    <name>logsearch.auth.external_auth.host_url</name>
     <value>{ambari_server_auth_host_url}</value>
     <display-name>External authentication url - host and port</display-name>
     <description>The host and port of the external server used to authenticate</description>
     <on-ambari-upgrade add="true"/>
   </property>
   <property>
-    <name>logsearch.external.auth.login_url</name>
+    <name>logsearch.auth.external_auth.login_url</name>
     <value>/api/v1/users/$USERNAME/privileges?fields=*</value>
     <display-name>External authentication url - path and query</display-name>
     <description>The path and query of the external server used to authenticate</description>

http://git-wip-us.apache.org/repos/asf/ambari/blob/3f790c8f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index 5c3954f..8b0ce75 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -75,9 +75,9 @@ if 'metrics_collector_hosts' in config['clusterHostInfo']:
 else:
   metrics_collector_hosts = ''
 
-logsearch_solr_metrics_collector_hosts = format(config['configurations']['logsearch-properties']['logsearch.solr.metrics.collector.hosts'])
-
+#####################################
 # Infra Solr configs
+#####################################
 infra_solr_znode = default('/configurations/infra-solr-env/infra_solr_znode', '/infra-solr')
 infra_solr_instance_count = len(config['clusterHostInfo']['infra_solr_hosts'])
 infra_solr_ssl_enabled = default('configurations/infra-solr-env/infra_solr_ssl_enabled',
False)
@@ -108,57 +108,17 @@ if security_enabled:
 #####################################
 logsearch_dir = '/usr/lib/ambari-logsearch-portal'
 
-logsearch_collection_service_logs_numshards_config = config['configurations']['logsearch-properties']['logsearch.collection.service.logs.numshards']
-logsearch_collection_audit_logs_numshards_config = config['configurations']['logsearch-properties']['logsearch.collection.audit.logs.numshards']
-
-if logsearch_collection_service_logs_numshards_config > 0:
-  logsearch_collection_service_logs_numshards = str(logsearch_collection_service_logs_numshards_config)
-else:
-  logsearch_collection_service_logs_numshards = format(str(infra_solr_instance_count))
-
-if logsearch_collection_audit_logs_numshards_config > 0:
-  logsearch_collection_audit_logs_numshards = str(logsearch_collection_audit_logs_numshards_config)
-else:
-  logsearch_collection_audit_logs_numshards = format(str(infra_solr_instance_count))
-
-logsearch_collection_service_logs_replication_factor = str(config['configurations']['logsearch-properties']['logsearch.collection.service.logs.replication.factor'])
-logsearch_collection_audit_logs_replication_factor = str(config['configurations']['logsearch-properties']['logsearch.collection.audit.logs.replication.factor'])
-
-logsearch_solr_collection_service_logs = default('/configurations/logsearch-properties/logsearch.solr.collection.service.logs',
'hadoop_logs')
-logsearch_solr_collection_audit_logs = default('/configurations/logsearch-properties/logsearch.solr.collection.audit.logs','audit_logs')
-
 logsearch_service_logs_max_retention = config['configurations']['logsearch-service_logs-solrconfig']['logsearch_service_logs_max_retention']
 logsearch_service_logs_merge_factor = config['configurations']['logsearch-service_logs-solrconfig']['logsearch_service_logs_merge_factor']
-logsearch_service_logs_fields = config['configurations']['logsearch-properties']['logsearch.service.logs.fields']
-logsearch_service_logs_split_interval_mins = config['configurations']['logsearch-properties']['logsearch.service.logs.split.interval.mins']
 
 logsearch_audit_logs_max_retention = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_max_retention']
 logsearch_audit_logs_merge_factor = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_merge_factor']
-logsearch_audit_logs_split_interval_mins = config['configurations']['logsearch-properties']['logsearch.audit.logs.split.interval.mins']
 
-logsearch_logfeeder_include_default_level = default('/configurations/logsearch-properties/logsearch.logfeeder.include.default.level',
'fatal,error,warn')
-
-logsearch_solr_audit_logs_zk_node = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_node',
zookeeper_quorum)
-logsearch_solr_audit_logs_zk_quorum = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_quorum',
infra_solr_znode)
+logsearch_solr_audit_logs_zk_node = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_node',
infra_solr_znode)
+logsearch_solr_audit_logs_zk_quorum = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_quorum',
zookeeper_quorum)
 logsearch_solr_audit_logs_zk_node = format(logsearch_solr_audit_logs_zk_node)
 logsearch_solr_audit_logs_zk_quorum = format(logsearch_solr_audit_logs_zk_quorum)
 
-# create custom properties - remove defaults
-logsearch_custom_properties = dict(config['configurations']['logsearch-properties'])
-logsearch_custom_properties.pop("logsearch.service.logs.fields", None)
-logsearch_custom_properties.pop("logsearch.audit.logs.split.interval.mins", None)
-logsearch_custom_properties.pop("logsearch.collection.service.logs.replication.factor", None)
-logsearch_custom_properties.pop("logsearch.solr.collection.service.logs", None)
-logsearch_custom_properties.pop("logsearch.solr.metrics.collector.hosts", None)
-logsearch_custom_properties.pop("logsearch.solr.collection.audit.logs", None)
-logsearch_custom_properties.pop("logsearch.logfeeder.include.default.level", None)
-logsearch_custom_properties.pop("logsearch.collection.audit.logs.replication.factor", None)
-logsearch_custom_properties.pop("logsearch.collection.service.logs.numshards", None)
-logsearch_custom_properties.pop("logsearch.service.logs.split.interval.mins", None)
-logsearch_custom_properties.pop("logsearch.collection.audit.logs.numshards", None)
-logsearch_custom_properties.pop("logsearch.external.auth.enabled", None)
-logsearch_custom_properties.pop("logsearch.external.auth.host_url", None)
-logsearch_custom_properties.pop("logsearch.external.auth.login_url", None)
 
 # logsearch-env configs
 logsearch_user = config['configurations']['logsearch-env']['logsearch_user']
@@ -215,9 +175,7 @@ hdfs_user = default('configurations/hadoop-env/hdfs_user', 'hdfs')
 mapred_user =  default('configurations/mapred-env/mapred_user', 'mapred')
 yarn_user =  default('configurations/yarn-env/yarn_user', 'yarn')
 
-#####################################
 # Logsearch auth configs
-#####################################
 
 logsearch_admin_credential_file = 'logsearch-admin.json'
 logsearch_admin_username = default('/configurations/logsearch-admin-json/logsearch_admin_username',
"admin")
@@ -236,9 +194,42 @@ if 'ambari_server_host' in config['clusterHostInfo']:
 else:
   ambari_server_auth_host_url = ''
 
-logsearch_auth_external_enabled = str(config['configurations']['logsearch-properties']['logsearch.external.auth.enabled']).lower()
-logsearch_auth_external_host_url = format(config['configurations']['logsearch-properties']['logsearch.external.auth.host_url'])
-logsearch_auth_external_login_url = config['configurations']['logsearch-properties']['logsearch.external.auth.login_url']
+# Logsearch propreties
+
+logsearch_properties = dict(config['configurations']['logsearch-properties'])
+
+logsearch_properties['logsearch.solr.zk_connect_string'] = zookeeper_quorum + infra_solr_znode
+logsearch_properties['logsearch.solr.audit.logs.zk_connect_string'] = logsearch_solr_audit_logs_zk_quorum
+ logsearch_solr_audit_logs_zk_node
+
+logsearch_properties['logsearch.solr.collection.history'] = 'history'
+logsearch_properties['logsearch.solr.history.config.name'] = 'history'
+logsearch_properties['logsearch.collection.history.replication.factor'] = '1'
+
+logsearch_properties['logsearch.solr.metrics.collector.hosts'] = format(logsearch_properties['logsearch.solr.metrics.collector.hosts'])
+
+logsearch_properties['logsearch.solr.jmx.port'] = infra_solr_jmx_port
+
+logsearch_properties['logsearch.login.credentials.file'] = logsearch_admin_credential_file
+logsearch_properties['logsearch.auth.file.enable'] = 'true'
+logsearch_properties['logsearch.auth.ldap.enable'] = 'false'
+logsearch_properties['logsearch.auth.simple.enable'] = 'false'
+logsearch_properties['logsearch.roles.allowed'] = 'AMBARI.ADMINISTRATOR'
+logsearch_properties['logsearch.auth.external_auth.host_url'] = format(logsearch_properties['logsearch.auth.external_auth.host_url'])
+
+logsearch_properties['logsearch.protocol'] = logsearch_ui_protocol
+
+if security_enabled:
+  logfeeder_properties['logsearch.solr.kerberos.enable'] = 'true'
+  logfeeder_properties['logsearch.solr.jaas.file'] = logsearch_jaas_file
+
+
+logsearch_solr_collection_service_logs = logsearch_properties['logsearch.solr.collection.service.logs']
+logsearch_service_logs_split_interval_mins = logsearch_properties['logsearch.service.logs.split.interval.mins']
+logsearch_collection_service_logs_numshards = logsearch_properties['logsearch.collection.service.logs.numshards']
+
+logsearch_solr_collection_audit_logs = logsearch_properties['logsearch.solr.collection.audit.logs']
+logsearch_audit_logs_split_interval_mins = logsearch_properties['logsearch.audit.logs.split.interval.mins']
+logsearch_collection_audit_logs_numshards = logsearch_properties['logsearch.collection.audit.logs.numshards']
 
 #####################################
 # Logfeeder configs
@@ -268,29 +259,31 @@ logfeeder_truststore_location = config['configurations']['logfeeder-env']['logfe
 logfeeder_truststore_password = config['configurations']['logfeeder-env']['logfeeder_truststore_password']
 logfeeder_truststore_type = config['configurations']['logfeeder-env']['logfeeder_truststore_type']
 
-logfeeder_checkpoint_folder = default('/configurations/logfeeder-env/logfeeder.checkpoint.folder',
-                                      '/etc/ambari-logsearch-logfeeder/conf/checkpoints')
+logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume',
'hbase', 'hdfs', 'hive', 'hst', 'infra',
+                                'kafka', 'knox', 'logsearch', 'nifi', 'oozie', 'ranger',
'spark', 'spark2', 'storm', 'yarn',
+                                'zeppelin', 'zookeeper']
 
-logfeeder_log_filter_enable = str(default('/configurations/logfeeder-properties/logfeeder.log.filter.enable',
True)).lower()
-logfeeder_solr_config_interval = default('/configurations/logfeeder-properties/logfeeder.solr.config.interval',
5)
+logfeeder_config_file_names = \
+  ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag
in logfeeder_supported_services]
 
-logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume',
'hbase', 'hdfs', 'hive', 'hst', 'infra', 'kafka',
-                                'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark',
'spark2', 'storm', 'yarn', 'zeppelin', 'zookeeper']
+default_config_files = ','.join(logfeeder_config_file_names)
 
-logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json'
% (tag) for tag in
-                                                                              logfeeder_supported_services]
+logfeeder_properties = dict(config['configurations']['logfeeder-properties'])
 
-default_config_files = ','.join(logfeeder_config_file_names)
+logfeeder_properties['logfeeder.metrics.collector.hosts'] = format(logfeeder_properties['logfeeder.metrics.collector.hosts'])
+logfeeder_properties['logfeeder.config.files'] = format(logfeeder_properties['logfeeder.config.files'])
+logfeeder_properties['logfeeder.solr.core.config.name'] = 'history'
+logfeeder_properties['logfeeder.solr.zk_connect_string'] = zookeeper_quorum + infra_solr_znode
 
-logfeeder_config_files = format(config['configurations']['logfeeder-properties']['logfeeder.config.files'])
-logfeeder_metrics_collector_hosts = format(config['configurations']['logfeeder-properties']['logfeeder.metrics.collector.hosts'])
+if security_enabled:
+  logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true'
+  logfeeder_properties['logfeeder.solr.jaas.file'] = logfeeder_jaas_file
 
-logfeeder_custom_properties = dict(config['configurations']['logfeeder-properties'])
-logfeeder_custom_properties.pop('logfeeder.config.files', None)
-logfeeder_custom_properties.pop('logfeeder.checkpoint.folder', None)
-logfeeder_custom_properties.pop('logfeeder.metrics.collector.hosts', None)
-logfeeder_custom_properties.pop('logfeeder.log.filter.enable', None)
-logfeeder_custom_properties.pop('logfeeder.solr.config.interval', None)
+logfeeder_checkpoint_folder = logfeeder_properties['logfeeder.checkpoint.folder']
+
+#####################################
+# Smoke command
+#####################################
 
 logsearch_server_hosts = config['clusterHostInfo']['logsearch_server_hosts']
 logsearch_server_host = ""

http://git-wip-us.apache.org/repos/asf/ambari/blob/3f790c8f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
index d3851d3..688a9b0 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
@@ -20,12 +20,12 @@ limitations under the License.
 from resource_management.core.resources.system import Directory, File
 from resource_management.libraries.functions.format import format
 from resource_management.core.source import InlineTemplate, Template
+from resource_management.libraries.resources.properties_file import PropertiesFile
 
 def setup_logfeeder():
   import params
 
-  Directory([params.logfeeder_log_dir, params.logfeeder_pid_dir,
-             params.logfeeder_checkpoint_folder],
+  Directory([params.logfeeder_log_dir, params.logfeeder_pid_dir, params.logfeeder_checkpoint_folder],
             mode=0755,
             cd_access='a',
             create_parents=True
@@ -43,9 +43,9 @@ def setup_logfeeder():
        content=''
        )
 
-  File(format("{logsearch_logfeeder_conf}/logfeeder.properties"),
-       content=Template("logfeeder.properties.j2"),
-       )
+  PropertiesFile(format("{logsearch_logfeeder_conf}/logfeeder.properties"),
+                 properties = params.logfeeder_properties
+                 )
 
   File(format("{logsearch_logfeeder_conf}/logfeeder-env.sh"),
        content=InlineTemplate(params.logfeeder_env_content),

http://git-wip-us.apache.org/repos/asf/ambari/blob/3f790c8f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
index aa89609..8d84093 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
@@ -23,6 +23,7 @@ from resource_management.core.resources.system import Directory, Execute,
File
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.decorator import retry
 from resource_management.core.source import InlineTemplate, Template
+from resource_management.libraries.resources.properties_file import PropertiesFile
 
 
 def setup_logsearch():
@@ -51,10 +52,8 @@ def setup_logsearch():
        content=''
        )
 
-  File(format("{logsearch_server_conf}/logsearch.properties"),
-       content=Template("logsearch.properties.j2"),
-       owner=params.logsearch_user,
-       group=params.user_group
+  PropertiesFile(format("{logsearch_server_conf}/logsearch.properties"),
+       properties=params.logsearch_properties
        )
 
   File(format("{logsearch_server_conf}/HadoopServiceConfig.json"),

http://git-wip-us.apache.org/repos/asf/ambari/blob/3f790c8f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2
deleted file mode 100644
index 86f0be4..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-logfeeder.checkpoint.folder={{logfeeder_checkpoint_folder}}
-logfeeder.metrics.collector.hosts={{logfeeder_metrics_collector_hosts}}
-logfeeder.config.files={{logfeeder_config_files}}
-logfeeder.log.filter.enable={{logfeeder_log_filter_enable}}
-logfeeder.solr.config.interval={{logfeeder_solr_config_interval}}
-logfeeder.solr.core.config.name=history
-logfeeder.solr.zk_connect_string={{zookeeper_quorum}}{{infra_solr_znode}}
-
-# Custom properties
-{% for key, value in logfeeder_custom_properties.items() %}
-{{key}}={{value}}
-{% endfor %}
-
-{% if security_enabled -%}
-logfeeder.solr.kerberos.enable=true
-logfeeder.solr.jaas.file={{logfeeder_jaas_file}}
-{% endif %}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/3f790c8f/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2
deleted file mode 100644
index 1bcee2d..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-logsearch.solr.zk_connect_string={{zookeeper_quorum}}{{infra_solr_znode}}
-
-# Service Logs
-logsearch.solr.collection.service.logs={{logsearch_solr_collection_service_logs}}
-
-logsearch.service.logs.split.interval.mins={{logsearch_service_logs_split_interval_mins}}
-logsearch.collection.service.logs.numshards={{logsearch_collection_service_logs_numshards}}
-logsearch.collection.service.logs.replication.factor={{logsearch_collection_service_logs_replication_factor}}
-
-logsearch.service.logs.fields={{logsearch_service_logs_fields}}
-
-# Audit logs
-logsearch.solr.audit.logs.zk_connect_string={{logsearch_solr_audit_logs_zk_quorum}}{{logsearch_solr_audit_logs_zk_node}}
-logsearch.solr.collection.audit.logs={{logsearch_solr_collection_audit_logs}}
-logsearch.solr.audit.logs.url=
-
-logsearch.audit.logs.split.interval.mins={{logsearch_audit_logs_split_interval_mins}}
-logsearch.collection.audit.logs.numshards={{logsearch_collection_audit_logs_numshards}}
-logsearch.collection.audit.logs.replication.factor={{logsearch_collection_audit_logs_replication_factor}}
-
-# History logs
-logsearch.solr.collection.history=history
-logsearch.solr.history.config.name=history
-logsearch.collection.history.replication.factor=1
-
-# Metrics
-logsearch.solr.metrics.collector.hosts={{logsearch_solr_metrics_collector_hosts}}
-logsearch.solr.jmx.port={{infra_solr_jmx_port}}
-
-# Logfeeder Settings
-
-logsearch.logfeeder.include.default.level={{logsearch_logfeeder_include_default_level}}
-
-# logsearch-admin.json
-logsearch.auth.file.enable=true
-logsearch.login.credentials.file={{logsearch_admin_credential_file}}
-
-logsearch.auth.ldap.enable=false
-logsearch.auth.simple.enable=false
-
-logsearch.auth.external_auth.enable={{logsearch_auth_external_enabled}}
-logsearch.auth.external_auth.host_url={{logsearch_auth_external_host_url}}
-logsearch.auth.external_auth.login_url={{logsearch_auth_external_login_url}}
-logsearch.roles.allowed=AMBARI.ADMINISTRATOR
-
-logsearch.protocol={{logsearch_ui_protocol}}
-
-# Custom properties
-{% for key, value in logsearch_custom_properties.items() %}
-{{key}}={{value}}
-{% endfor %}
-
-{% if security_enabled -%}
-logsearch.solr.kerberos.enable=true
-logsearch.solr.jaas.file={{logsearch_jaas_file}}
-{% endif %}
-

http://git-wip-us.apache.org/repos/asf/ambari/blob/3f790c8f/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
index f4dbd63..7f9acba 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -60,8 +60,13 @@ class TestLogFeeder(RMFTestCase):
                               mode=0644,
                               content=''
                               )
-    self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties',
-                              content=Template('logfeeder.properties.j2')
+    self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties',
+                              properties={'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints',
+                                          'logfeeder.config.files': 'global.config.json,output.config.json,input.config-accumulo.json,input.config-ambari.json,input.config-ams.json,input.config-atlas.json,input.config-falcon.json,input.config-flume.json,input.config-hbase.json,input.config-hdfs.json,input.config-hive.json,input.config-hst.json,input.config-infra.json,input.config-kafka.json,input.config-knox.json,input.config-logsearch.json,input.config-nifi.json,input.config-oozie.json,input.config-ranger.json,input.config-spark.json,input.config-spark2.json,input.config-storm.json,input.config-yarn.json,input.config-zeppelin.json,input.config-zookeeper.json',
+                                          'logfeeder.metrics.collector.hosts': '',
+                                          'logfeeder.solr.core.config.name': 'history',
+                                          'logfeeder.solr.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr'
+                                         }
                               )
     self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.sh',
                               mode=0755,
@@ -75,11 +80,12 @@ class TestLogFeeder(RMFTestCase):
                               encoding='utf-8'
                               )
 
-    logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume',
'hbase', 'hdfs', 'hive', 'hst', 'infra', 'kafka',
-                                    'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark',
'spark2', 'storm', 'yarn', 'zeppelin', 'zookeeper']
+    logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume',
'hbase', 'hdfs', 'hive', 'hst',
+                                    'infra', 'kafka', 'knox', 'logsearch', 'nifi', 'oozie',
'ranger', 'spark', 'spark2', 'storm',
+                                    'yarn', 'zeppelin', 'zookeeper']
 
-    logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json'
% (tag) for tag
-                                                                                  in logfeeder_supported_services]
+    logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + \
+                                  ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services]
 
     for file_name in logfeeder_config_file_names:
       self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/' + file_name,

http://git-wip-us.apache.org/repos/asf/ambari/blob/3f790c8f/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
index 5a2e6e8..0378a94 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
@@ -70,10 +70,32 @@ class TestLogSearch(RMFTestCase):
                               mode = 0644,
                               content = ''
     )
-    self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/logsearch.properties',
-                              owner = 'logsearch',
-                              group='hadoop',
-                              content = Template('logsearch.properties.j2')
+    self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-portal/conf/logsearch.properties',
+                              properties = {'logsearch.audit.logs.split.interval.mins': '1',
+                                            'logsearch.auth.external_auth.enabled': 'false',
+                                            'logsearch.auth.external_auth.host_url': 'http://c6401.ambari.apache.org:8080',
+                                            'logsearch.auth.external_auth.login_url': '/api/v1/users/$USERNAME/privileges?fields=*',
+                                            'logsearch.auth.file.enable': 'true',
+                                            'logsearch.auth.ldap.enable': 'false',
+                                            'logsearch.auth.simple.enable': 'false',
+                                            'logsearch.collection.audit.logs.numshards':
'10',
+                                            'logsearch.collection.audit.logs.replication.factor':
'1',
+                                            'logsearch.collection.history.replication.factor':
'1',
+                                            'logsearch.collection.service.logs.numshards':
'10',
+                                            'logsearch.collection.service.logs.replication.factor':
'1',
+                                            'logsearch.login.credentials.file': 'logsearch-admin.json',
+                                            'logsearch.protocol': 'http',
+                                            'logsearch.roles.allowed': 'AMBARI.ADMINISTRATOR',
+                                            'logsearch.service.logs.split.interval.mins':
'1',
+                                            'logsearch.solr.audit.logs.zk_connect_string':
'c6401.ambari.apache.org:2181/infra-solr',
+                                            'logsearch.solr.collection.audit.logs': 'audit_logs',
+                                            'logsearch.solr.collection.history': 'history',
+                                            'logsearch.solr.collection.service.logs': 'hadoop_logs',
+                                            'logsearch.solr.history.config.name': 'history',
+                                            'logsearch.solr.metrics.collector.hosts': '',
+                                            'logsearch.solr.jmx.port': '1',
+                                            'logsearch.solr.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr'
+                              }
     )
     self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/HadoopServiceConfig.json',
                               owner = 'logsearch',

http://git-wip-us.apache.org/repos/asf/ambari/blob/3f790c8f/ambari-server/src/test/python/stacks/2.4/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json
index a016ce0..8ac6fa7 100644
--- a/ambari-server/src/test/python/stacks/2.4/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json
@@ -250,6 +250,7 @@
         "infra_solr_datadir": "/opt/ambari_infra_solr/data",
         "infra_solr_log_dir": "/var/log/ambari-infra-solr",
         "infra_solr_client_log_dir": "/var/log/ambari-infra-solr-client",
+        "infra_solr_jmx_port" : "1",
         "content": "# By default the script will use JAVA_HOME to determine which java\n#
to use, but you can set a specific path for Solr to use without\n# affecting other Java applications
on your server/workstation.\nSOLR_JAVA_HOME={{java64_home}}\n\n# Increase Java Min/Max Heap
as needed to support your indexing / query needs\nSOLR_JAVA_MEM=\"-Xms{{solr_min_mem}} -Xmx{{solr_max_mem}}\"\n\n#
Enable verbose GC logging\nGC_LOG_OPTS=\"-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails
\\\n-XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime\"\n\n#
These GC settings have shown to work well for a number of common Solr workloads\nGC_TUNE=\"-XX:NewRatio=3
\\\n-XX:SurvivorRatio=4 \\\n-XX:TargetSurvivorRatio=90 \\\n-XX:MaxTenuringThreshold=8 \\\n-XX:+UseConcMarkSweepGC
\\\n-XX:+UseParNewGC \\\n-XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \\\n-XX:+CMSScavengeBeforeRemark
\\\n-XX:PretenureSizeThreshold=64m \\\n-XX:+UseCMSInitiatingOc
 cupancyOnly \\\n-XX:CMSInitiatingOccupancyFraction=50 \\\n-XX:CMSMaxAbortablePrecleanTime=6000
\\\n-XX:+CMSParallelRemarkEnabled \\\n-XX:+ParallelRefProcEnabled\"\n\n# Set the ZooKeeper
connection string if using an external ZooKeeper ensemble\n# e.g. host1:2181,host2:2181/chroot\n#
Leave empty if not using SolrCloud\nZK_HOST=\"{{zookeeper_quorum}}{{solr_znode}}\"\n\n# Set
the ZooKeeper client timeout (for SolrCloud mode)\nZK_CLIENT_TIMEOUT=\"60000\"\n\n# By default
the start script uses \"localhost\"; override the hostname here\n# for production SolrCloud
environments to control the hostname exposed to cluster state\n#SOLR_HOST=\"192.168.1.1\"\n\n#
By default the start script uses UTC; override the timezone if needed\n#SOLR_TIMEZONE=\"UTC\"\n\n#
Set to true to activate the JMX RMI connector to allow remote JMX client applications\n# to
monitor the JVM hosting Solr; set to \"false\" to disable that behavior\n# (false is recommended
in production environments)\nENABLE_REMOTE_JMX_OPTS
 =\"false\"\n\n# The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here\n#
RMI_PORT=18983\n\n# Anything you add to the SOLR_OPTS variable will be included in the java\n#
start command line as-is, in ADDITION to other options. If you specify the\n# -a option on
start script, those options will be appended as well. Examples:\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000\"\n#SOLR_OPTS=\"$SOLR_OPTS
-Dsolr.autoCommit.maxTime=60000\"\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.clustering.enabled=true\"\n\n#
Location where the bin/solr script will save PID files for running instances\n# If not set,
the script will create PID files in $SOLR_TIP/bin\nSOLR_PID_DIR={{solr_piddir}}\n\n# Path
to a directory where Solr creates index files, the specified directory\n# must contain a solr.xml;
by default, Solr will use server/solr\nSOLR_HOME={{infra_solr_datadir}}\n\n# Solr provides
a default Log4J configuration properties file in server/resources\n# however, you may want
to cu
 stomize the log settings and file appender location\n# so you can point the script to use
a different log4j.properties file\nLOG4J_PROPS={{logsearch_solr_datadir}}/resources/log4j.properties\n\n#
Location where Solr should write logs to; should agree with the file appender\n# settings
in server/resources/log4j.properties\nSOLR_LOGS_DIR={{solr_log_dir}}\n\n# Sets the port Solr
binds to, default is 8983\nSOLR_PORT={{solr_port}}\n\n# Be sure to update the paths to the
correct keystore for your environment\n{% if logsearch_solr_ssl_enabled %}\nSOLR_SSL_KEY_STORE={{logsearch_solr_keystore_location}}\nSOLR_SSL_KEY_STORE_PASSWORD={{logsearch_solr_keystore_password}}\nSOLR_SSL_TRUST_STORE={{logsearch_solr_keystore_location}}\nSOLR_SSL_TRUST_STORE_PASSWORD={{logsearch_solr_keystore_password}}\nSOLR_SSL_NEED_CLIENT_AUTH=false\nSOLR_SSL_WANT_CLIENT_AUTH=false\n{%
endif %}\n\n# Uncomment to set a specific SSL port (-Djetty.ssl.port=N); if not set\n# and
you are using SSL, then the start script 
 will use SOLR_PORT for the SSL port\n#SOLR_SSL_PORT="
       },
       "infra-solr-xml": {
@@ -280,17 +281,21 @@
         "content": "&lt;?xml version=\"1.0\" encoding=\"UTF-8\" ?&gt;\n&lt;!--\n
 Licensed to the Apache Software Foundation (ASF) under one or more\n  contributor license
agreements.  See the NOTICE file distributed with\n  this work for additional information
regarding copyright ownership.\n  The ASF licenses this file to You under the Apache License,
Version 2.0\n  (the \"License\"); you may not use this file except in compliance with\n  the
License.  You may obtain a copy of the License at\n\n      http://www.apache.org/licenses/LICENSE-2.0\n\n
 Unless required by applicable law or agreed to in writing, software\n  distributed under
the License is distributed on an \"AS IS\" BASIS,\n  WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.\n  See the License for the specific language governing permissions
and\n  limitations under the License.\n--&gt;\n&lt;!DOCTYPE log4j:configuration SYSTEM
\"log4j.dtd\"&gt;\n&lt;log4j:configuration xmlns:log4j=\"http://jakarta.
 apache.org/log4j/\"&gt;\n  &lt;appender name=\"console\" class=\"org.apache.log4j.ConsoleAppender\"&gt;\n
   &lt;param name=\"Target\" value=\"System.out\" /&gt;\n    &lt;layout class=\"org.apache.log4j.PatternLayout\"&gt;\n
     &lt;param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\"
/&gt;\n    &lt;/layout&gt;\n  &lt;/appender&gt;\n\n  &lt;appender
name=\"rolling_file\" class=\"org.apache.log4j.RollingFileAppender\"&gt; \n    &lt;param
name=\"file\" value=\"{{logsearch_log_dir}}/logsearch.log\" /&gt; \n    &lt;param
name=\"append\" value=\"true\" /&gt; \n    &lt;param name=\"maxFileSize\" value=\"10MB\"
/&gt; \n    &lt;param name=\"maxBackupIndex\" value=\"10\" /&gt; \n    &lt;layout
class=\"org.apache.log4j.PatternLayout\"&gt; \n      &lt;param name=\"ConversionPattern\"
value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\"/&gt; \n    &lt;/layout&gt; \n  &lt;/appender&gt;
\n\n  &lt;appender name=\"performance_analyzer\" class=\"org.apache.log4j.RollingFileAppender\"
 &gt;\n    &lt;param name=\"file\" value=\"{{logsearch_log_dir}}/logsearch-performance.log\"
/&gt;\n    &lt;param name=\"Threshold\" value=\"info\" /&gt;\n    &lt;param
name=\"append\" value=\"true\" /&gt;\n    &lt;param name=\"maxFileSize\" value=\"10MB\"
/&gt; \n    &lt;param name=\"maxBackupIndex\" value=\"10\" /&gt; \n    &lt;layout
class=\"org.apache.log4j.PatternLayout\"&gt;\n      &lt;param name=\"ConversionPattern\"
value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\" /&gt;\n    &lt;/layout&gt;\n  &lt;/appender&gt;\n
 \n  &lt;logger name=\"org.apache.ambari.logsearch.perfomance\" additivity=\"false\"&gt;\n
  &lt;appender-ref ref=\"performance_analyzer\" /&gt;\n  &lt;/logger&gt;\n\n
 &lt;category name=\"org.apache.ambari.logsearch\" additivity=\"false\"&gt;\n    &lt;priority
value=\"info\" /&gt;\n    &lt;appender-ref ref=\"rolling_file\" /&gt;\n  &lt;/category&gt;\n\n
 &lt;root&gt;\n    &lt;priority value=\"warn\" /&gt;\n    &lt;appender-ref
ref=\"rolling_file\" /&gt;\n  &lt;/root&gt;\
 n&lt;/log4j:configuration&gt;"
       },
       "logsearch-properties": {
+        "logsearch.solr.collection.service.logs" : "hadoop_logs",
+        "logsearch.solr.collection.audit.logs" : "audit_logs",
+        "logsearch.service.logs.split.interval.mins": "1",
+        "logsearch.audit.logs.split.interval.mins" : "1",
         "logsearch.collection.service.logs.numshards": "10",
         "logsearch.collection.service.logs.replication.factor": "1",
         "logsearch.collection.audit.logs.numshards": "10",
         "logsearch.collection.audit.logs.replication.factor": "1",
-        "logsearch.app.max.memory": "1g",
         "logsearch.solr.metrics.collector.hosts" : "{metrics_collector_hosts}",
-        "logsearch.external.auth.enabled" : "false",
-        "logsearch.external.auth.host_url" : "{ambari_server_auth_host_url}",
-        "logsearch.external.auth.login_url" : "/api/v1/users/$USERNAME/privileges?fields=*"
+        "logsearch.auth.external_auth.enabled" : "false",
+        "logsearch.auth.external_auth.host_url" : "{ambari_server_auth_host_url}",
+        "logsearch.auth.external_auth.login_url" : "/api/v1/users/$USERNAME/privileges?fields=*"
       },
       "logfeeder-properties": {
+        "logfeeder.checkpoint.folder" : "/etc/ambari-logsearch-logfeeder/conf/checkpoints",
         "logfeeder.metrics.collector.hosts" : "{metrics_collector_hosts}",
         "logfeeder.config.files" : "{default_config_files}"
       },

http://git-wip-us.apache.org/repos/asf/ambari/blob/3f790c8f/ambari-web/app/data/HDP2/site_properties.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js
index 1fad1d6..9311e75 100644
--- a/ambari-web/app/data/HDP2/site_properties.js
+++ b/ambari-web/app/data/HDP2/site_properties.js
@@ -2215,21 +2215,21 @@ var hdp2properties = [
     "index": 6
   },
   {
-    "name": "logsearch.external.auth.enabled",
+    "name": "logsearch.auth.external_auth.enable",
     "serviceName": "LOGSEARCH",
     "filename": "logsearch-properties.xml",
     "category": "Advanced logsearch-properties",
     "index": 7
   },
   {
-    "name": "logsearch.external.auth.host_url",
+    "name": "logsearch.auth.external_auth.host_url",
     "serviceName": "LOGSEARCH",
     "filename": "logsearch-properties.xml",
     "category": "Advanced logsearch-properties",
     "index": 8
   },
   {
-    "name": "logsearch.external.auth.login_url",
+    "name": "logsearch.auth.external_auth.login_url",
     "serviceName": "LOGSEARCH",
     "filename": "logsearch-properties.xml",
     "category": "Advanced logsearch-properties",


Mime
View raw message