Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 5DDC2200C62 for ; Wed, 26 Apr 2017 22:00:17 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 5C63F160BB4; Wed, 26 Apr 2017 20:00:17 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3141216090A for ; Wed, 26 Apr 2017 22:00:16 +0200 (CEST) Received: (qmail 47780 invoked by uid 500); 26 Apr 2017 20:00:15 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 47769 invoked by uid 99); 26 Apr 2017 20:00:15 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 26 Apr 2017 20:00:15 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 20176E1835; Wed, 26 Apr 2017 20:00:15 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ncole@apache.org To: commits@ambari.apache.org Date: Wed, 26 Apr 2017 20:00:15 -0000 Message-Id: <80a2f979df6345628ec4e718668c9e92@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] ambari git commit: AMBARI-20846. Use exclude list of mount device types on docker containers (dgrinenko via ncole) archived-at: Wed, 26 Apr 2017 20:00:17 -0000 Repository: ambari Updated Branches: refs/heads/branch-2.4 40747a956 -> 1ac59c3ff http://git-wip-us.apache.org/repos/asf/ambari/blob/1ac59c3f/ambari-web/app/utils/configs/mount_points_based_initializer_mixin.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/configs/mount_points_based_initializer_mixin.js b/ambari-web/app/utils/configs/mount_points_based_initializer_mixin.js deleted file mode 100644 index 59a3985..0000000 --- a/ambari-web/app/utils/configs/mount_points_based_initializer_mixin.js +++ /dev/null @@ -1,340 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -var App = require('app'); - -/** - * Regexp used to determine if mount point is windows-like - * - * @type {RegExp} - */ -var winRegex = /^([a-z]):\\?$/; - -App.MountPointsBasedInitializerMixin = Em.Mixin.create({ - - /** - * Map for methods used as value-modifiers for configProperties with values as mount point(s) - * Used if mount point is win-like (@see winRegex) - * Key: id - * Value: method-name - * - * @type {{default: string, file: string, slashes: string}} - */ - winReplacersMap: { - default: '_defaultWinReplace', - file: '_winReplaceWithFile', - slashes: '_defaultWinReplaceWithAdditionalSlashes' - }, - - /** - * Initializer for configs with value as one of the possible mount points - * Only hosts that contains on the components from initializer.components are processed - * Hosts with Windows needs additional processing (@see winReplacersMap) - * Value example: '/', '/some/cool/dir' - * - * @param {configProperty} configProperty - * @param {topologyLocalDB} localDB - * @param {object} dependencies - * @param {object} initializer - * @return {Object} - */ - _initAsSingleMountPoint: function (configProperty, localDB, dependencies, initializer) { - var hostsInfo = this._updateHostInfo(localDB.hosts); - var setOfHostNames = this._getSetOfHostNames(localDB, initializer); - var winReplacersMap = this.get('winReplacersMap'); - // In Add Host Wizard, if we did not select this slave component for any host, then we don't process any further. - if (!setOfHostNames.length) { - return configProperty; - } - var allMountPoints = this._getAllMountPoints(setOfHostNames, hostsInfo, localDB); - - var mPoint = allMountPoints[0].mountpoint; - if (mPoint === "/") { - mPoint = Em.get(configProperty, 'recommendedValue'); - } - else { - var mp = mPoint.toLowerCase(); - if (winRegex.test(mp)) { - var methodName = winReplacersMap[initializer.winReplacer]; - mPoint = this[methodName].call(this, configProperty, mp); - } - else { - mPoint = mPoint + Em.get(configProperty, 'recommendedValue'); - } - } - Em.setProperties(configProperty, { - value: mPoint, - recommendedValue: mPoint - }); - - return configProperty; - }, - - /** - * Initializer for configs with value as all of the possible mount points - * Only hosts that contains on the components from initializer.components are processed - * Hosts with Windows needs additional processing (@see winReplacersMap) - * Value example: '/\n/some/cool/dir' (`\n` - is divider) - * - * @param {Object} configProperty - * @param {topologyLocalDB} localDB - * @param {object} dependencies - * @param {object} initializer - * @return {Object} - */ - _initAsMultipleMountPoints: function (configProperty, localDB, dependencies, initializer) { - var hostsInfo = this._updateHostInfo(localDB.hosts); - var self = this; - var setOfHostNames = this._getSetOfHostNames(localDB, initializer); - var winReplacersMap = this.get('winReplacersMap'); - // In Add Host Wizard, if we did not select this slave component for any host, then we don't process any further. - if (!setOfHostNames.length) { - return configProperty; - } - - var allMountPoints = this._getAllMountPoints(setOfHostNames, hostsInfo, localDB); - var mPoint = ''; - - allMountPoints.forEach(function (eachDrive) { - if (eachDrive.mountpoint === '/') { - mPoint += Em.get(configProperty, 'recommendedValue') + "\n"; - } - else { - var mp = eachDrive.mountpoint.toLowerCase(); - if (winRegex.test(mp)) { - var methodName = winReplacersMap[initializer.winReplacer]; - mPoint += self[methodName].call(this, configProperty, mp); - } - else { - mPoint += eachDrive.mountpoint + Em.get(configProperty, 'recommendedValue') + "\n"; - } - } - }, this); - - Em.setProperties(configProperty, { - value: mPoint, - recommendedValue: mPoint - }); - - return configProperty; - }, - - /** - * Replace drive-based windows-path with 'file:///' - * - * @param {configProperty} configProperty - * @param {string} mountPoint - * @returns {string} - * @private - */ - _winReplaceWithFile: function (configProperty, mountPoint) { - var winDriveUrl = mountPoint.toLowerCase().replace(winRegex, 'file:///$1:'); - return winDriveUrl + Em.get(configProperty, 'recommendedValue') + '\n'; - }, - - /** - * Replace drive-based windows-path - * - * @param {configProperty} configProperty - * @param {string} mountPoint - * @returns {string} - * @private - */ - _defaultWinReplace: function (configProperty, mountPoint) { - var winDrive = mountPoint.toLowerCase().replace(winRegex, '$1:'); - var winDir = Em.get(configProperty, 'recommendedValue').replace(/\//g, '\\'); - return winDrive + winDir + '\n'; - }, - - /** - * Same to _defaultWinReplace, but with extra-slash in the end - * - * @param {configProperty} configProperty - * @param {string} mountPoint - * @returns {string} - * @private - */ - _defaultWinReplaceWithAdditionalSlashes: function (configProperty, mountPoint) { - var winDrive = mountPoint.toLowerCase().replace(winRegex, '$1:'); - var winDir = Em.get(configProperty, 'recommendedValue').replace(/\//g, '\\\\'); - return winDrive + winDir + '\n'; - }, - - /** - * Update information from localDB using App.Host-model - * - * @param {object} hostsInfo - * @returns {object} - * @private - */ - _updateHostInfo: function (hostsInfo) { - App.Host.find().forEach(function (item) { - if (!hostsInfo[item.get('id')]) { - hostsInfo[item.get('id')] = { - name: item.get('id'), - cpu: item.get('cpu'), - memory: item.get('memory'), - disk_info: item.get('diskInfo'), - bootStatus: "REGISTERED", - isInstalled: true - }; - } - }); - return hostsInfo; - }, - - /** - * Determines if mount point is valid - * Criterias: - *
    - *
  • Should has available space
    • - *
  • Should not be home-dir
    • - *
  • Should not be docker-dir
    • - *
  • Should not be boot-dir
    • - *
  • Should not be dev-dir
    • - *
  • Valid mount point started from /usr/hdp/ should be /usr/hdp/current - * or /usr/hdp/ e.g. /usr/hdp/2.5.0.0 - *
    • - *
- * - * @param {{mountpoint: string, available: number}} mPoint - * @returns {function} true - valid, false - invalid - * @private - */ - _filterMountPoint: function (localDB) { - var stackVersionNumber = [Em.getWithDefault(localDB.selectedStack || {}, 'repository_version', null)].compact(); - return function(mPoint) { - var isAvailable = mPoint.available !== 0; - if (!isAvailable) { - return false; - } - - var stackRoot = '/usr/hdp'; - var notHome = !['/', '/home'].contains(mPoint.mountpoint); - var notDocker = !['/etc/resolv.conf', '/etc/hostname', '/etc/hosts'].contains(mPoint.mountpoint); - var notBoot = mPoint.mountpoint && !(mPoint.mountpoint.startsWith('/boot') - || mPoint.mountpoint.startsWith('/mnt') - || mPoint.mountpoint.startsWith('/tmp')); - var notDev = !(['devtmpfs', 'tmpfs', 'vboxsf', 'CDFS'].contains(mPoint.type)); - var validStackRootMount = !(mPoint.mountpoint.startsWith(stackRoot) && !['current'].concat(stackVersionNumber).filter(function(i) { - return mPoint.mountpoint === stackRoot + '/' + i; - }).length); - - return notHome && notDocker && notBoot && notDev && validStackRootMount; - }; - }, - - /** - * Get list of hostNames from localDB which contains needed components - * - * @param {topologyLocalDB} localDB - * @param {object} initializer - * @returns {string[]} - * @private - */ - _getSetOfHostNames: function (localDB, initializer) { - var masterComponentHostsInDB = Em.getWithDefault(localDB, 'masterComponentHosts', []); - var slaveComponentHostsInDB = Em.getWithDefault(localDB, 'slaveComponentHosts', []); - var hosts = masterComponentHostsInDB.filter(function (master) { - return initializer.components.contains(master.component); - }).mapProperty('hostName'); - - var sHosts = slaveComponentHostsInDB.find(function (slave) { - return initializer.components.contains(slave.componentName); - }); - if (sHosts) { - hosts = hosts.concat(sHosts.hosts.mapProperty('hostName')); - } - return hosts; - }, - - /** - * Get list of all unique valid mount points for hosts - * - * @param {string[]} setOfHostNames - * @param {object} hostsInfo - * @param {topologyLocalDB} localDB - * @returns {string[]} - * @private - */ - _getAllMountPoints: function (setOfHostNames, hostsInfo, localDB) { - var allMountPoints = [], - mountPointFilter = this._filterMountPoint(localDB); - for (var i = 0; i < setOfHostNames.length; i++) { - var hostname = setOfHostNames[i]; - var mountPointsPerHost = hostsInfo[hostname].disk_info; - var mountPointAsRoot = mountPointsPerHost.findProperty('mountpoint', '/'); - - // If Server does not send any host details information then atleast one mountpoint should be presumed as root - // This happens in a single container Linux Docker environment. - if (!mountPointAsRoot) { - mountPointAsRoot = { - mountpoint: '/' - }; - } - - mountPointsPerHost.filter(mountPointFilter).forEach(function (mPoint) { - if( !allMountPoints.findProperty("mountpoint", mPoint.mountpoint)) { - allMountPoints.push(mPoint); - } - }, this); - } - - if (!allMountPoints.length) { - allMountPoints.push(mountPointAsRoot); - } - return allMountPoints; - }, - - /** - * Settings for single_mountpoint-initializer - * Used for configs with value as one of the possible mount points - * - * @see _initAsSingleMountPoint - * @param {string|string[]} components - * @param {string} winReplacer - * @returns {{components: string[], winReplacer: string, type: string}} - */ - getSingleMountPointConfig: function (components, winReplacer) { - winReplacer = winReplacer || 'default'; - return { - components: Em.makeArray(components), - winReplacer: winReplacer, - type: 'single_mountpoint' - }; - }, - - /** - * Settings for multiple_mountpoints-initializer - * Used for configs with value as all of the possible mount points - * - * @see _initAsMultipleMountPoints - * @param {string|string[]} components - * @param {string} winReplacer - * @returns {{components: string[], winReplacer: string, type: string}} - */ - getMultipleMountPointsConfig: function (components, winReplacer) { - winReplacer = winReplacer || 'default'; - return { - components: Em.makeArray(components), - winReplacer: winReplacer, - type: 'multiple_mountpoints' - }; - } - -}); http://git-wip-us.apache.org/repos/asf/ambari/blob/1ac59c3f/ambari-web/test/utils/ajax/ajax_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/utils/ajax/ajax_test.js b/ambari-web/test/utils/ajax/ajax_test.js index 747ccff..35de05f 100644 --- a/ambari-web/test/utils/ajax/ajax_test.js +++ b/ambari-web/test/utils/ajax/ajax_test.js @@ -23,11 +23,16 @@ describe('App.ajax', function() { beforeEach(function() { App.ajax.send.restore(); + sinon.stub(App.logger, 'setTimer'); sinon.spy(App.ajax, 'send'); // no sense to test stubbed function, so going to spy on it App.set('apiPrefix', '/api/v1'); App.set('clusterName', 'tdk'); }); + afterEach(function() { + App.logger.setTimer.restore(); + }); + describe('#send', function() { it('Without sender', function() { expect(App.ajax.send({})).to.equal(null); @@ -162,7 +167,7 @@ describe('App.ajax', function() { }); }); }); - + describe('#abortRequests', function () { var xhr = { @@ -192,6 +197,6 @@ describe('App.ajax', function() { it('should clear requests array', function () { expect(requests).to.have.length(0); }); - + }); }); http://git-wip-us.apache.org/repos/asf/ambari/blob/1ac59c3f/ambari-web/test/utils/configs/config_initializer_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/utils/configs/config_initializer_test.js b/ambari-web/test/utils/configs/config_initializer_test.js index a5031b1..747a3ca 100644 --- a/ambari-web/test/utils/configs/config_initializer_test.js +++ b/ambari-web/test/utils/configs/config_initializer_test.js @@ -849,312 +849,6 @@ describe('App.ConfigInitializer', function () { }); - describe('config with mount points', function () { - - var localDB = { - masterComponentHosts: [ - { - component: 'NAMENODE', - hostName: 'h0' - }, - { - component: 'SECONDARY_NAMENODE', - hostName: 'h4' - }, - { - component: 'APP_TIMELINE_SERVER', - hostName: 'h0' - }, - { - component: 'ZOOKEEPER_SERVER', - hostName: 'h0' - }, - { - component: 'ZOOKEEPER_SERVER', - hostName: 'h1' - }, - { - component: 'OOZIE_SERVER', - hostName: 'h0' - }, - { - component: 'OOZIE_SERVER', - hostName: 'h1' - }, - { - component: 'NIMBUS', - hostName: 'h2' - }, - { - component: 'FALCON_SERVER', - hostName: 'h3' - }, - { - component: 'KAFKA_BROKER', - hostName: 'h0' - }, - { - component: 'KAFKA_BROKER', - hostName: 'h1' - } - ], - slaveComponentHosts: [ - { - componentName: 'DATANODE', - hosts: [ - { - hostName: 'h0' - }, - { - hostName: 'h1' - } - ] - }, - { - componentName: 'TASKTRACKER', - hosts: [ - { - hostName: 'h0' - }, - { - hostName: 'h1' - } - ] - }, - { - componentName: 'NODEMANAGER', - hosts: [ - { - hostName: 'h0' - }, - { - hostName: 'h1' - }, - { - hostName: 'h4' - } - ] - }, - { - componentName: 'HBASE_REGIONSERVER', - hosts: [ - { - hostName: 'h0' - }, - { - hostName: 'h1' - } - ] - }, - { - componentName: 'SUPERVISOR', - hosts: [ - { - hostName: 'h0' - }, - { - hostName: 'h1' - } - ] - } - ], - hosts: { - h0: { - disk_info: [ - { - mountpoint: '/' - }, - { - mountpoint: '/home' - }, - { - mountpoint: '/boot' - }, - { - mountpoint: '/boot/efi' - }, - { - mountpoint: '/mnt' - }, - { - mountpoint: '/mnt/efi' - }, - { - mountpoint: '/media/disk0', - available: '100000000' - }, - { - mountpoint: '/mount0', - available: '100000000' - } - ] - }, - h4: { - disk_info: [ - { - mountpoint: 'c:', - available: '100000000' - } - ] - } - } - }, - cases = [ - { - name: 'dfs.namenode.name.dir', - isOnlyFirstOneNeeded: false, - value: '/media/disk0/default\n/mount0/default\n' - }, - { - name: 'dfs.name.dir', - isOnlyFirstOneNeeded: false, - value: '/media/disk0/default\n/mount0/default\n' - }, - { - name: 'fs.checkpoint.dir', - isOnlyFirstOneNeeded: true, - value: 'file:///c:/default\n' - }, - { - name: 'dfs.namenode.checkpoint.dir', - isOnlyFirstOneNeeded: true, - value: 'file:///c:/default\n' - }, - { - name: 'dfs.data.dir', - isOnlyFirstOneNeeded: false, - value: '/media/disk0/default\n/mount0/default\n/media/disk1/default\n/mount1/default\n' - }, - { - name: 'dfs.datanode.data.dir', - isOnlyFirstOneNeeded: false, - value: '/media/disk0/default\n/mount0/default\n/media/disk1/default\n/mount1/default\n' - }, - { - name: 'mapred.local.dir', - isOnlyFirstOneNeeded: false, - value: '/media/disk0/default\n/mount0/default\n/media/disk1/default\n/mount1/default\n' - }, - { - name: 'yarn.nodemanager.log-dirs', - isOnlyFirstOneNeeded: false, - value: '/media/disk0/default\n/mount0/default\n/media/disk1/default\n/mount1/default\nc:\\default\n' - }, - { - name: 'yarn.nodemanager.local-dirs', - isOnlyFirstOneNeeded: false, - value: '/media/disk0/default\n/mount0/default\n/media/disk1/default\n/mount1/default\nc:\\default\n' - }, - { - name: 'yarn.timeline-service.leveldb-timeline-store.path', - isOnlyFirstOneNeeded: true, - value: '/media/disk0/default' - }, - { - name: 'yarn.timeline-service.leveldb-state-store.path', - isOnlyFirstOneNeeded: true, - value: '/media/disk0/default' - }, - { - name: 'dataDir', - isOnlyFirstOneNeeded: true, - value: '/media/disk0/default' - }, - { - name: 'oozie_data_dir', - isOnlyFirstOneNeeded: true, - value: '/media/disk0/default' - }, - { - name: 'storm.local.dir', - isOnlyFirstOneNeeded: true, - value: '/media/disk0/default' - }, - { - name: '*.falcon.graph.storage.directory', - isOnlyFirstOneNeeded: true, - value: '/default' - }, - { - name: '*.falcon.graph.serialize.path', - isOnlyFirstOneNeeded: true, - value: '/default' - }, - { - name: 'log.dirs', - isOnlyFirstOneNeeded: false, - value: '/media/disk0/default\n/mount0/default\n/media/disk1/default\n/mount1/default\n' - } - ]; - - beforeEach(function () { - sinon.stub(App.Host, 'find').returns([ - Em.Object.create({ - id: 'h1', - diskInfo: [ - { - mountpoint: '/media/disk1', - type: 'devtmpfs' - }, - { - mountpoint: '/media/disk1', - type: 'tmpfs' - }, - { - mountpoint: '/media/disk1', - type: 'vboxsf' - }, - { - mountpoint: '/media/disk1', - type: 'CDFS' - }, - { - mountpoint: '/media/disk1', - available: '0' - }, - { - mountpoint: '/media/disk1', - available: '100000000' - }, - { - mountpoint: '/mount1', - available: '100000000' - } - ] - }), - Em.Object.create({ - id: 'h2', - diskInfo: [ - { - mountpoint: '/' - } - ] - }), - Em.Object.create({ - id: 'h3', - diskInfo: [] - }) - ]); - }); - - afterEach(function () { - App.Host.find.restore(); - }); - - cases.forEach(function (item) { - it(item.name, function () { - serviceConfigProperty.setProperties({ - name: item.name, - recommendedValue: '/default' - }); - App.ConfigInitializer.initialValue(serviceConfigProperty, localDB, {}); - expect(serviceConfigProperty.get('value')).to.equal(item.value); - expect(serviceConfigProperty.get('recommendedValue')).to.equal(item.value); - }); - }); - - }); - describe('initializerTypes', function () { var types = App.ConfigInitializer.get('initializerTypes'); Em.keys(types).forEach(function(type) { @@ -1219,155 +913,4 @@ describe('App.ConfigInitializer', function () { }); }); - describe('#_filterMountPoint', function() { - [ - { - mPoint: { - mountpoint: '/' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/home' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/etc/resolv.conf' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/etc/hostname' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/etc/hosts' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/boot' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/mnt' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/tmp' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/some-dir', - type: 'devtmpfs' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/some-dir', - type: 'tmpfs' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/some-dir', - type: 'vboxsf' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/some-dir', - type: 'CDFS' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/usr/hdp' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/usr/hdp/1' - }, - localDB: {}, - e: false - }, - { - mPoint: { - mountpoint: '/usr/hdp/current' - }, - localDB: {}, - e: true - }, - { - mPoint: { - mountpoint: '/usr/hdp/2.5' - }, - localDB: { - selectedStack: { - repository_version: '2.5' - } - }, - e: true - }, - { - mPoint: { - mountpoint: '/usr/hdp/2.5.0' - }, - localDB: { - selectedStack: { - repository_version: '2.5' - } - }, - e: false - }, - { - mPoint: { - mountpoint: '/normal/directory' - }, - localDB: { - selectedStack: { - repository_version: '2.5' - } - }, - e: true - } - ].forEach(function(test) { - it('mount point "{0}" should be {1}'.format(test.mPoint.mountpoint, test.e ? 'valid' : 'invalid'), function() { - var fFn = App.ConfigInitializer._filterMountPoint(test.localDB); - expect(fFn(test.mPoint)).to.be.equal(test.e); - }); - }); - }); });