hadoop-ozone-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject [hadoop-ozone] branch master updated: HDDS-1335. Add basic UI for showing missing containers and keys
Date Tue, 28 Jan 2020 12:32:27 GMT
This is an automated email from the ASF dual-hosted git repository.

elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git


The following commit(s) were added to refs/heads/master by this push:
     new ef79f33  HDDS-1335. Add basic UI for showing missing containers and keys
ef79f33 is described below

commit ef79f3332dde071da7299dc76077dc594dfa3264
Author: Vivek Ratnavel Subramanian <vivekratnavel90@gmail.com>
AuthorDate: Tue Jan 28 13:31:24 2020 +0100

    HDDS-1335. Add basic UI for showing missing containers and keys
    
    Closes #492
---
 .../webapps/recon/ozone-recon-web/api/db.json      |  85 ++++++++
 .../webapps/recon/ozone-recon-web/api/routes.json  |   3 +-
 .../webapps/recon/ozone-recon-web/src/App.less     |   4 +-
 .../src/components/OverviewCard/OverviewCard.less  |   4 +
 .../src/components/OverviewCard/OverviewCard.tsx   |   9 +-
 .../src/constants/breadcrumbs.constants.tsx        |   3 +-
 .../webapps/recon/ozone-recon-web/src/routes.tsx   |   5 +
 .../MissingContainers.less}                        |  10 -
 .../views/MissingContainers/MissingContainers.tsx  | 218 +++++++++++++++++++++
 .../src/views/Overview/Overview.less               |   3 +
 .../src/views/Overview/Overview.tsx                |  35 +++-
 11 files changed, 354 insertions(+), 25 deletions(-)

diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
index 578e280..3e3d413 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/db.json
@@ -271,5 +271,90 @@
         "leaderElections": 5
       }
     ]
+  },
+  "missingContainers": {
+    "totalCount": 2,
+    "containers": [
+      {
+        "id": 1,
+        "keys": 3876,
+        "datanodes": [
+          "localhost1.storage.enterprise.com",
+          "localhost3.storage.enterprise.com",
+          "localhost5.storage.enterprise.com"
+        ]
+      },
+      {
+        "id": 2,
+        "keys": 5943,
+        "datanodes": [
+          "localhost1.storage.enterprise.com",
+          "localhost3.storage.enterprise.com",
+          "localhost5.storage.enterprise.com"
+        ]
+      }
+    ]
+  },
+  "keys": {
+    "totalCount": 534,
+    "keys": [
+      {
+        "Volume": "vol-0-20448",
+        "Bucket": "bucket-0-12811",
+        "Key": "key-0-77505",
+        "DataSize": 10240,
+        "Versions": [
+          0
+        ],
+        "Blocks": {
+          "0": [
+            {
+              "containerID": 1,
+              "localID": 103206297511460860
+            }
+          ]
+        },
+        "CreationTime": "2019-11-26T21:18:43.688Z",
+        "ModificationTime": "2019-11-26T21:18:46.062Z"
+      },
+      {
+        "Volume": "vol-0-20448",
+        "Bucket": "bucket-0-12811",
+        "Key": "key-21-64511",
+        "DataSize": 5692407,
+        "Versions": [
+          0
+        ],
+        "Blocks": {
+          "0": [
+            {
+              "containerID": 1,
+              "localID": 103206299949795380
+            }
+          ]
+        },
+        "CreationTime": "2019-11-26T21:19:20.855Z",
+        "ModificationTime": "2019-11-26T21:19:20.991Z"
+      },
+      {
+        "Volume": "vol-0-20448",
+        "Bucket": "bucket-0-12811",
+        "Key": "key-22-69104",
+        "DataSize": 189407,
+        "Versions": [
+          0
+        ],
+        "Blocks": {
+          "0": [
+            {
+              "containerID": 1,
+              "localID": 103206300033091740
+            }
+          ]
+        },
+        "CreationTime": "2019-11-26T21:19:22.126Z",
+        "ModificationTime": "2019-11-26T21:19:22.251Z"
+      }
+    ]
   }
 }
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json
index 028ffc3..df5cec3 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/api/routes.json
@@ -1,3 +1,4 @@
 {
-  "/api/v1/*": "/$1"
+  "/api/v1/*": "/$1",
+  "/containers/:id/keys": "/keys"
 }
\ No newline at end of file
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.less
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.less
index 2896a82..5de722c 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/App.less
@@ -51,10 +51,10 @@
 
         .ant-table-tbody {
           tr.ant-table-row:hover {
-            background: rgba(60, 90, 100, 0.04);
+            background: rgba(60, 90, 100, 0.01);
 
             & > td {
-              background: rgba(60, 90, 100, 0.04);
+              background: rgba(60, 90, 100, 0.01);
             }
           }
         }
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.less
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.less
index 774c791..536b1b4 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.less
@@ -36,3 +36,7 @@
     }
   }
 }
+
+.card-error {
+  border-color: #f5222d;
+}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.tsx
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.tsx
index cb591d2..fcf3fd2 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/components/OverviewCard/OverviewCard.tsx
@@ -32,13 +32,15 @@ interface OverviewCardProps extends RouteComponentProps<any> {
   loading?: boolean;
   linkToUrl?: string;
   capacityPercent?: number;
+  error?: boolean;
 }
 
 const defaultProps = {
   hoverable: false,
   loading: false,
   capacityPercent: -1,
-  linkToUrl: ''
+  linkToUrl: '',
+  error: false
 };
 
 interface OverviewCardWrapperProps {
@@ -62,8 +64,9 @@ class OverviewCard extends React.Component<OverviewCardProps> {
   static defaultProps = defaultProps;
 
   render() {
-    let {icon, data, title, loading, hoverable, capacityPercent, linkToUrl} = this.props;
+    let {icon, data, title, loading, hoverable, capacityPercent, linkToUrl, error} = this.props;
     let meta = <Meta title={data} description={title}/>;
+    const errorClass = error ? 'card-error' : '';
     if (capacityPercent && capacityPercent > -1) {
       meta = <div className="ant-card-percentage">
         {meta}
@@ -74,7 +77,7 @@ class OverviewCard extends React.Component<OverviewCardProps> {
 
     return (
         <OverviewCardWrapper linkToUrl={linkToUrl}>
-          <Card className="overview-card" loading={loading} hoverable={hoverable}>
+          <Card className={`overview-card ${errorClass}`} loading={loading} hoverable={hoverable}>
             <Row type="flex" justify="space-between">
               <Col span={18}>
                 <Row>
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx
index 4cd1cac..93773b9 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/constants/breadcrumbs.constants.tsx
@@ -23,5 +23,6 @@ interface IBreadcrumbNameMap {
 export const breadcrumbNameMap: IBreadcrumbNameMap = {
   '/Overview': 'Overview',
   '/Datanodes': 'Datanodes',
-  '/Pipelines': 'Pipelines'
+  '/Pipelines': 'Pipelines',
+  '/MissingContainers': 'Missing Containers'
 };
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/routes.tsx
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/routes.tsx
index 37f4338..8d520db 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/routes.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/routes.tsx
@@ -21,6 +21,7 @@ import {Datanodes} from './views/Datanodes/Datanodes';
 import {Pipelines} from "./views/Pipelines/Pipelines";
 import {NotFound} from './views/NotFound/NotFound';
 import {IRoute} from "./routes.types";
+import {MissingContainers} from "./views/MissingContainers/MissingContainers";
 
 export const routes: IRoute[] = [
   {
@@ -36,6 +37,10 @@ export const routes: IRoute[] = [
     component: Pipelines
   },
   {
+    path: "/MissingContainers",
+    component: MissingContainers
+  },
+  {
     path: "/:NotFound",
     component: NotFound,
   }
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.less
similarity index 87%
copy from hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less
copy to hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.less
index c8e74d2..5d5f1e3 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.less
@@ -15,13 +15,3 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-.overview-content {
-  margin: 20px 5px;
-  .icon-small {
-    font-size: 16px;
-  }
-  .meta {
-    font-size: 12px;
-  }
-}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.tsx
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.tsx
new file mode 100644
index 0000000..f97ec5e
--- /dev/null
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/MissingContainers/MissingContainers.tsx
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import axios from 'axios';
+import {Table} from 'antd';
+import './MissingContainers.less';
+import {PaginationConfig} from "antd/lib/pagination";
+import prettyBytes from "pretty-bytes";
+import moment from "moment";
+
+interface MissingContainerResponse {
+  id: number;
+  keys: number;
+  datanodes: string[];
+}
+
+interface MissingContainersResponse  {
+  totalCount: number;
+  containers: MissingContainerResponse[];
+}
+
+interface KeyResponse {
+  Volume: string;
+  Bucket: string;
+  Key: string;
+  DataSize: number;
+  Versions: number[];
+  Blocks: any;
+  CreationTime: string;
+  ModificationTime: string;
+}
+
+interface ContainerKeysResponse {
+  totalCount: number;
+  keys: KeyResponse[];
+}
+
+const COLUMNS = [
+  {
+    title: 'Container ID',
+    dataIndex: 'id',
+    key: 'id'
+  },
+  {
+    title: 'No. of Keys',
+    dataIndex: 'keys',
+    key: 'keys'
+  },
+  {
+    title: 'Datanodes',
+    dataIndex: 'datanodes',
+    key: 'datanodes',
+    render: (datanodes: string[]) => <div>{datanodes.map(datanode => <div
key={datanode}>{datanode}</div>)}</div>
+  }
+];
+
+const KEY_TABLE_COLUMNS = [
+  {
+    title: 'Volume',
+    dataIndex: 'Volume',
+    key: 'Volume'
+  },
+  {
+    title: 'Bucket',
+    dataIndex: 'Bucket',
+    key: 'Bucket'
+  },
+  {
+    title: 'Key',
+    dataIndex: 'Key',
+    key: 'Key'
+  },
+  {
+    title: 'Size',
+    dataIndex: 'DataSize',
+    key: 'DataSize',
+    render: (dataSize: number) => <div>{prettyBytes(dataSize)}</div>
+  },
+  {
+    title: 'Date Created',
+    dataIndex: 'CreationTime',
+    key: 'CreationTime',
+    render: (date: number) => moment(date).format('lll')
+  },
+  {
+    title: 'Date Modified',
+    dataIndex: 'ModificationTime',
+    key: 'ModificationTime',
+    render: (date: number) => moment(date).format('lll')
+  }
+];
+
+interface ExpandedRow {
+  [key: number]: ExpandedRowState
+}
+
+interface ExpandedRowState {
+  containerId: number;
+  loading: boolean;
+  dataSource: KeyResponse[];
+  totalCount: number;
+}
+
+interface MissingContainersState {
+  loading: boolean;
+  dataSource: MissingContainerResponse[];
+  totalCount: number;
+  expandedRowData: ExpandedRow
+}
+
+export class MissingContainers extends React.Component<any, MissingContainersState>
{
+
+  constructor(props: any) {
+    super(props);
+    this.state = {
+      loading: false,
+      dataSource: [],
+      totalCount: 0,
+      expandedRowData: {}
+    }
+  }
+
+  componentDidMount(): void {
+    // Fetch missing containers on component mount
+    this.setState({
+      loading: true
+    });
+    axios.get('/api/v1/missingContainers').then(response => {
+      const missingContainersResponse: MissingContainersResponse = response.data;
+      const totalCount = missingContainersResponse.totalCount;
+      const missingContainers: MissingContainerResponse[] = missingContainersResponse.containers;
+      this.setState({
+        loading: false,
+        dataSource: missingContainers,
+        totalCount: totalCount
+      });
+    });
+  }
+
+  onShowSizeChange = (current: number, pageSize: number) => {
+    // TODO: Implement this method once server side pagination is enabled
+    console.log(current, pageSize);
+  };
+
+  onRowExpandClick = (expanded: boolean, record: MissingContainerResponse) => {
+    if (expanded) {
+      this.setState(({expandedRowData}) => {
+        const expandedRowState: ExpandedRowState = expandedRowData[record.id] ?
+            Object.assign({}, expandedRowData[record.id], {loading: true}) :
+            {containerId: record.id, loading: true, dataSource: [], totalCount: 0};
+        return {
+          expandedRowData: Object.assign({}, expandedRowData, {[record.id]: expandedRowState})
+        }
+      });
+      axios.get(`/api/v1/containers/${record.id}/keys`).then(response => {
+        const containerKeysResponse: ContainerKeysResponse = response.data;
+        this.setState(({expandedRowData}) => {
+          const expandedRowState: ExpandedRowState =
+              Object.assign({}, expandedRowData[record.id],
+                  {loading: false, dataSource: containerKeysResponse.keys, totalCount: containerKeysResponse.totalCount});
+          return {
+            expandedRowData: Object.assign({}, expandedRowData, {[record.id]: expandedRowState})
+          }
+        });
+      });
+    }
+  };
+
+  expandedRowRender = (record: MissingContainerResponse) => {
+    const {expandedRowData} = this.state;
+    const containerId = record.id;
+    if (expandedRowData[containerId]) {
+      const containerKeys: ExpandedRowState = expandedRowData[containerId];
+      const paginationConfig: PaginationConfig = {
+        showTotal: (total: number, range) => `${range[0]}-${range[1]} of ${total} keys`
+      };
+      return <Table loading={containerKeys.loading} dataSource={containerKeys.dataSource}
+                    columns={KEY_TABLE_COLUMNS} pagination={paginationConfig}/>
+    }
+    return <div>Loading...</div>;
+  };
+
+  render () {
+    const {dataSource, loading, totalCount} = this.state;
+    const paginationConfig: PaginationConfig = {
+      showTotal: (total: number, range) => `${range[0]}-${range[1]} of ${total} missing
containers`,
+      showSizeChanger: true,
+      onShowSizeChange: this.onShowSizeChange
+    };
+    return (
+        <div className="missing-containers-container">
+          <div className="page-header">
+            Missing Containers ({totalCount})
+          </div>
+          <div className="content-div">
+            <Table dataSource={dataSource} columns={COLUMNS} loading={loading} pagination={paginationConfig}
+                   rowKey="id" expandedRowRender={this.expandedRowRender} expandRowByClick={true}
onExpand={this.onRowExpandClick}/>
+          </div>
+        </div>
+    );
+  }
+}
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less
index c8e74d2..5a8a069 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.less
@@ -24,4 +24,7 @@
   .meta {
     font-size: 12px;
   }
+  .padded-text {
+    padding-left: 5px;
+  }
 }
diff --git a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.tsx
b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.tsx
index 0d4d31b..cf0b911 100644
--- a/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.tsx
+++ b/hadoop-ozone/recon/src/main/resources/webapps/recon/ozone-recon-web/src/views/Overview/Overview.tsx
@@ -17,7 +17,7 @@
  */
 
 import React from 'react';
-import {Row, Col, Icon} from 'antd';
+import {Row, Col, Icon, Tooltip} from 'antd';
 import OverviewCard from 'components/OverviewCard/OverviewCard';
 import axios from 'axios';
 import prettyBytes from 'pretty-bytes';
@@ -34,6 +34,7 @@ interface OverviewState {
   volumes: string;
   buckets: string;
   keys: string;
+  missingContainersCount: number;
 }
 
 export class Overview extends React.Component<any, OverviewState> {
@@ -49,7 +50,8 @@ export class Overview extends React.Component<any, OverviewState>
{
       containers: '',
       volumes: '',
       buckets: '',
-      keys: ''
+      keys: '',
+      missingContainersCount: 0
     }
   }
 
@@ -57,8 +59,12 @@ export class Overview extends React.Component<any, OverviewState>
{
     this.setState({
       loading: true
     });
-    axios.get('/api/v1/stats').then(response => {
-      const stats = response.data;
+    axios.all([
+        axios.get('/api/v1/stats'),
+        axios.get('/api/v1/missingContainers')
+    ]).then(axios.spread((statsResponse, missingContainersResponse) => {
+      const stats = statsResponse.data;
+      const missingContainers = missingContainersResponse.data;
       const clusterUsedPercent = getCapacityPercent(stats.capacity.used, stats.capacity.total);
       this.setState({
         loading: false,
@@ -69,16 +75,28 @@ export class Overview extends React.Component<any, OverviewState>
{
         containers: stats.containers,
         volumes: stats.volumes,
         buckets: stats.buckets,
-        keys: stats.keys
+        keys: stats.keys,
+        missingContainersCount: missingContainers.totalCount
       });
-    });
+    }));
   }
 
   render() {
-    const {loading, datanodes, pipelines, clusterCapacity, clusterUsedPercent, containers,
volumes, buckets, keys} = this.state;
+    const {loading, datanodes, pipelines, clusterCapacity, clusterUsedPercent, containers,
volumes, buckets,
+      keys, missingContainersCount} = this.state;
     const datanodesElement = <span>
       <Icon type="check-circle" theme="filled" className="icon-success icon-small"/>
{datanodes} <span className="ant-card-meta-description meta">HEALTHY</span>
     </span>;
+    const containersTooltip = missingContainersCount === 1 ? "container is missing" : "containers
are missing";
+    const containersLink = missingContainersCount > 0 ? '/MissingContainers' : '';
+    const containersElement = missingContainersCount > 0 ?
+        <span>
+          <Tooltip placement="bottom" title={`${missingContainersCount} ${containersTooltip}`}>
+            <Icon type="exclamation-circle" theme="filled" className="icon-failure icon-small"/>
+          </Tooltip>
+          <span className="padded-text">{containers}</span>
+        </span>
+        : containers;
     return (
         <div className="overview-content">
           <Row gutter={[25, 25]}>
@@ -95,7 +113,8 @@ export class Overview extends React.Component<any, OverviewState>
{
                             capacityPercent={clusterUsedPercent}/>
             </Col>
             <Col xs={24} sm={18} md={12} lg={12} xl={6}>
-              <OverviewCard loading={loading} title={"Containers"} data={containers} icon={"container"}/>
+              <OverviewCard loading={loading} title={"Containers"} data={containersElement}
icon={"container"}
+                            error={missingContainersCount > 0} linkToUrl={containersLink}/>
             </Col>
           </Row>
           <Row gutter={[25, 25]}>


---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org


Mime
View raw message