ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lpus...@apache.org
Subject [23/57] [abbrv] ambari git commit: AMBARI-21890.Ambari Files View - browser going to hung state while opening a HDFS folder which has huge number of files(>10000)(Venkata Sairam)
Date Tue, 12 Sep 2017 10:04:40 GMT
AMBARI-21890.Ambari Files View - browser going to hung state while opening a HDFS folder which
has huge number of files(>10000)(Venkata Sairam)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f6ecbd1d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f6ecbd1d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f6ecbd1d

Branch: refs/heads/feature-branch-AMBARI-21307
Commit: f6ecbd1d7fa5f90ec020e64dbb403b3c3c2bdeb3
Parents: 5b1a63b
Author: Venkata Sairam <venkatasairam.lanka@gmail.com>
Authored: Fri Sep 8 08:38:54 2017 +0530
Committer: Venkata Sairam <venkatasairam.lanka@gmail.com>
Committed: Fri Sep 8 08:38:54 2017 +0530

----------------------------------------------------------------------
 .../view/commons/hdfs/FileOperationService.java |  41 +++-
 .../resources/ui/app/components/file-search.js  |  10 +-
 .../main/resources/ui/app/controllers/files.js  |  20 +-
 .../src/main/resources/ui/app/routes/files.js   |  16 +-
 .../ui/app/templates/components/file-row.hbs    |   2 +-
 .../ui/app/templates/components/file-search.hbs |   2 +-
 .../main/resources/ui/app/templates/files.hbs   |   8 +-
 .../view/filebrowser/FilebrowserTest.java       |   4 +-
 .../ambari/view/utils/hdfs/DirListInfo.java     |  97 +++++++++
 .../ambari/view/utils/hdfs/DirStatus.java       |  75 +++++++
 .../apache/ambari/view/utils/hdfs/HdfsApi.java  | 124 ++++++++++--
 .../ambari/view/utils/hdfs/HdfsApiTest.java     | 201 +++++++++++++++++++
 12 files changed, 557 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/f6ecbd1d/contrib/views/commons/src/main/java/org/apache/ambari/view/commons/hdfs/FileOperationService.java
----------------------------------------------------------------------
diff --git a/contrib/views/commons/src/main/java/org/apache/ambari/view/commons/hdfs/FileOperationService.java
b/contrib/views/commons/src/main/java/org/apache/ambari/view/commons/hdfs/FileOperationService.java
index d6e484d..6fa1056 100644
--- a/contrib/views/commons/src/main/java/org/apache/ambari/view/commons/hdfs/FileOperationService.java
+++ b/contrib/views/commons/src/main/java/org/apache/ambari/view/commons/hdfs/FileOperationService.java
@@ -18,12 +18,17 @@
 
 package org.apache.ambari.view.commons.hdfs;
 
+import com.google.common.base.Strings;
 import org.apache.ambari.view.ViewContext;
 import org.apache.ambari.view.commons.exceptions.NotFoundFormattedException;
 import org.apache.ambari.view.commons.exceptions.ServiceFormattedException;
+import org.apache.ambari.view.utils.hdfs.DirListInfo;
+import org.apache.ambari.view.utils.hdfs.DirStatus;
 import org.apache.ambari.view.utils.hdfs.HdfsApi;
 import org.apache.ambari.view.utils.hdfs.HdfsApiException;
 import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.ws.rs.*;
 import javax.ws.rs.core.*;
@@ -41,6 +46,14 @@ import java.util.Map;
  * File operations service
  */
 public class FileOperationService extends HdfsService {
+  private final static Logger LOG =
+      LoggerFactory.getLogger(FileOperationService.class);
+
+
+  private static final String FILES_VIEW_MAX_FILE_PER_PAGE = "views.files.max.files.per.page";
+  private static final int DEFAULT_FILES_VIEW_MAX_FILE_PER_PAGE = 5000;
+
+  private Integer maxFilesPerPage = DEFAULT_FILES_VIEW_MAX_FILE_PER_PAGE;
 
   /**
    * Constructor
@@ -48,6 +61,19 @@ public class FileOperationService extends HdfsService {
    */
   public FileOperationService(ViewContext context) {
     super(context);
+    setMaxFilesPerPage(context);
+  }
+
+  private void setMaxFilesPerPage(ViewContext context) {
+    String maxFilesPerPageProperty = context.getAmbariProperty(FILES_VIEW_MAX_FILE_PER_PAGE);
+    LOG.info("maxFilesPerPageProperty = {}", maxFilesPerPageProperty);
+    if(!Strings.isNullOrEmpty(maxFilesPerPageProperty)){
+      try {
+        maxFilesPerPage = Integer.parseInt(maxFilesPerPageProperty);
+      }catch(Exception e){
+        LOG.error("{} should be integer, but it is {}, using default value of {}", FILES_VIEW_MAX_FILE_PER_PAGE
, maxFilesPerPageProperty, DEFAULT_FILES_VIEW_MAX_FILE_PER_PAGE);
+      }
+    }
   }
 
   /**
@@ -56,21 +82,30 @@ public class FileOperationService extends HdfsService {
    */
   public FileOperationService(ViewContext context, Map<String, String> customProperties)
{
     super(context, customProperties);
+    this.setMaxFilesPerPage(context);
   }
 
   /**
    * List dir
    * @param path path
+   * @param nameFilter : name on which filter is applied
    * @return response with dir content
    */
   @GET
   @Path("/listdir")
   @Produces(MediaType.APPLICATION_JSON)
-  public Response listdir(@QueryParam("path") String path) {
+  public Response listdir(@QueryParam("path") String path, @QueryParam("nameFilter") String
nameFilter) {
     try {
       JSONObject response = new JSONObject();
-      response.put("files", getApi().fileStatusToJSON(getApi().listdir(path)));
-      response.put("meta", getApi().fileStatusToJSON(getApi().getFileStatus(path)));
+      Map<String, Object> parentInfo = getApi().fileStatusToJSON(getApi().getFileStatus(path));
+      DirStatus dirStatus = getApi().listdir(path, nameFilter, maxFilesPerPage);
+      DirListInfo dirListInfo = dirStatus.getDirListInfo();
+      parentInfo.put("originalSize", dirListInfo.getOriginalSize());
+      parentInfo.put("truncated", dirListInfo.isTruncated());
+      parentInfo.put("finalSize", dirListInfo.getFinalSize());
+      parentInfo.put("nameFilter", dirListInfo.getNameFilter());
+      response.put("files", getApi().fileStatusToJSON(dirStatus.getFileStatuses()));
+      response.put("meta", parentInfo);
       return Response.ok(response).build();
     } catch (WebApplicationException ex) {
       throw ex;

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6ecbd1d/contrib/views/files/src/main/resources/ui/app/components/file-search.js
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/main/resources/ui/app/components/file-search.js b/contrib/views/files/src/main/resources/ui/app/components/file-search.js
index b65749c..68ec280 100644
--- a/contrib/views/files/src/main/resources/ui/app/components/file-search.js
+++ b/contrib/views/files/src/main/resources/ui/app/components/file-search.js
@@ -23,11 +23,6 @@ export default Ember.Component.extend({
   classNameBindings: ['expanded::col-md-9', 'expanded::col-md-offset-3'],
   expanded: false,
 
-  searchText: '',
-
-  throttleTyping: Ember.observer('searchText', function() {
-    Ember.run.debounce(this, this.searchFiles, 500);
-  }),
 
   searchFiles: function() {
     this.sendAction('searchAction', this.get('searchText'));
@@ -38,5 +33,10 @@ export default Ember.Component.extend({
   },
   focusOut: function() {
     this.set('expanded', false);
+  },
+  actions : {
+      throttleTyping: function() {
+        Ember.run.debounce(this, this.searchFiles, 1000);
+      }
   }
 });

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6ecbd1d/contrib/views/files/src/main/resources/ui/app/controllers/files.js
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/main/resources/ui/app/controllers/files.js b/contrib/views/files/src/main/resources/ui/app/controllers/files.js
index 8b5bb7b..30d9896 100644
--- a/contrib/views/files/src/main/resources/ui/app/controllers/files.js
+++ b/contrib/views/files/src/main/resources/ui/app/controllers/files.js
@@ -28,9 +28,9 @@ export default Ember.Controller.extend({
   isSelected: Ember.computed('selectedFilesCount', 'selectedFolderCount', function() {
     return (this.get('selectedFilesCount') + this.get('selectedFolderCount')) !== 0;
   }),
-
-  queryParams: ['path'],
+  queryParams: ['path', 'filter'],
   path: '/',
+  filter: '',
   columns: columnConfig,
 
   currentMessagesCount: Ember.computed.alias('logger.currentMessagesCount'),
@@ -71,16 +71,10 @@ export default Ember.Controller.extend({
     return parentPath;
   }),
 
-  sortedContent: Ember.computed.sort('model', 'sortProperty'),
+  arrangedContent:  Ember.computed.sort('model', 'sortProperty'),
 
-  arrangedContent: Ember.computed('model', 'sortProperty', 'validSearchText', function()
{
-    var searchText = this.get('validSearchText');
-    if(!Ember.isBlank(searchText)) {
-      return this.get('sortedContent').filter(function(entry) {
-        return !!entry.get('name').match(searchText);
-      });
-    }
-    return this.get('sortedContent');
+  metaInfo: Ember.computed('model', function() {
+    return this.get('model.meta');
   }),
 
   selectedFilePathsText: function () {
@@ -144,7 +138,7 @@ export default Ember.Controller.extend({
     selectAll: function(selectStatus) {
       this.get('fileSelectionService').deselectAll();
       if(selectStatus === false) {
-        this.get('fileSelectionService').selectFiles(this.get('sortedContent'));
+        this.get('fileSelectionService').selectFiles(this.get('arrangedContent'));
       }
     },
 
@@ -155,7 +149,7 @@ export default Ember.Controller.extend({
 
     //Context Menu actions
     openFolder: function(path) {
-      this.transitionToRoute({queryParams: {path: path}});
+      this.transitionToRoute({queryParams: {path: path, filter:''}});
     }
   },
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6ecbd1d/contrib/views/files/src/main/resources/ui/app/routes/files.js
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/main/resources/ui/app/routes/files.js b/contrib/views/files/src/main/resources/ui/app/routes/files.js
index 140732f..be7a515 100644
--- a/contrib/views/files/src/main/resources/ui/app/routes/files.js
+++ b/contrib/views/files/src/main/resources/ui/app/routes/files.js
@@ -26,13 +26,15 @@ export default Ember.Route.extend(FileOperationMixin, {
   queryParams: {
     path: {
       refreshModel: true
+    },
+    filter: {
+      refreshModel: true
     }
   },
   model: function(params) {
     this.store.unloadAll('file');
-    return this.store.query('file', {path: params.path});
+    return this.store.query('file', {path: params.path, nameFilter:params.filter});
   },
-
   setupController: function(controller, model) {
     this._super(controller, model);
     controller.set('searchText', '');
@@ -44,7 +46,17 @@ export default Ember.Route.extend(FileOperationMixin, {
     refreshCurrentRoute: function() {
       this.refresh();
     },
+    searchAction : function(searchText) {
+     this.set('controller.filter', searchText);
 
+     this.transitionTo({
+       queryParams: {
+         path: this.get('currentPath'),
+         filter: searchText
+       }
+     });
+
+    },
     error: function(error, transition) {
       this.get('fileSelectionService').reset();
       let path = transition.queryParams.path;

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6ecbd1d/contrib/views/files/src/main/resources/ui/app/templates/components/file-row.hbs
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/main/resources/ui/app/templates/components/file-row.hbs
b/contrib/views/files/src/main/resources/ui/app/templates/components/file-row.hbs
index 72ed840..5198504 100644
--- a/contrib/views/files/src/main/resources/ui/app/templates/components/file-row.hbs
+++ b/contrib/views/files/src/main/resources/ui/app/templates/components/file-row.hbs
@@ -19,7 +19,7 @@
 <div class="row">
   <div class={{get-value-from-columns columnHeaders 'name' 'columnClass'}}>
     {{#if file.isDirectory}}
-      {{#link-to 'files' (query-params path=file.path) bubbles=false title=file.name}}{{fa-icon
"folder-o"}} {{shorten-text file.name 40}} {{/link-to}}
+      {{#link-to 'files' (query-params path=file.path filter='') bubbles=false title=file.name}}{{fa-icon
"folder-o"}} {{shorten-text file.name 40}} {{/link-to}}
     {{else}}
       <span title={{ file.name }}>{{fa-icon "file-o"}} {{shorten-text file.name 40}}</span>
     {{/if}}

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6ecbd1d/contrib/views/files/src/main/resources/ui/app/templates/components/file-search.hbs
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/main/resources/ui/app/templates/components/file-search.hbs
b/contrib/views/files/src/main/resources/ui/app/templates/components/file-search.hbs
index 298d672..f3dc8f9 100644
--- a/contrib/views/files/src/main/resources/ui/app/templates/components/file-search.hbs
+++ b/contrib/views/files/src/main/resources/ui/app/templates/components/file-search.hbs
@@ -16,5 +16,5 @@
 * limitations under the License.
 }}
 
-{{input type="text" placeholder="Search in current directory..." class="form-control input-sm"
value=searchText}}
+{{input type="text" placeholder="Search in current directory..." class="form-control input-sm"
action='throttleTyping' on="key-down" value=searchText}}
 <span class="input-group-addon">{{fa-icon icon='search'}}</span>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6ecbd1d/contrib/views/files/src/main/resources/ui/app/templates/files.hbs
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/main/resources/ui/app/templates/files.hbs b/contrib/views/files/src/main/resources/ui/app/templates/files.hbs
index 63e0dd8..5714cf3 100644
--- a/contrib/views/files/src/main/resources/ui/app/templates/files.hbs
+++ b/contrib/views/files/src/main/resources/ui/app/templates/files.hbs
@@ -29,7 +29,11 @@
       {{else}}
           <span class="context-text" style="    z-index: 1;
       position: relative;">
-        Total: <strong>{{arrangedContent.length}}</strong> files or folders
+        {{#if metaInfo.truncated}}
+            Showing <strong>{{arrangedContent.length}}</strong> files or folders
of <strong>{{metaInfo.originalSize}}</strong>
+        {{else}}
+            Total: <strong>{{arrangedContent.length}}</strong> files or folders
+        {{/if}}
       </span>
       {{/if}}
     </div>
@@ -82,7 +86,7 @@
         </div>
         <div class="col-md-4 col-xs-4">
             <div class="row">
-              {{file-search searchText=searchText searchAction="searchFiles"}}
+              {{file-search searchText=filter searchAction="searchAction"}}
             </div>
         </div>
     </div>

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6ecbd1d/contrib/views/files/src/test/java/org/apache/ambari/view/filebrowser/FilebrowserTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/files/src/test/java/org/apache/ambari/view/filebrowser/FilebrowserTest.java
b/contrib/views/files/src/test/java/org/apache/ambari/view/filebrowser/FilebrowserTest.java
index f431f66..6ddc8f6 100644
--- a/contrib/views/files/src/test/java/org/apache/ambari/view/filebrowser/FilebrowserTest.java
+++ b/contrib/views/files/src/test/java/org/apache/ambari/view/filebrowser/FilebrowserTest.java
@@ -110,7 +110,7 @@ public class FilebrowserTest{
     FileOperationService.MkdirRequest request = new FileOperationService.MkdirRequest();
     request.path = "/tmp1";
     fileBrowserService.fileOps().mkdir(request);
-    Response response = fileBrowserService.fileOps().listdir("/");
+    Response response = fileBrowserService.fileOps().listdir("/", null);
     JSONObject responseObject = (JSONObject) response.getEntity();
     JSONArray statuses = (JSONArray) responseObject.get("files");
     System.out.println(response.getEntity());
@@ -140,7 +140,7 @@ public class FilebrowserTest{
   public void testUploadFile() throws Exception {
     Response response = uploadFile("/tmp/", "testUpload", ".tmp", "Hello world");
     Assert.assertEquals(200, response.getStatus());
-    Response listdir = fileBrowserService.fileOps().listdir("/tmp");
+    Response listdir = fileBrowserService.fileOps().listdir("/tmp", null);
     JSONObject responseObject = (JSONObject) listdir.getEntity();
     JSONArray statuses = (JSONArray) responseObject.get("files");
     System.out.println(statuses.size());

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6ecbd1d/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/DirListInfo.java
----------------------------------------------------------------------
diff --git a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/DirListInfo.java
b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/DirListInfo.java
new file mode 100644
index 0000000..6bd13bb
--- /dev/null
+++ b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/DirListInfo.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.utils.hdfs;
+
+public class DirListInfo {
+  private int originalSize;
+  private boolean truncated;
+  private int finalSize;
+  private String nameFilter;
+
+  public DirListInfo(int originalSize, boolean truncated, int finalSize, String nameFilter)
{
+    this.originalSize = originalSize;
+    this.truncated = truncated;
+    this.finalSize = finalSize;
+    this.nameFilter = nameFilter;
+  }
+
+  public int getOriginalSize() {
+    return originalSize;
+  }
+
+  public void setOriginalSize(int originalSize) {
+    this.originalSize = originalSize;
+  }
+
+  public boolean isTruncated() {
+    return truncated;
+  }
+
+  public void setTruncated(boolean truncated) {
+    this.truncated = truncated;
+  }
+
+  public int getFinalSize() {
+    return finalSize;
+  }
+
+  public void setFinalSize(int finalSize) {
+    this.finalSize = finalSize;
+  }
+
+  public String getNameFilter() {
+    return nameFilter;
+  }
+
+  public void setNameFilter(String nameFilter) {
+    this.nameFilter = nameFilter;
+  }
+
+  @Override
+  public String toString() {
+    return "DirListInfo{" +
+        "originalSize=" + originalSize +
+        ", truncated=" + truncated +
+        ", finalSize=" + finalSize +
+        ", nameFilter='" + nameFilter + '\'' +
+        '}';
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    DirListInfo that = (DirListInfo) o;
+
+    if (originalSize != that.originalSize) return false;
+    if (truncated != that.truncated) return false;
+    if (finalSize != that.finalSize) return false;
+    return nameFilter != null ? nameFilter.equals(that.nameFilter) : that.nameFilter == null;
+  }
+
+  @Override
+  public int hashCode() {
+    int result = originalSize;
+    result = 31 * result + (truncated ? 1 : 0);
+    result = 31 * result + finalSize;
+    result = 31 * result + (nameFilter != null ? nameFilter.hashCode() : 0);
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6ecbd1d/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/DirStatus.java
----------------------------------------------------------------------
diff --git a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/DirStatus.java
b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/DirStatus.java
new file mode 100644
index 0000000..f922b00
--- /dev/null
+++ b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/DirStatus.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.view.utils.hdfs;
+
+import org.apache.hadoop.fs.FileStatus;
+
+import java.util.Arrays;
+
+public class DirStatus {
+  private DirListInfo dirListInfo;
+  private FileStatus [] fileStatuses;
+
+  public DirStatus(FileStatus[] fileStatuses, DirListInfo dirListInfo) {
+    this.fileStatuses = fileStatuses;
+    this.dirListInfo = dirListInfo;
+  }
+
+  public DirListInfo getDirListInfo() {
+    return dirListInfo;
+  }
+
+  public void setDirListInfo(DirListInfo dirListInfo) {
+    this.dirListInfo = dirListInfo;
+  }
+
+  public FileStatus[] getFileStatuses() {
+    return fileStatuses;
+  }
+
+  public void setFileStatuses(FileStatus[] fileStatuses) {
+    this.fileStatuses = fileStatuses;
+  }
+
+  @Override
+  public String toString() {
+    return "DirStatus{" +
+        "dirListInfo=" + dirListInfo +
+        ", fileStatuses=" + Arrays.toString(fileStatuses) +
+        '}';
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    DirStatus dirStatus = (DirStatus) o;
+
+    if (dirListInfo != null ? !dirListInfo.equals(dirStatus.dirListInfo) : dirStatus.dirListInfo
!= null) return false;
+    // Probably incorrect - comparing Object[] arrays with Arrays.equals
+    return Arrays.equals(fileStatuses, dirStatus.fileStatuses);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = dirListInfo != null ? dirListInfo.hashCode() : 0;
+    result = 31 * result + Arrays.hashCode(fileStatuses);
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6ecbd1d/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java
----------------------------------------------------------------------
diff --git a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java
b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java
index 90fa483..8b987be 100644
--- a/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java
+++ b/contrib/views/utils/src/main/java/org/apache/ambari/view/utils/hdfs/HdfsApi.java
@@ -18,6 +18,7 @@
 
 package org.apache.ambari.view.utils.hdfs;
 
+import com.google.common.base.Strings;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -40,6 +41,7 @@ import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
 import java.util.LinkedHashMap;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
@@ -51,13 +53,14 @@ public class HdfsApi {
       LoggerFactory.getLogger(HdfsApi.class);
 
   private final Configuration conf;
-  private final Map<String, String> authParams;
+  private Map<String, String> authParams;
 
   private FileSystem fs;
   private UserGroupInformation ugi;
 
   /**
    * Constructor
+   *
    * @param configurationBuilder hdfs configuration builder
    * @throws IOException
    * @throws InterruptedException
@@ -76,6 +79,38 @@ public class HdfsApi {
     });
   }
 
+  /**
+   * for testing
+   * @throws IOException
+   * @throws InterruptedException
+   * @throws HdfsApiException
+   */
+  HdfsApi(Configuration configuration, FileSystem fs, UserGroupInformation ugi) throws IOException,
+      InterruptedException, HdfsApiException {
+    if(null != configuration){
+      conf = configuration;
+    }else {
+      conf = new Configuration();
+    }
+
+    UserGroupInformation.setConfiguration(conf);
+    if(null != ugi){
+      this.ugi = ugi;
+    }else {
+      this.ugi = UserGroupInformation.getCurrentUser();
+    }
+
+    if(null != fs){
+      this.fs = fs;
+    }else {
+      this.fs = execute(new PrivilegedExceptionAction<FileSystem>() {
+        public FileSystem run() throws IOException {
+          return FileSystem.get(conf);
+        }
+      });
+    }
+  }
+
   private UserGroupInformation getProxyUser() throws IOException {
     UserGroupInformation proxyuser;
     if (authParams.containsKey("proxyuser")) {
@@ -101,6 +136,7 @@ public class HdfsApi {
 
   /**
    * List dir operation
+   *
    * @param path path
    * @return array of FileStatus objects
    * @throws FileNotFoundException
@@ -117,7 +153,56 @@ public class HdfsApi {
   }
 
   /**
+   *
+   * @param path : list files and dirs in this path
+   * @param nameFilter : if not empty or null, then file names that contain this are only
sent.
+   * @param maxAllowedSize : maximum number of files sent in output. -1 means infinite.
+   * @return
+   * @throws FileNotFoundException
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public DirStatus listdir(final String path, final String nameFilter, int maxAllowedSize)
throws FileNotFoundException,
+      IOException, InterruptedException {
+    FileStatus[] fileStatuses = this.listdir(path);
+    return filterAndTruncateDirStatus(nameFilter, maxAllowedSize, fileStatuses);
+  }
+
+  public DirStatus filterAndTruncateDirStatus(String nameFilter, int maxAllowedSize, FileStatus[]
fileStatuses) {
+    if(null == fileStatuses){
+      return new DirStatus(null, new DirListInfo(0, false, 0, nameFilter));
+    }
+
+    int originalSize = fileStatuses.length;
+    boolean truncated = false;
+
+    if (!Strings.isNullOrEmpty(nameFilter)) {
+      List<FileStatus> filteredList = new LinkedList<>();
+      for(FileStatus fileStatus : fileStatuses){
+        if(maxAllowedSize >=0 && maxAllowedSize <= filteredList.size()){
+          truncated = true;
+          break;
+        }
+        if(fileStatus.getPath().getName().contains(nameFilter)){
+          filteredList.add(fileStatus);
+        }
+      }
+      fileStatuses = filteredList.toArray(new FileStatus[0]);
+    }
+
+    if(maxAllowedSize >=0 && fileStatuses.length > maxAllowedSize) { // in
cases where name filter loop is not executed.
+      truncated = true;
+      fileStatuses = Arrays.copyOf(fileStatuses, maxAllowedSize);
+    }
+
+    int finalSize = fileStatuses.length;
+
+    return new DirStatus(fileStatuses, new DirListInfo(originalSize, truncated, finalSize,
nameFilter));
+  }
+
+  /**
    * Get file status
+   *
    * @param path path
    * @return file status
    * @throws IOException
@@ -135,6 +220,7 @@ public class HdfsApi {
 
   /**
    * Make directory
+   *
    * @param path path
    * @return success
    * @throws IOException
@@ -151,6 +237,7 @@ public class HdfsApi {
 
   /**
    * Rename
+   *
    * @param src source path
    * @param dst destination path
    * @return success
@@ -168,6 +255,7 @@ public class HdfsApi {
 
   /**
    * Check is trash enabled
+   *
    * @return true if trash is enabled
    * @throws Exception
    */
@@ -182,6 +270,7 @@ public class HdfsApi {
 
   /**
    * Home directory
+   *
    * @return home directory
    * @throws Exception
    */
@@ -195,6 +284,7 @@ public class HdfsApi {
 
   /**
    * Hdfs Status
+   *
    * @return home directory
    * @throws Exception
    */
@@ -208,6 +298,7 @@ public class HdfsApi {
 
   /**
    * Trash directory
+   *
    * @return trash directory
    * @throws Exception
    */
@@ -236,7 +327,7 @@ public class HdfsApi {
   /**
    * Trash directory path.
    *
-   * @param    filePath        the path to the file
+   * @param filePath the path to the file
    * @return trash directory path for the file
    * @throws Exception
    */
@@ -251,6 +342,7 @@ public class HdfsApi {
 
   /**
    * Empty trash
+   *
    * @return
    * @throws Exception
    */
@@ -266,6 +358,7 @@ public class HdfsApi {
 
   /**
    * Move to trash
+   *
    * @param path path
    * @return success
    * @throws IOException
@@ -282,7 +375,8 @@ public class HdfsApi {
 
   /**
    * Delete
-   * @param path path
+   *
+   * @param path      path
    * @param recursive delete recursive
    * @return success
    * @throws IOException
@@ -299,7 +393,8 @@ public class HdfsApi {
 
   /**
    * Create file
-   * @param path path
+   *
+   * @param path      path
    * @param overwrite overwrite existent file
    * @return output stream
    * @throws IOException
@@ -316,6 +411,7 @@ public class HdfsApi {
 
   /**
    * Open file
+   *
    * @param path path
    * @return input stream
    * @throws IOException
@@ -332,7 +428,8 @@ public class HdfsApi {
 
   /**
    * Change permissions
-   * @param path path
+   *
+   * @param path        path
    * @param permissions permissions in format rwxrwxrwx
    * @throws IOException
    * @throws InterruptedException
@@ -353,7 +450,8 @@ public class HdfsApi {
 
   /**
    * Copy file
-   * @param src source path
+   *
+   * @param src  source path
    * @param dest destination path
    * @throws java.io.IOException
    * @throws InterruptedException
@@ -380,8 +478,9 @@ public class HdfsApi {
 
   /**
    * Executes action on HDFS using doAs
+   *
    * @param action strategy object
-   * @param <T> result type
+   * @param <T>    result type
    * @return result of operation
    * @throws IOException
    * @throws InterruptedException
@@ -419,10 +518,9 @@ public class HdfsApi {
    * Converts a Hadoop permission into a Unix permission symbolic representation
    * (i.e. -rwxr--r--) or default if the permission is NULL.
    *
-   * @param p
-   *          Hadoop permission.
+   * @param p Hadoop permission.
    * @return the Unix permission symbolic representation or default if the
-   *         permission is NULL.
+   * permission is NULL.
    */
   private static String permissionToString(FsPermission p) {
     return (p == null) ? "default" : "-" + p.getUserAction().SYMBOL
@@ -435,8 +533,7 @@ public class HdfsApi {
    * specified URL.
    * <p/>
    *
-   * @param status
-   *          Hadoop file status.
+   * @param status Hadoop file status.
    * @return The JSON representation of the file status.
    */
   public Map<String, Object> fileStatusToJSON(FileStatus status) {
@@ -465,8 +562,7 @@ public class HdfsApi {
    * specified URL.
    * <p/>
    *
-   * @param status
-   *          Hadoop file status array.
+   * @param status Hadoop file status array.
    * @return The JSON representation of the file status array.
    */
   @SuppressWarnings("unchecked")

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6ecbd1d/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/HdfsApiTest.java
----------------------------------------------------------------------
diff --git a/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/HdfsApiTest.java
b/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/HdfsApiTest.java
new file mode 100644
index 0000000..e7a6752
--- /dev/null
+++ b/contrib/views/utils/src/test/java/org/apache/ambari/view/utils/hdfs/HdfsApiTest.java
@@ -0,0 +1,201 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.view.utils.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+
+public class HdfsApiTest {
+  private FileSystem fs;
+  private HdfsApi hdfsApi;
+  private Configuration conf;
+  private MiniDFSCluster hdfsCluster;
+
+  @Before
+  public void setup() throws IOException, HdfsApiException, InterruptedException {
+    File baseDir = new File("./target/hdfs/" + "HdfsApiTest.filterAndTruncateDirStatus").getAbsoluteFile();
+    FileUtil.fullyDelete(baseDir);
+
+    conf = new Configuration();
+    conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
+    MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
+    hdfsCluster = builder.build();
+    String hdfsURI = hdfsCluster.getURI() + "/";
+    conf.set("webhdfs.url", hdfsURI);
+    conf.set("fs.defaultFS", hdfsURI);
+    fs = FileSystem.get(conf);
+    hdfsApi = new HdfsApi(conf, fs, null);
+
+  }
+
+  @After
+  public void tearDown(){
+    hdfsCluster.shutdown();
+  }
+
+  @Test
+  public void filterAndTruncateDirStatus() throws Exception {
+    {
+      // null fileStatuses
+      DirStatus dirStatus = hdfsApi.filterAndTruncateDirStatus("", 0, null);
+      Assert.assertEquals(new DirStatus(null, new DirListInfo(0, false, 0, "")), dirStatus);
+    }
+
+    {
+      FileStatus[] fileStatuses = getFileStatuses(10);
+      DirStatus dirStatus1 = hdfsApi.filterAndTruncateDirStatus("", 0, fileStatuses);
+      Assert.assertEquals(new DirStatus(new FileStatus[0], new DirListInfo(10, true, 0, "")),
dirStatus1);
+    }
+
+    {
+      int originalSize = 10;
+      int maxAllowedSize = 5;
+      String nameFilter = "";
+      FileStatus[] fileStatuses = getFileStatuses(originalSize);
+      DirStatus dirStatus2 = hdfsApi.filterAndTruncateDirStatus(nameFilter, maxAllowedSize,
fileStatuses);
+      Assert.assertEquals(new DirStatus(Arrays.copyOf(fileStatuses, maxAllowedSize), new
DirListInfo(originalSize, true, maxAllowedSize, nameFilter)), dirStatus2);
+    }
+
+    {
+      int originalSize = 10;
+      int maxAllowedSize = 10;
+      String nameFilter = "";
+      FileStatus[] fileStatuses = getFileStatuses(originalSize);
+      DirStatus dirStatus2 = hdfsApi.filterAndTruncateDirStatus(nameFilter, maxAllowedSize,
fileStatuses);
+      Assert.assertEquals(new DirStatus(Arrays.copyOf(fileStatuses, maxAllowedSize), new
DirListInfo(originalSize, false, maxAllowedSize, nameFilter)), dirStatus2);
+    }
+
+    {
+      int originalSize = 11;
+      int maxAllowedSize = 2;
+      String nameFilter = "1";
+      FileStatus[] fileStatuses = getFileStatuses(originalSize);
+      DirStatus dirStatus = hdfsApi.filterAndTruncateDirStatus(nameFilter, maxAllowedSize,
fileStatuses);
+
+      Assert.assertEquals(new DirStatus(new FileStatus[]{fileStatuses[1], fileStatuses[10]},
new DirListInfo(originalSize, false, 2, nameFilter)), dirStatus);
+    }
+
+    {
+      int originalSize = 20;
+      int maxAllowedSize = 3;
+      String nameFilter = "1";
+      FileStatus[] fileStatuses = getFileStatuses(originalSize);
+      DirStatus dirStatus = hdfsApi.filterAndTruncateDirStatus(nameFilter, maxAllowedSize,
fileStatuses);
+
+      Assert.assertEquals(new DirStatus(new FileStatus[]{fileStatuses[1], fileStatuses[10],
fileStatuses[11]}, new DirListInfo(originalSize, true, 3, nameFilter)), dirStatus);
+    }
+
+    {
+      int originalSize = 12;
+      int maxAllowedSize = 3;
+      String nameFilter = "1";
+      FileStatus[] fileStatuses = getFileStatuses(originalSize);
+      DirStatus dirStatus = hdfsApi.filterAndTruncateDirStatus(nameFilter, maxAllowedSize,
fileStatuses);
+
+      Assert.assertEquals(new DirStatus(new FileStatus[]{fileStatuses[1], fileStatuses[10],
fileStatuses[11]}, new DirListInfo(originalSize, false, 3, nameFilter)), dirStatus);
+    }
+
+    {
+      int originalSize = 13;
+      int maxAllowedSize = 3;
+      String nameFilter = "1";
+      FileStatus[] fileStatuses = getFileStatuses(originalSize);
+      DirStatus dirStatus = hdfsApi.filterAndTruncateDirStatus(nameFilter, maxAllowedSize,
fileStatuses);
+
+      Assert.assertEquals(new DirStatus(new FileStatus[]{fileStatuses[1], fileStatuses[10],
fileStatuses[11]}, new DirListInfo(originalSize, true, 3, nameFilter)), dirStatus);
+    }
+
+    {
+      int originalSize = 0;
+      int maxAllowedSize = 3;
+      String nameFilter = "1";
+      FileStatus[] fileStatuses = getFileStatuses(originalSize);
+      DirStatus dirStatus = hdfsApi.filterAndTruncateDirStatus(nameFilter, maxAllowedSize,
fileStatuses);
+
+      Assert.assertEquals(new DirStatus(new FileStatus[0], new DirListInfo(originalSize,
false, originalSize, nameFilter)), dirStatus);
+    }
+
+    {
+      int originalSize = 20;
+      int maxAllowedSize = 3;
+      String nameFilter = "";
+      FileStatus[] fileStatuses = getFileStatuses(originalSize);
+      DirStatus dirStatus = hdfsApi.filterAndTruncateDirStatus(nameFilter, maxAllowedSize,
fileStatuses);
+
+      Assert.assertEquals(new DirStatus(new FileStatus[]{fileStatuses[0], fileStatuses[1],
fileStatuses[2]}, new DirListInfo(originalSize, true, maxAllowedSize, nameFilter)), dirStatus);
+    }
+
+    {
+      int originalSize = 20;
+      int maxAllowedSize = 3;
+      String nameFilter = null;
+      FileStatus[] fileStatuses = getFileStatuses(originalSize);
+      DirStatus dirStatus = hdfsApi.filterAndTruncateDirStatus(nameFilter, maxAllowedSize,
fileStatuses);
+
+      Assert.assertEquals(new DirStatus(new FileStatus[]{fileStatuses[0], fileStatuses[1],
fileStatuses[2]}, new DirListInfo(originalSize, true, maxAllowedSize, nameFilter)), dirStatus);
+    }
+
+    {
+      int originalSize = 3;
+      int maxAllowedSize = 3;
+      String nameFilter = null;
+      FileStatus[] fileStatuses = getFileStatuses(originalSize);
+      DirStatus dirStatus = hdfsApi.filterAndTruncateDirStatus(nameFilter, maxAllowedSize,
fileStatuses);
+
+      Assert.assertEquals(new DirStatus(new FileStatus[]{fileStatuses[0], fileStatuses[1],
fileStatuses[2]}, new DirListInfo(originalSize, false, maxAllowedSize, nameFilter)), dirStatus);
+    }
+
+    {
+      int originalSize = 20;
+      int maxAllowedSize = 3;
+      String nameFilter = "a";
+      FileStatus[] fileStatuses = getFileStatuses(originalSize);
+      DirStatus dirStatus = hdfsApi.filterAndTruncateDirStatus(nameFilter, maxAllowedSize,
fileStatuses);
+
+      Assert.assertEquals(new DirStatus(new FileStatus[0], new DirListInfo(originalSize,
false, 0, nameFilter)), dirStatus);
+    }
+
+  }
+
+  private FileStatus[] getFileStatuses(int numberOfFiles) {
+    FileStatus[] fileStatuses = new FileStatus[numberOfFiles];
+    for(int i = 0 ; i < numberOfFiles; i++){
+      fileStatuses[i] = getFileStatus("/"+i);
+    }
+
+    return fileStatuses;
+  }
+
+  private FileStatus getFileStatus(String path) {
+    return new FileStatus(10, false, 3, 1000, 10000, new Path(path));
+  }
+
+}
\ No newline at end of file


Mime
View raw message