hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rohithsharm...@apache.org
Subject hadoop git commit: YARN-7157. Add admin configuration to filter per-user's apps in secure cluster. Contributed by Sunil G.
Date Mon, 18 Sep 2017 13:13:30 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 4f8240bd8 -> 508e93e48


YARN-7157. Add admin configuration to filter per-user's apps in secure cluster. Contributed
by Sunil G.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/508e93e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/508e93e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/508e93e4

Branch: refs/heads/branch-3.0
Commit: 508e93e48c46a9d304a58cad8d9f59fd5763fa7c
Parents: 4f8240b
Author: Rohith Sharma K S <rohithsharmaks@apache.org>
Authored: Mon Sep 18 22:56:43 2017 +1000
Committer: Rohith Sharma K S <rohithsharmaks@apache.org>
Committed: Mon Sep 18 22:56:52 2017 +1000

----------------------------------------------------------------------
 .../hadoop/yarn/conf/YarnConfiguration.java     |  7 ++
 .../src/main/resources/yarn-default.xml         |  9 +++
 .../server/resourcemanager/ClientRMService.java | 19 ++++-
 .../resourcemanager/TestClientRMService.java    | 74 ++++++++++++++++++++
 4 files changed, 108 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/508e93e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index eb3afea..8b791f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3100,6 +3100,13 @@ public class YarnConfiguration extends Configuration {
   public static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS =
       NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX + "opts";
 
+  /*
+   * Support to view apps for given user in secure cluster.
+   */
+  public static final String DISPLAY_APPS_FOR_LOGGED_IN_USER =
+      RM_PREFIX + "display.per-user-apps";
+  public static final boolean DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER = false;
+
   // RM and NM CSRF props
   public static final String REST_CSRF = "webapp.rest-csrf.";
   public static final String RM_CSRF_PREFIX = RM_PREFIX + REST_CSRF;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/508e93e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index afde222..c2434e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3297,4 +3297,13 @@
     <value>false</value>
   </property>
 
+  <property>
+    <name>yarn.resourcemanager.display.per-user-apps</name>
+    <value>false</value>
+      <description>
+        Flag to enable display of applications per user as an admin
+        configuration.
+      </description>
+  </property>
+
 </configuration>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/508e93e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index df38893..d08005e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -207,6 +207,8 @@ public class ClientRMService extends AbstractService implements
   private ReservationSystem reservationSystem;
   private ReservationInputValidator rValidator;
 
+  private boolean displayPerUserApps = false;
+
   private static final EnumSet<RMAppState> ACTIVE_APP_STATES = EnumSet.of(
       RMAppState.ACCEPTED, RMAppState.RUNNING);
 
@@ -264,7 +266,11 @@ public class ClientRMService extends AbstractService implements
       }
       refreshServiceAcls(conf, RMPolicyProvider.getInstance());
     }
-    
+
+    this.displayPerUserApps  = conf.getBoolean(
+        YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER,
+        YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
+
     this.server.start();
     clientBindAddress = conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST,
                                                YarnConfiguration.RM_ADDRESS,
@@ -898,6 +904,12 @@ public class ClientRMService extends AbstractService implements
         continue;
       }
 
+      // Given RM is configured to display apps per user, skip apps to which
+      // this caller doesn't have access to view.
+      if (displayPerUserApps && !allowAccess) {
+        continue;
+      }
+
       reports.add(application.createAndGetApplicationReport(
           callerUGI.getUserName(), allowAccess));
     }
@@ -1766,4 +1778,9 @@ public class ClientRMService extends AbstractService implements
     return application;
   }
 
+  @VisibleForTesting
+  public void setDisplayPerUserApps(boolean displayPerUserApps) {
+    this.displayPerUserApps = displayPerUserApps;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/508e93e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index ea733a4..1e4bfbd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -1118,6 +1118,12 @@ public class TestClientRMService {
     assertEquals("Incorrect number of applications for user", 3,
         rmService.getApplications(request).getApplicationList().size());
 
+    rmService.setDisplayPerUserApps(true);
+    userSet.clear();
+    assertEquals("Incorrect number of applications for user", 6,
+        rmService.getApplications(request).getApplicationList().size());
+    rmService.setDisplayPerUserApps(false);
+
     // Check tags
     request = GetApplicationsRequest.newInstance(
         ApplicationsRequestScope.ALL, null, null, null, null, null, null,
@@ -2061,4 +2067,72 @@ public class TestClientRMService {
     rpc.stopProxy(client, conf);
     new File(excludeFile).delete();
   }
+
+  @Test
+  public void testGetApplicationsWithPerUserApps()
+      throws IOException, YarnException {
+    /*
+     * Submit 3 applications alternately in two queues
+     */
+    // Basic setup
+    YarnScheduler yarnScheduler = mockYarnScheduler();
+    RMContext rmContext = mock(RMContext.class);
+    mockRMContext(yarnScheduler, rmContext);
+    RMStateStore stateStore = mock(RMStateStore.class);
+    when(rmContext.getStateStore()).thenReturn(stateStore);
+    doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
+        .getRMTimelineCollectorManager();
+
+    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
+        mock(ApplicationACLsManager.class), new Configuration());
+    when(rmContext.getDispatcher().getEventHandler())
+        .thenReturn(new EventHandler<Event>() {
+          public void handle(Event event) {
+          }
+        });
+
+    // Simulate Queue ACL manager which returns false always
+    QueueACLsManager queueAclsManager = mock(QueueACLsManager.class);
+    when(queueAclsManager.checkAccess(any(UserGroupInformation.class),
+        any(QueueACL.class), any(RMApp.class), any(String.class),
+        anyListOf(String.class))).thenReturn(false);
+
+    // Simulate app ACL manager which returns false always
+    ApplicationACLsManager appAclsManager = mock(ApplicationACLsManager.class);
+    when(appAclsManager.checkAccess(eq(UserGroupInformation.getCurrentUser()),
+        any(ApplicationAccessType.class), any(String.class),
+        any(ApplicationId.class))).thenReturn(false);
+    ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
+        appManager, appAclsManager, queueAclsManager, null);
+    rmService.init(new Configuration());
+
+    // Initialize appnames and queues
+    String[] queues = {QUEUE_1, QUEUE_2};
+    String[] appNames = {MockApps.newAppName(), MockApps.newAppName(),
+        MockApps.newAppName()};
+    ApplicationId[] appIds = {getApplicationId(101), getApplicationId(102),
+        getApplicationId(103)};
+    List<String> tags = Arrays.asList("Tag1", "Tag2", "Tag3");
+
+    long[] submitTimeMillis = new long[3];
+    // Submit applications
+    for (int i = 0; i < appIds.length; i++) {
+      ApplicationId appId = appIds[i];
+      SubmitApplicationRequest submitRequest = mockSubmitAppRequest(appId,
+          appNames[i], queues[i % queues.length],
+          new HashSet<String>(tags.subList(0, i + 1)));
+      rmService.submitApplication(submitRequest);
+      submitTimeMillis[i] = System.currentTimeMillis();
+    }
+
+    // Test different cases of ClientRMService#getApplications()
+    GetApplicationsRequest request = GetApplicationsRequest.newInstance();
+    assertEquals("Incorrect total number of apps", 6,
+        rmService.getApplications(request).getApplicationList().size());
+
+    rmService.setDisplayPerUserApps(true);
+    assertEquals("Incorrect number of applications for user", 0,
+        rmService.getApplications(request).getApplicationList().size());
+    rmService.setDisplayPerUserApps(false);
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message