hive-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Work logged] (HIVE-21909) Publish the LLAP Daemon capacity through ZooKeeper and honor the capacity when scheduling new tasks
Date Wed, 03 Jul 2019 14:59:01 GMT

     [ https://issues.apache.org/jira/browse/HIVE-21909?focusedWorklogId=271672&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-271672
]

ASF GitHub Bot logged work on HIVE-21909:
-----------------------------------------

                Author: ASF GitHub Bot
            Created on: 03/Jul/19 14:58
            Start Date: 03/Jul/19 14:58
    Worklog Time Spent: 10m 
      Work Description: asinkovits commented on pull request #695: HIVE-21909: Publish the
LLAP Daemon capacity through ZooKeeper and ho…
URL: https://github.com/apache/hive/pull/695#discussion_r300005114
 
 

 ##########
 File path: llap-client/src/test/org/apache/hadoop/hive/llap/registry/impl/TestLlapZookeeperRegistryImpl.java
 ##########
 @@ -0,0 +1,124 @@
+package org.apache.hadoop.hive.llap.registry.impl;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryOneTime;
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.registry.LlapServiceInstance;
+import org.apache.hadoop.hive.registry.ServiceInstanceSet;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.MockitoAnnotations;
+import org.mockito.Spy;
+import org.mockito.internal.util.reflection.Fields;
+import org.mockito.internal.util.reflection.InstanceField;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static java.lang.Integer.parseInt;
+import static org.junit.Assert.assertEquals;
+
+public class TestLlapZookeeperRegistryImpl {
+
+  @Spy
+  private HiveConf mockConf = new HiveConf();
+
+  private LlapZookeeperRegistryImpl registry;
+
+  private CuratorFramework curatorFramework;
+  private TestingServer server;
+
+  @Before
+  public void setUp() throws Exception {
+    MockitoAnnotations.initMocks(this);
+    registry = new LlapZookeeperRegistryImpl("TestLlapZookeeperRegistryImpl", mockConf);
+
+    server = new TestingServer();
+    server.start();
+
+    curatorFramework = CuratorFrameworkFactory.
+            builder().
+            connectString(server.getConnectString()).
+            sessionTimeoutMs(1000).
+            retryPolicy(new RetryOneTime(1000)).
+            build();
+    curatorFramework.start();
+
+    trySetMock(registry, CuratorFramework.class, curatorFramework);
+
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    curatorFramework.close();
+    server.stop();
+  }
+
+  @Test
+  public void testRegister() throws Exception {
+    // Given
+    int expectedExecutorCount = HiveConf.getIntVar(mockConf, HiveConf.ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
+    int expectedQueueSize = HiveConf.getIntVar(mockConf, HiveConf.ConfVars.LLAP_DAEMON_TASK_SCHEDULER_WAIT_QUEUE_SIZE);
+
+    // When
+    registry.register();
+    ServiceInstanceSet<LlapServiceInstance> serviceInstanceSet =
+      registry.getInstances("LLAP", 1000);
+
+    // Then
+    Collection<LlapServiceInstance> llaps = serviceInstanceSet.getAll();
+    assertEquals(1, llaps.size());
+    LlapServiceInstance serviceInstance = llaps.iterator().next();
+    Map<String, String> attributes = serviceInstance.getProperties();
+
+    assertEquals(expectedQueueSize,
+      parseInt(attributes.get(LlapRegistryService.LLAP_DAEMON_TASK_SCHEDULER_ENABLED_WAIT_QUEUE_SIZE)));
+    assertEquals(expectedExecutorCount,
+      parseInt(attributes.get(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS)));
+  }
+
+  @Test
+  public void testUpdate() throws Exception {
+    // Given
+    String expectedExecutorCount = "2";
+    String expectedQueueSize = "20";
+    Map<String, String> capacityValues = new HashMap<>(2);
+    capacityValues.put(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS, expectedExecutorCount);
+    capacityValues.put(LlapRegistryService.LLAP_DAEMON_TASK_SCHEDULER_ENABLED_WAIT_QUEUE_SIZE,
expectedQueueSize);
+
+    // When
+    registry.register();
+    registry.updateRegistration(capacityValues.entrySet());
+    ServiceInstanceSet<LlapServiceInstance> serviceInstanceSet =
+            registry.getInstances("LLAP", 1000);
+
+    // Then
+    Collection<LlapServiceInstance> llaps = serviceInstanceSet.getAll();
+    assertEquals(1, llaps.size());
+    LlapServiceInstance serviceInstance = llaps.iterator().next();
+    Map<String, String> attributes = serviceInstance.getProperties();
+
+    assertEquals(expectedQueueSize,
+            attributes.get(LlapRegistryService.LLAP_DAEMON_TASK_SCHEDULER_ENABLED_WAIT_QUEUE_SIZE));
+    assertEquals(expectedExecutorCount,
+            attributes.get(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS));
+  }
+
+  static <T> void trySetMock(Object o, Class<T> clazz, T mock) {
+    List<InstanceField> instanceFields = Fields
+      .allDeclaredFieldsOf(o)
+      .filter(instanceField -> !clazz.isAssignableFrom(instanceField.jdkField().getType()))
+      .instanceFields();
+    if (instanceFields.size() != 1) {
 
 Review comment:
   It would be easier, but I think this way it's more robust.
 
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Issue Time Tracking
-------------------

    Worklog Id:     (was: 271672)
    Time Spent: 1h 40m  (was: 1.5h)

> Publish the LLAP Daemon capacity through ZooKeeper and honor the capacity when scheduling
new tasks
> ---------------------------------------------------------------------------------------------------
>
>                 Key: HIVE-21909
>                 URL: https://issues.apache.org/jira/browse/HIVE-21909
>             Project: Hive
>          Issue Type: Sub-task
>          Components: llap
>            Reporter: Peter Vary
>            Assignee: Antal Sinkovits
>            Priority: Major
>              Labels: pull-request-available
>         Attachments: HIVE-21909.01.patch
>
>          Time Spent: 1h 40m
>  Remaining Estimate: 0h
>
> The changes in LlapDaemonNode status (or executor number) should be reflected in the
ZooKeeper information, and we should make sure TezAMs are reading and honor the changes.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message