hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From snem...@apache.org
Subject [hadoop] branch branch-3.1 updated: Revert "YARN-9128. Use SerializationUtils from apache commons to serialize / deserialize ResourceMappings. Contributed by Zoltan Siegl"
Date Wed, 09 Oct 2019 17:59:16 GMT
This is an automated email from the ASF dual-hosted git repository.

snemeth pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.1 by this push:
     new aabc18a  Revert "YARN-9128. Use SerializationUtils from apache commons to serialize
/ deserialize ResourceMappings. Contributed by Zoltan Siegl"
aabc18a is described below

commit aabc18a777d43694a4c0c0ef983a72857ec7cdbc
Author: Szilard Nemeth <snemeth@apache.org>
AuthorDate: Wed Oct 9 19:59:02 2019 +0200

    Revert "YARN-9128. Use SerializationUtils from apache commons to serialize / deserialize
ResourceMappings. Contributed by Zoltan Siegl"
    
    This reverts commit 73bc8ef9b86cb9b3ee9bbc1786b49600e0ee600d.
---
 .../container/ResourceMappings.java                |  32 ++++--
 .../container/TestResourceMappings.java            | 118 ---------------------
 2 files changed, 22 insertions(+), 128 deletions(-)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ResourceMappings.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ResourceMappings.java
index c1c3b5d..d673341 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ResourceMappings.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ResourceMappings.java
@@ -18,10 +18,11 @@
 
 package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
 
-import org.apache.commons.lang3.SerializationException;
-import org.apache.commons.lang3.SerializationUtils;
-
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -29,6 +30,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.io.IOUtils;
+
 /**
  * This class is used to store assigned resource to a single container by
  * resource types.
@@ -88,11 +91,16 @@ public class ResourceMappings {
     @SuppressWarnings("unchecked")
     public static AssignedResources fromBytes(byte[] bytes)
         throws IOException {
-      final List<Serializable> resources;
+      ObjectInputStream ois = null;
+      List<Serializable> resources;
       try {
-        resources = SerializationUtils.deserialize(bytes);
-      } catch (SerializationException e) {
+        ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
+        ois = new ObjectInputStream(bis);
+        resources = (List<Serializable>) ois.readObject();
+      } catch (ClassNotFoundException e) {
         throw new IOException(e);
+      } finally {
+        IOUtils.closeQuietly(ois);
       }
       AssignedResources ar = new AssignedResources();
       ar.updateAssignedResources(resources);
@@ -100,11 +108,15 @@ public class ResourceMappings {
     }
 
     public byte[] toBytes() throws IOException {
-      final byte[] bytes;
+      ObjectOutputStream oos = null;
+      byte[] bytes;
       try {
-        bytes = SerializationUtils.serialize((Serializable) resources);
-      } catch (SerializationException e) {
-        throw new IOException(e);
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        oos = new ObjectOutputStream(bos);
+        oos.writeObject(resources);
+        bytes = bos.toByteArray();
+      } finally {
+        IOUtils.closeQuietly(oos);
       }
       return bytes;
     }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestResourceMappings.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestResourceMappings.java
deleted file mode 100644
index 561ce0c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestResourceMappings.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-import java.util.List;
-
-public class TestResourceMappings {
-
-  private static final ResourceMappings.AssignedResources testResources =
-      new ResourceMappings.AssignedResources();
-
-  @BeforeClass
-  public static void setup() {
-    testResources.updateAssignedResources(ImmutableList.of(
-        Device.Builder.newInstance()
-            .setId(0)
-            .setDevPath("/dev/hdwA0")
-            .setMajorNumber(256)
-            .setMinorNumber(0)
-            .setBusID("0000:80:00.0")
-            .setHealthy(true)
-            .build(),
-        Device.Builder.newInstance()
-            .setId(1)
-            .setDevPath("/dev/hdwA1")
-            .setMajorNumber(256)
-            .setMinorNumber(0)
-            .setBusID("0000:80:00.1")
-            .setHealthy(true)
-            .build()
-    ));
-  }
-
-  @Test
-  public void testSerializeAssignedResourcesWithSerializationUtils() {
-    try {
-      byte[] serializedString = testResources.toBytes();
-
-      ResourceMappings.AssignedResources deserialized =
-          ResourceMappings.AssignedResources.fromBytes(serializedString);
-
-      Assert.assertEquals(testResources.getAssignedResources(),
-          deserialized.getAssignedResources());
-
-    } catch (IOException e) {
-      e.printStackTrace();
-      Assert.fail(String.format("Serialization of test AssignedResources " +
-          "failed with %s", e.getMessage()));
-    }
-  }
-
-  @Test
-  public void testAssignedResourcesCanDeserializePreviouslySerializedValues() {
-    try {
-      byte[] serializedString = toBytes(testResources.getAssignedResources());
-
-      ResourceMappings.AssignedResources deserialized =
-          ResourceMappings.AssignedResources.fromBytes(serializedString);
-
-      Assert.assertEquals(testResources.getAssignedResources(),
-          deserialized.getAssignedResources());
-
-    } catch (IOException e) {
-      e.printStackTrace();
-      Assert.fail(String.format("Deserialization of test AssignedResources " +
-          "failed with %s", e.getMessage()));
-    }
-  }
-
-  /**
-   * This was the legacy way to serialize resources. This is here for
-   * backward compatibility to ensure that after YARN-9128 we can still
-   * deserialize previously serialized resources.
-   *
-   * @param resources the list of resources
-   * @return byte array representation of the resource
-   * @throws IOException
-   */
-  private byte[] toBytes(List<Serializable> resources) throws IOException {
-    ObjectOutputStream oos = null;
-    byte[] bytes;
-    try {
-      ByteArrayOutputStream bos = new ByteArrayOutputStream();
-      oos = new ObjectOutputStream(bos);
-      oos.writeObject(resources);
-      bytes = bos.toByteArray();
-    } finally {
-      IOUtils.closeQuietly(oos);
-    }
-    return bytes;
-  }
-}
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message