Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 3EDE0200D14 for ; Tue, 19 Sep 2017 00:12:02 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 3D9611609E2; Mon, 18 Sep 2017 22:12:02 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id AFCDA1609DF for ; Tue, 19 Sep 2017 00:12:00 +0200 (CEST) Received: (qmail 16405 invoked by uid 500); 18 Sep 2017 22:11:49 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 14577 invoked by uid 99); 18 Sep 2017 22:11:48 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 18 Sep 2017 22:11:47 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D80D7F5812; Mon, 18 Sep 2017 22:11:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Mon, 18 Sep 2017 22:11:55 -0000 Message-Id: <8848e81f34ac461c907a56747d0a39fb@git.apache.org> In-Reply-To: <4fca8be1525f4e5482cdcccee6204656@git.apache.org> References: <4fca8be1525f4e5482cdcccee6204656@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [12/50] [abbrv] hadoop git commit: Revert "HDFS-11156. Add new op GETFILEBLOCKLOCATIONS to WebHDFS REST API. Contributed by Weiwei Yang." archived-at: Mon, 18 Sep 2017 22:12:02 -0000 Revert "HDFS-11156. Add new op GETFILEBLOCKLOCATIONS to WebHDFS REST API. Contributed by Weiwei Yang." This reverts commit 7fcc73fc0d248aae1edbd4e1514c5818f6198928. Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/390c2b5d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/390c2b5d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/390c2b5d Branch: refs/heads/HDFS-7240 Commit: 390c2b5df0c4e25a2156d8daefd4670efe82b191 Parents: 2f86711 Author: Andrew Wang Authored: Thu Sep 14 15:12:26 2017 -0700 Committer: Andrew Wang Committed: Thu Sep 14 15:12:26 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 53 ---- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 62 +---- .../hadoop/hdfs/web/resources/GetOpParam.java | 12 +- .../web/resources/NamenodeWebHdfsMethods.java | 16 -- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 33 --- .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 188 +------------ .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 276 ------------------- 7 files changed, 6 insertions(+), 634 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/390c2b5d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index 7ec5fe5..dcd73bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectReader; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.ContentSummary.Builder; import org.apache.hadoop.fs.FileChecksum; @@ -645,56 +644,4 @@ class JsonUtilClient { } } - static BlockLocation[] toBlockLocationArray(Map json) - throws IOException{ - final Map rootmap = - (Map)json.get(BlockLocation.class.getSimpleName() + "s"); - final List array = JsonUtilClient.getList(rootmap, - BlockLocation.class.getSimpleName()); - - Preconditions.checkNotNull(array); - final BlockLocation[] locations = new BlockLocation[array.size()]; - int i = 0; - for (Object object : array) { - final Map m = (Map) object; - locations[i++] = JsonUtilClient.toBlockLocation(m); - } - return locations; - } - - /** Convert a Json map to BlockLocation. **/ - static BlockLocation toBlockLocation(Map m) - throws IOException{ - if(m == null) { - return null; - } - - long length = ((Number) m.get("length")).longValue(); - long offset = ((Number) m.get("offset")).longValue(); - boolean corrupt = Boolean. - getBoolean(m.get("corrupt").toString()); - String[] storageIds = toStringArray(getList(m, "storageIds")); - String[] cachedHosts = toStringArray(getList(m, "cachedHosts")); - String[] hosts = toStringArray(getList(m, "hosts")); - String[] names = toStringArray(getList(m, "names")); - String[] topologyPaths = toStringArray(getList(m, "topologyPaths")); - StorageType[] storageTypes = toStorageTypeArray( - getList(m, "storageTypes")); - return new BlockLocation(names, hosts, cachedHosts, - topologyPaths, storageIds, storageTypes, - offset, length, corrupt); - } - - static String[] toStringArray(List list) { - if (list == null) { - return null; - } else { - final String[] array = new String[list.size()]; - int i = 0; - for (Object object : list) { - array[i++] = object.toString(); - } - return array; - } - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/390c2b5d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 1159e50..ee8d5c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -1616,68 +1616,14 @@ public class WebHdfsFileSystem extends FileSystem final long offset, final long length) throws IOException { statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.GET_FILE_BLOCK_LOCATIONS); - BlockLocation[] locations = null; - try { - locations = getFileBlockLocations( - GetOpParam.Op.GETFILEBLOCKLOCATIONS, - p, offset, length); - } catch (RemoteException e) { - // See the error message from ExceptionHandle - if(e.getMessage() != null && - e.getMessage().contains( - "Invalid value for webhdfs parameter") && - e.getMessage().contains( - GetOpParam.Op.GETFILEBLOCKLOCATIONS.toString())) { - // Old webhdfs server doesn't support GETFILEBLOCKLOCATIONS - // operation, fall back to query again using old API - // GET_BLOCK_LOCATIONS. - LOG.info("Invalid webhdfs operation parameter " - + GetOpParam.Op.GETFILEBLOCKLOCATIONS + ". Fallback to use " - + GetOpParam.Op.GET_BLOCK_LOCATIONS + " instead."); - locations = getFileBlockLocations( - GetOpParam.Op.GET_BLOCK_LOCATIONS, - p, offset, length); - } - } - return locations; - } - /** - * Get file block locations implementation. Provide a operation - * parameter to determine how to get block locations from a webhdfs - * server. Older server only supports GET_BLOCK_LOCATIONS but - * not GETFILEBLOCKLOCATIONS. - * - * @param path path to the file - * @param offset start offset in the given file - * @param length of the file to get locations for - * @param operation - * Valid operation is either - * {@link org.apache.hadoop.hdfs.web.resources.GetOpParam.Op - * #GET_BLOCK_LOCATIONS} or - * {@link org.apache.hadoop.hdfs.web.resources.GetOpParam.Op - * #GET_BLOCK_LOCATIONS} - * @throws IOException - * Http connection error, decoding error or given - * operation is not valid - */ - @VisibleForTesting - protected BlockLocation[] getFileBlockLocations( - GetOpParam.Op operation, final Path path, - final long offset, final long length) throws IOException { - return new FsPathResponseRunner(operation, path, + final HttpOpParam.Op op = GetOpParam.Op.GET_BLOCK_LOCATIONS; + return new FsPathResponseRunner(op, p, new OffsetParam(offset), new LengthParam(length)) { @Override BlockLocation[] decodeResponse(Map json) throws IOException { - switch(operation) { - case GETFILEBLOCKLOCATIONS: - return JsonUtilClient.toBlockLocationArray(json); - case GET_BLOCK_LOCATIONS: - return DFSUtilClient.locatedBlocks2Locations( - JsonUtilClient.toLocatedBlocks(json)); - default : - throw new IOException("Unknown operation " + operation.name()); - } + return DFSUtilClient.locatedBlocks2Locations( + JsonUtilClient.toLocatedBlocks(json)); } }.run(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/390c2b5d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java index d32af33..6c2c674 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java @@ -33,18 +33,8 @@ public class GetOpParam extends HttpOpParam { GETHOMEDIRECTORY(false, HttpURLConnection.HTTP_OK), GETDELEGATIONTOKEN(false, HttpURLConnection.HTTP_OK, true), - /** - * GET_BLOCK_LOCATIONS is a private/stable API op. It returns a - * {@link org.apache.hadoop.hdfs.protocol.LocatedBlocks} - * json object. - */ + /** GET_BLOCK_LOCATIONS is a private unstable op. */ GET_BLOCK_LOCATIONS(false, HttpURLConnection.HTTP_OK), - /** - * GETFILEBLOCKLOCATIONS is the public op that complies with - * {@link org.apache.hadoop.fs.FileSystem#getFileBlockLocations} - * interface. - */ - GETFILEBLOCKLOCATIONS(false, HttpURLConnection.HTTP_OK), GETACLSTATUS(false, HttpURLConnection.HTTP_OK), GETXATTRS(false, HttpURLConnection.HTTP_OK), GETTRASHROOT(false, HttpURLConnection.HTTP_OK), http://git-wip-us.apache.org/repos/asf/hadoop/blob/390c2b5d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 139680c..e400847 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -54,7 +54,6 @@ import javax.ws.rs.core.StreamingOutput; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -993,21 +992,6 @@ public class NamenodeWebHdfsMethods { return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } } - case GETFILEBLOCKLOCATIONS: - { - final long offsetValue = offset.getValue(); - final Long lengthValue = length.getValue(); - - FileSystem fs = FileSystem.get(conf != null ? - conf : new Configuration()); - BlockLocation[] locations = fs.getFileBlockLocations( - new org.apache.hadoop.fs.Path(fullpath), - offsetValue, - lengthValue != null? lengthValue: Long.MAX_VALUE); - final String js = JsonUtil.toJsonString("BlockLocations", - JsonUtil.toJsonMap(locations)); - return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); - } case GET_BLOCK_LOCATIONS: { final long offsetValue = offset.getValue(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/390c2b5d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 94752f5..5752948 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -470,37 +470,4 @@ public class JsonUtil { public static String toJsonString(BlockStoragePolicy storagePolicy) { return toJsonString(BlockStoragePolicy.class, toJsonMap(storagePolicy)); } - - public static Map toJsonMap(BlockLocation[] locations) - throws IOException { - if(locations == null) { - return null; - } - final Map m = new TreeMap(); - Object[] blockLocations = new Object[locations.length]; - for(int i=0; i toJsonMap( - final BlockLocation blockLocation) throws IOException { - if (blockLocation == null) { - return null; - } - - final Map m = new TreeMap(); - m.put("length", blockLocation.getLength()); - m.put("offset", blockLocation.getOffset()); - m.put("corrupt", blockLocation.isCorrupt()); - m.put("storageTypes", toJsonArray(blockLocation.getStorageTypes())); - m.put("storageIds", blockLocation.getStorageIds()); - m.put("cachedHosts", blockLocation.getCachedHosts()); - m.put("hosts", blockLocation.getHosts()); - m.put("names", blockLocation.getNames()); - m.put("topologyPaths", blockLocation.getTopologyPaths()); - return m; - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/390c2b5d/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md index 03834eb..84e8a57 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md @@ -50,7 +50,6 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop * [`CHECKACCESS`](#Check_access) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).access) * [`GETALLSTORAGEPOLICY`](#Get_all_Storage_Policies) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getAllStoragePolicies) * [`GETSTORAGEPOLICY`](#Get_Storage_Policy) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getStoragePolicy) - * [`GETFILEBLOCKLOCATIONS`](#Get_File_Block_Locations) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileBlockLocations) * HTTP PUT * [`CREATE`](#Create_and_Write_to_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).create) * [`MKDIRS`](#Make_a_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).mkdirs) @@ -1069,7 +1068,7 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).unsetStor { "BlockStoragePolicy": { "copyOnCreateFile": false, - "creationFallbacks": [], + "creationFallbacks": [], "id":7, "name":"HOT", "replicationFallbacks":["ARCHIVE"], @@ -1079,51 +1078,6 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).unsetStor See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getStoragePolicy -### Get File Block Locations - -* Submit a HTTP GET request. - - curl -i "http://:/webhdfs/v1/?op=GETFILEBLOCKLOCATIONS - - The client receives a response with a [`BlockLocations` JSON Object](#Block_Locations_JSON_Schema): - - HTTP/1.1 200 OK - Content-Type: application/json - Transfer-Encoding: chunked - - { - "BlockLocations" : - { - "BlockLocation": - [ - { - "cachedHosts" : [], - "corrupt" : false, - "hosts" : ["host"], - "length" : 134217728, // length of this block - "names" : ["host:ip"], - "offset" : 0, // offset of the block in the file - "storageIds" : ["storageid"], - "storageTypes" : ["DISK"], // enum {RAM_DISK, SSD, DISK, ARCHIVE} - "topologyPaths" : ["/default-rack/hostname:ip"] - }, { - "cachedHosts" : [], - "corrupt" : false, - "hosts" : ["host"], - "length" : 62599364, - "names" : ["host:ip"], - "offset" : 134217728, - "storageIds" : ["storageid"], - "storageTypes" : ["DISK"], - "topologyPaths" : ["/default-rack/hostname:ip"] - }, - ... - ] - } - } - -See also: [`offset`](#Offset), [`length`](#Length), [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileBlockLocations - Extended Attributes(XAttrs) Operations -------------------------------------- @@ -2082,146 +2036,6 @@ A `BlockStoragePolicies` JSON object represents an array of `BlockStoragePolicy` } ``` -#### BlockLocations JSON Schema - -A `BlockLocations` JSON object represents an array of `BlockLocation` JSON objects. - -```json -{ - "name" : "BlockLocations", - "properties": - { - "BlockLocations": - { - "type" : "object", - "properties": - { - "BlockLocation": - { - "description": "An array of BlockLocation", - "type" : "array", - "items" : blockLocationProperties //See BlockLocation Properties - } - } - } - } -} -``` - -See also [`BlockLocation` Properties](#BlockLocation_Properties), [`GETFILEBLOCKLOCATIONS`](#Get_File_Block_Locations), [BlockLocation](../../api/org/apache/hadoop/fs/BlockLocation.html) - -### BlockLocation JSON Schema - -```json -{ - "name" : "BlockLocation", - "properties": - { - "BlockLocation": blockLocationProperties //See BlockLocation Properties - } -} -``` - -See also [`BlockLocation` Properties](#BlockLocation_Properties), [`GETFILEBLOCKLOCATIONS`](#Get_File_Block_Locations), [BlockLocation](../../api/org/apache/hadoop/fs/BlockLocation.html) - -#### BlockLocation Properties - -JavaScript syntax is used to define `blockLocationProperties` so that it can be referred in both `BlockLocation` and `BlockLocations` JSON schemas. - -```javascript -var blockLocationProperties = -{ - "type" : "object", - "properties": - { - "cachedHosts": - { - "description": "Datanode hostnames with a cached replica", - "type" : "array", - "required" : "true", - "items" : - { - "description": "A datanode hostname", - "type" : "string" - } - }, - "corrupt": - { - "description": "True if the block is corrupted", - "type" : "boolean", - "required" : "true" - }, - "hosts": - { - "description": "Datanode hostnames store the block", - "type" : "array", - "required" : "true", - "items" : - { - "description": "A datanode hostname", - "type" : "string" - } - }, - "length": - { - "description": "Length of the block", - "type" : "integer", - "required" : "true" - }, - "names": - { - "description": "Datanode IP:xferPort for accessing the block", - "type" : "array", - "required" : "true", - "items" : - { - "description": "DatanodeIP:xferPort", - "type" : "string" - } - }, - "offset": - { - "description": "Offset of the block in the file", - "type" : "integer", - "required" : "true" - }, - "storageIds": - { - "description": "Storage ID of each replica", - "type" : "array", - "required" : "true", - "items" : - { - "description": "Storage ID", - "type" : "string" - } - }, - "storageTypes": - { - "description": "Storage type of each replica", - "type" : "array", - "required" : "true", - "items" : - { - "description": "Storage type", - "enum" : ["RAM_DISK", "SSD", "DISK", "ARCHIVE"] - } - }, - "topologyPaths": - { - "description": "Datanode addresses in network topology", - "type" : "array", - "required" : "true", - "items" : - { - "description": "/rack/host:ip", - "type" : "string" - } - } - } -}; -``` - HTTP Query Parameter Dictionary ------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/390c2b5d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index 77e5be1..de051b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -29,7 +29,6 @@ import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.PrintWriter; import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.SocketException; @@ -39,16 +38,8 @@ import java.net.URISyntaxException; import java.net.URL; import java.security.PrivilegedExceptionAction; import java.util.Arrays; -import java.util.List; -import java.util.Map; import java.util.Random; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.core.MediaType; - import com.google.common.collect.ImmutableList; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; @@ -80,12 +71,8 @@ import org.apache.hadoop.hdfs.TestDFSClientRetries; import org.apache.hadoop.hdfs.TestFileCreation; import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies; -import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; @@ -96,8 +83,6 @@ import org.apache.hadoop.hdfs.web.resources.LengthParam; import org.apache.hadoop.hdfs.web.resources.NoRedirectParam; import org.apache.hadoop.hdfs.web.resources.OffsetParam; import org.apache.hadoop.hdfs.web.resources.Param; -import org.apache.hadoop.http.HttpServer2; -import org.apache.hadoop.http.HttpServerFunctionalTest; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryPolicy.RetryAction; import org.apache.hadoop.io.retry.RetryPolicy.RetryAction.RetryDecision; @@ -114,12 +99,8 @@ import org.junit.Assert; import org.junit.Test; import org.mockito.internal.util.reflection.Whitebox; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.type.MapType; - import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.anyLong; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.spy; @@ -974,76 +955,6 @@ public class TestWebHDFS { Assert.assertTrue(storageTypes != null && storageTypes.length > 0 && storageTypes[0] == StorageType.DISK); } - - // Query webhdfs REST API to get block locations - InetSocketAddress addr = cluster.getNameNode().getHttpAddress(); - - // Case 1 - // URL without length or offset parameters - URL url1 = new URL("http", addr.getHostString(), addr.getPort(), - WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS"); - LOG.info("Sending GETFILEBLOCKLOCATIONS request " + url1); - - String response1 = getResponse(url1, "GET"); - LOG.info("The output of GETFILEBLOCKLOCATIONS request " + response1); - // Parse BlockLocation array from json output using object mapper - BlockLocation[] locationArray1 = toBlockLocationArray(response1); - - // Verify the result from rest call is same as file system api - verifyEquals(locations, locationArray1); - - // Case 2 - // URL contains length and offset parameters - URL url2 = new URL("http", addr.getHostString(), addr.getPort(), - WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS" - + "&length=" + LENGTH + "&offset=" + OFFSET); - LOG.info("Sending GETFILEBLOCKLOCATIONS request " + url2); - - String response2 = getResponse(url2, "GET"); - LOG.info("The output of GETFILEBLOCKLOCATIONS request " + response2); - BlockLocation[] locationArray2 = toBlockLocationArray(response2); - - verifyEquals(locations, locationArray2); - - // Case 3 - // URL contains length parameter but without offset parameters - URL url3 = new URL("http", addr.getHostString(), addr.getPort(), - WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS" - + "&length=" + LENGTH); - LOG.info("Sending GETFILEBLOCKLOCATIONS request " + url3); - - String response3 = getResponse(url3, "GET"); - LOG.info("The output of GETFILEBLOCKLOCATIONS request " + response3); - BlockLocation[] locationArray3 = toBlockLocationArray(response3); - - verifyEquals(locations, locationArray3); - - // Case 4 - // URL contains offset parameter but without length parameter - URL url4 = new URL("http", addr.getHostString(), addr.getPort(), - WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS" - + "&offset=" + OFFSET); - LOG.info("Sending GETFILEBLOCKLOCATIONS request " + url4); - - String response4 = getResponse(url4, "GET"); - LOG.info("The output of GETFILEBLOCKLOCATIONS request " + response4); - BlockLocation[] locationArray4 = toBlockLocationArray(response4); - - verifyEquals(locations, locationArray4); - - // Case 5 - // URL specifies offset exceeds the file length - URL url5 = new URL("http", addr.getHostString(), addr.getPort(), - WebHdfsFileSystem.PATH_PREFIX + "/foo?op=GETFILEBLOCKLOCATIONS" - + "&offset=1200"); - LOG.info("Sending GETFILEBLOCKLOCATIONS request " + url5); - - String response5 = getResponse(url5, "GET"); - LOG.info("The output of GETFILEBLOCKLOCATIONS request " + response5); - BlockLocation[] locationArray5 = toBlockLocationArray(response5); - - // Expected an empty array of BlockLocation - verifyEquals(new BlockLocation[] {}, locationArray5); } finally { if (cluster != null) { cluster.shutdown(); @@ -1051,66 +962,6 @@ public class TestWebHDFS { } } - private BlockLocation[] toBlockLocationArray(String json) - throws IOException { - ObjectMapper mapper = new ObjectMapper(); - MapType subType = mapper.getTypeFactory().constructMapType( - Map.class, - String.class, - BlockLocation[].class); - MapType rootType = mapper.getTypeFactory().constructMapType( - Map.class, - mapper.constructType(String.class), - mapper.constructType(subType)); - - Map> jsonMap = mapper - .readValue(json, rootType); - Map locationMap = jsonMap - .get("BlockLocations"); - BlockLocation[] locationArray = locationMap.get( - BlockLocation.class.getSimpleName()); - return locationArray; - } - - private void verifyEquals(BlockLocation[] locations1, - BlockLocation[] locations2) throws IOException { - for(int i=0; i - * First time call it return a wrapped json response with a - * IllegalArgumentException - *

- * Second time call it return a valid GET_BLOCK_LOCATIONS - * json response - *

- * Third time call it return a wrapped json response with - * a random IOException - * - */ - public static class MockWebHdfsServlet extends HttpServlet { - - private static final long serialVersionUID = 1L; - private static int respondTimes = 0; - private static final String RANDOM_EXCEPTION_MSG = - "This is a random exception"; - - @Override - public void doGet(HttpServletRequest request, - HttpServletResponse response) throws ServletException, IOException { - response.setHeader("Content-Type", - MediaType.APPLICATION_JSON); - String param = request.getParameter("op"); - if(respondTimes == 0) { - Exception mockException = new IllegalArgumentException( - "Invalid value for webhdfs parameter \"op\". " - + "" + "No enum constant " + param); - sendException(request, response, mockException); - } else if (respondTimes == 1) { - sendResponse(request, response); - } else if (respondTimes == 2) { - Exception mockException = new IOException(RANDOM_EXCEPTION_MSG); - sendException(request, response, mockException); - } - respondTimes++; - } - - private void sendResponse(HttpServletRequest request, - HttpServletResponse response) throws IOException { - response.setStatus(HttpServletResponse.SC_OK); - // Construct a LocatedBlock for testing - DatanodeInfo d = DFSTestUtil.getLocalDatanodeInfo(); - DatanodeInfo[] ds = new DatanodeInfo[1]; - ds[0] = d; - ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 121, 1); - LocatedBlock l1 = new LocatedBlock(b1, ds); - l1.setStartOffset(0); - l1.setCorrupt(false); - List ls = Arrays.asList(l1); - LocatedBlocks locatedblocks = - new LocatedBlocks(10, false, ls, l1, - true, null, null); - - try (PrintWriter pw = response.getWriter()) { - pw.write(JsonUtil.toJsonString(locatedblocks)); - } - } - - private void sendException(HttpServletRequest request, - HttpServletResponse response, - Exception mockException) throws IOException { - response.setStatus(HttpServletResponse.SC_BAD_REQUEST); - String errJs = JsonUtil.toJsonString(mockException); - try (PrintWriter pw = response.getWriter()) { - pw.write(errJs); - } - } - } - - @Test - public void testGetFileBlockLocationsBackwardsCompatibility() - throws Exception { - final Configuration conf = WebHdfsTestUtil.createConf(); - final String pathSpec = WebHdfsFileSystem.PATH_PREFIX + "/*"; - HttpServer2 http = null; - try { - http = HttpServerFunctionalTest.createTestServer(conf); - http.addServlet("test", pathSpec, MockWebHdfsServlet.class); - http.start(); - - // Write the address back to configuration so - // WebHdfsFileSystem could connect to the mock server - conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, - "localhost:" + http.getConnectorAddress(0).getPort()); - - final WebHdfsFileSystem webFS = WebHdfsTestUtil.getWebHdfsFileSystem( - conf, WebHdfsConstants.WEBHDFS_SCHEME); - - WebHdfsFileSystem spyFs = spy(webFS); - BlockLocation[] locations = spyFs - .getFileBlockLocations(new Path("p"), 0, 100); - - // Verify result - assertEquals(1, locations.length); - assertEquals(121, locations[0].getLength()); - - // Verify the fall back - // The function should be called exactly 2 times - // 1st time handles GETFILEBLOCKLOCATIONS and found it is not supported - // 2nd time fall back to handle GET_FILE_BLOCK_LOCATIONS - verify(spyFs, times(2)).getFileBlockLocations(any(), - any(), anyLong(), anyLong()); - - // Verify it doesn't erroneously fall back - // When server returns a different error, it should directly - // throw an exception. - try { - spyFs.getFileBlockLocations(new Path("p"), 0, 100); - } catch (Exception e) { - assertTrue(e instanceof IOException); - assertEquals(e.getMessage(), MockWebHdfsServlet.RANDOM_EXCEPTION_MSG); - // Totally this function has been called 3 times - verify(spyFs, times(3)).getFileBlockLocations(any(), - any(), anyLong(), anyLong()); - } - } finally { - if(http != null) { - http.stop(); - } - } - } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org