Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 534DC18C81 for ; Wed, 7 Oct 2015 07:16:07 +0000 (UTC) Received: (qmail 65408 invoked by uid 500); 7 Oct 2015 07:16:04 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 65272 invoked by uid 500); 7 Oct 2015 07:16:04 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 64744 invoked by uid 99); 7 Oct 2015 07:16:04 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 07 Oct 2015 07:16:04 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id F20F3E0B51; Wed, 7 Oct 2015 07:16:03 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wheat9@apache.org To: common-commits@hadoop.apache.org Date: Wed, 07 Oct 2015 07:16:11 -0000 Message-Id: <156a7d5539764360be2940afab90fa38@git.apache.org> In-Reply-To: <573e33ce62b6453a90950917c77551d5@git.apache.org> References: <573e33ce62b6453a90950917c77551d5@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [09/19] hadoop git commit: HDFS-9170. Move libhdfs / fuse-dfs / libwebhdfs to hdfs-client. Contributed by Haohui Mai. http://git-wip-us.apache.org/repos/asf/hadoop/blob/960b19ed/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c deleted file mode 100644 index b082c08..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c +++ /dev/null @@ -1,402 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "hdfs_http_query.h" -#include -#include -#include -#include -#include - -#define PERM_STR_LEN 4 // "644" + one byte for NUL -#define SHORT_STR_LEN 6 // 65535 + NUL -#define LONG_STR_LEN 21 // 2^64-1 = 18446744073709551615 + NUL - -/** - * Create query based on NameNode hostname, - * NameNode port, path, operation and other parameters - * - * @param host NameNode hostName - * @param nnPort Port of NameNode - * @param path Absolute path for the corresponding file - * @param op Operations - * @param paraNum Number of remaining parameters - * @param paraNames Names of remaining parameters - * @param paraValues Values of remaining parameters - * @param url Holding the created URL - * @return 0 on success and non-zero value to indicate error - */ -static int createQueryURL(const char *host, unsigned int nnPort, - const char *path, const char *op, int paraNum, - const char **paraNames, const char **paraValues, - char **queryUrl) -{ - size_t length = 0; - int i = 0, offset = 0, ret = 0; - char *url = NULL; - const char *protocol = "http://"; - const char *prefix = "/webhdfs/v1"; - - if (!paraNames || !paraValues) { - return EINVAL; - } - length = strlen(protocol) + strlen(host) + strlen(":") + - SHORT_STR_LEN + strlen(prefix) + strlen(path) + - strlen ("?op=") + strlen(op); - for (i = 0; i < paraNum; i++) { - if (paraNames[i] && paraValues[i]) { - length += 2 + strlen(paraNames[i]) + strlen(paraValues[i]); - } - } - url = malloc(length); // The '\0' has already been included - // when using SHORT_STR_LEN - if (!url) { - return ENOMEM; - } - - offset = snprintf(url, length, "%s%s:%d%s%s?op=%s", - protocol, host, nnPort, prefix, path, op); - if (offset >= length || offset < 0) { - ret = EIO; - goto done; - } - for (i = 0; i < paraNum; i++) { - if (!paraNames[i] || !paraValues[i] || paraNames[i][0] == '\0' || - paraValues[i][0] == '\0') { - continue; - } - offset += snprintf(url + offset, length - offset, - "&%s=%s", paraNames[i], paraValues[i]); - if (offset >= length || offset < 0) { - ret = EIO; - goto done; - } - } -done: - if (ret) { - free(url); - return ret; - } - *queryUrl = url; - return 0; -} - -int createUrlForMKDIR(const char *host, int nnPort, - const char *path, const char *user, char **url) -{ - const char *userPara = "user.name"; - return createQueryURL(host, nnPort, path, "MKDIRS", 1, - &userPara, &user, url); -} - -int createUrlForGetFileStatus(const char *host, int nnPort, const char *path, - const char *user, char **url) -{ - const char *userPara = "user.name"; - return createQueryURL(host, nnPort, path, "GETFILESTATUS", 1, - &userPara, &user, url); -} - -int createUrlForLS(const char *host, int nnPort, const char *path, - const char *user, char **url) -{ - const char *userPara = "user.name"; - return createQueryURL(host, nnPort, path, "LISTSTATUS", - 1, &userPara, &user, url); -} - -int createUrlForNnAPPEND(const char *host, int nnPort, const char *path, - const char *user, char **url) -{ - const char *userPara = "user.name"; - return createQueryURL(host, nnPort, path, "APPEND", - 1, &userPara, &user, url); -} - -int createUrlForMKDIRwithMode(const char *host, int nnPort, const char *path, - int mode, const char *user, char **url) -{ - int strlength; - char permission[PERM_STR_LEN]; - const char *paraNames[2], *paraValues[2]; - - paraNames[0] = "permission"; - paraNames[1] = "user.name"; - memset(permission, 0, PERM_STR_LEN); - strlength = snprintf(permission, PERM_STR_LEN, "%o", mode); - if (strlength < 0 || strlength >= PERM_STR_LEN) { - return EIO; - } - paraValues[0] = permission; - paraValues[1] = user; - - return createQueryURL(host, nnPort, path, "MKDIRS", 2, - paraNames, paraValues, url); -} - -int createUrlForRENAME(const char *host, int nnPort, const char *srcpath, - const char *destpath, const char *user, char **url) -{ - const char *paraNames[2], *paraValues[2]; - paraNames[0] = "destination"; - paraNames[1] = "user.name"; - paraValues[0] = destpath; - paraValues[1] = user; - - return createQueryURL(host, nnPort, srcpath, - "RENAME", 2, paraNames, paraValues, url); -} - -int createUrlForCHMOD(const char *host, int nnPort, const char *path, - int mode, const char *user, char **url) -{ - int strlength; - char permission[PERM_STR_LEN]; - const char *paraNames[2], *paraValues[2]; - - paraNames[0] = "permission"; - paraNames[1] = "user.name"; - memset(permission, 0, PERM_STR_LEN); - strlength = snprintf(permission, PERM_STR_LEN, "%o", mode); - if (strlength < 0 || strlength >= PERM_STR_LEN) { - return EIO; - } - paraValues[0] = permission; - paraValues[1] = user; - - return createQueryURL(host, nnPort, path, "SETPERMISSION", - 2, paraNames, paraValues, url); -} - -int createUrlForDELETE(const char *host, int nnPort, const char *path, - int recursive, const char *user, char **url) -{ - const char *paraNames[2], *paraValues[2]; - paraNames[0] = "recursive"; - paraNames[1] = "user.name"; - if (recursive) { - paraValues[0] = "true"; - } else { - paraValues[0] = "false"; - } - paraValues[1] = user; - - return createQueryURL(host, nnPort, path, "DELETE", - 2, paraNames, paraValues, url); -} - -int createUrlForCHOWN(const char *host, int nnPort, const char *path, - const char *owner, const char *group, - const char *user, char **url) -{ - const char *paraNames[3], *paraValues[3]; - paraNames[0] = "owner"; - paraNames[1] = "group"; - paraNames[2] = "user.name"; - paraValues[0] = owner; - paraValues[1] = group; - paraValues[2] = user; - - return createQueryURL(host, nnPort, path, "SETOWNER", - 3, paraNames, paraValues, url); -} - -int createUrlForOPEN(const char *host, int nnPort, const char *path, - const char *user, size_t offset, size_t length, char **url) -{ - int strlength; - char offsetStr[LONG_STR_LEN], lengthStr[LONG_STR_LEN]; - const char *paraNames[3], *paraValues[3]; - - paraNames[0] = "offset"; - paraNames[1] = "length"; - paraNames[2] = "user.name"; - memset(offsetStr, 0, LONG_STR_LEN); - memset(lengthStr, 0, LONG_STR_LEN); - strlength = snprintf(offsetStr, LONG_STR_LEN, "%lu", offset); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - strlength = snprintf(lengthStr, LONG_STR_LEN, "%lu", length); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - paraValues[0] = offsetStr; - paraValues[1] = lengthStr; - paraValues[2] = user; - - return createQueryURL(host, nnPort, path, "OPEN", - 3, paraNames, paraValues, url); -} - -int createUrlForUTIMES(const char *host, int nnPort, const char *path, - long unsigned mTime, long unsigned aTime, - const char *user, char **url) -{ - int strlength; - char modTime[LONG_STR_LEN], acsTime[LONG_STR_LEN]; - const char *paraNames[3], *paraValues[3]; - - memset(modTime, 0, LONG_STR_LEN); - memset(acsTime, 0, LONG_STR_LEN); - strlength = snprintf(modTime, LONG_STR_LEN, "%lu", mTime); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - strlength = snprintf(acsTime, LONG_STR_LEN, "%lu", aTime); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - paraNames[0] = "modificationtime"; - paraNames[1] = "accesstime"; - paraNames[2] = "user.name"; - paraValues[0] = modTime; - paraValues[1] = acsTime; - paraValues[2] = user; - - return createQueryURL(host, nnPort, path, "SETTIMES", - 3, paraNames, paraValues, url); -} - -int createUrlForNnWRITE(const char *host, int nnPort, - const char *path, const char *user, - int16_t replication, size_t blockSize, char **url) -{ - int strlength; - char repStr[SHORT_STR_LEN], blockSizeStr[LONG_STR_LEN]; - const char *paraNames[4], *paraValues[4]; - - memset(repStr, 0, SHORT_STR_LEN); - memset(blockSizeStr, 0, LONG_STR_LEN); - if (replication > 0) { - strlength = snprintf(repStr, SHORT_STR_LEN, "%u", replication); - if (strlength < 0 || strlength >= SHORT_STR_LEN) { - return EIO; - } - } - if (blockSize > 0) { - strlength = snprintf(blockSizeStr, LONG_STR_LEN, "%lu", blockSize); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - } - paraNames[0] = "overwrite"; - paraNames[1] = "replication"; - paraNames[2] = "blocksize"; - paraNames[3] = "user.name"; - paraValues[0] = "true"; - paraValues[1] = repStr; - paraValues[2] = blockSizeStr; - paraValues[3] = user; - - return createQueryURL(host, nnPort, path, "CREATE", - 4, paraNames, paraValues, url); -} - -int createUrlForSETREPLICATION(const char *host, int nnPort, - const char *path, int16_t replication, - const char *user, char **url) -{ - char repStr[SHORT_STR_LEN]; - const char *paraNames[2], *paraValues[2]; - int strlength; - - memset(repStr, 0, SHORT_STR_LEN); - if (replication > 0) { - strlength = snprintf(repStr, SHORT_STR_LEN, "%u", replication); - if (strlength < 0 || strlength >= SHORT_STR_LEN) { - return EIO; - } - } - paraNames[0] = "replication"; - paraNames[1] = "user.name"; - paraValues[0] = repStr; - paraValues[1] = user; - - return createQueryURL(host, nnPort, path, "SETREPLICATION", - 2, paraNames, paraValues, url); -} - -int createUrlForGetBlockLocations(const char *host, int nnPort, - const char *path, size_t offset, - size_t length, const char *user, char **url) -{ - char offsetStr[LONG_STR_LEN], lengthStr[LONG_STR_LEN]; - const char *paraNames[3], *paraValues[3]; - int strlength; - - memset(offsetStr, 0, LONG_STR_LEN); - memset(lengthStr, 0, LONG_STR_LEN); - if (offset > 0) { - strlength = snprintf(offsetStr, LONG_STR_LEN, "%lu", offset); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - } - if (length > 0) { - strlength = snprintf(lengthStr, LONG_STR_LEN, "%lu", length); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - } - paraNames[0] = "offset"; - paraNames[1] = "length"; - paraNames[2] = "user.name"; - paraValues[0] = offsetStr; - paraValues[1] = lengthStr; - paraValues[2] = user; - - return createQueryURL(host, nnPort, path, "GET_BLOCK_LOCATIONS", - 3, paraNames, paraValues, url); -} - -int createUrlForReadFromDatanode(const char *dnHost, int dnPort, - const char *path, size_t offset, - size_t length, const char *user, - const char *namenodeRpcAddr, char **url) -{ - char offsetStr[LONG_STR_LEN], lengthStr[LONG_STR_LEN]; - const char *paraNames[4], *paraValues[4]; - int strlength; - - memset(offsetStr, 0, LONG_STR_LEN); - memset(lengthStr, 0, LONG_STR_LEN); - if (offset > 0) { - strlength = snprintf(offsetStr, LONG_STR_LEN, "%lu", offset); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - } - if (length > 0) { - strlength = snprintf(lengthStr, LONG_STR_LEN, "%lu", length); - if (strlength < 0 || strlength >= LONG_STR_LEN) { - return EIO; - } - } - - paraNames[0] = "offset"; - paraNames[1] = "length"; - paraNames[2] = "user.name"; - paraNames[3] = "namenoderpcaddress"; - paraValues[0] = offsetStr; - paraValues[1] = lengthStr; - paraValues[2] = user; - paraValues[3] = namenodeRpcAddr; - - return createQueryURL(dnHost, dnPort, path, "OPEN", - 4, paraNames, paraValues, url); -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/960b19ed/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h deleted file mode 100644 index 432797b..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#ifndef _HDFS_HTTP_QUERY_H_ -#define _HDFS_HTTP_QUERY_H_ - -#include /* for size_t */ -#include /* for int16_t */ - -/** - * Create the URL for a MKDIR request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the dir to create - * @param user User name - * @param url Holding the generated URL for MKDIR request - * @return 0 on success and non-zero value on errors - */ -int createUrlForMKDIR(const char *host, int nnPort, - const char *path, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a MKDIR (with mode) request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the dir to create - * @param mode Mode of MKDIR - * @param user User name - * @param url Holding the generated URL for MKDIR request - * @return 0 on success and non-zero value on errors - */ -int createUrlForMKDIRwithMode(const char *host, int nnPort, const char *path, - int mode, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a RENAME request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param srcpath Source path - * @param dstpath Destination path - * @param user User name - * @param url Holding the generated URL for RENAME request - * @return 0 on success and non-zero value on errors - */ -int createUrlForRENAME(const char *host, int nnPort, const char *srcpath, - const char *dstpath, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a CHMOD request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Target path - * @param mode New mode for the file - * @param user User name - * @param url Holding the generated URL for CHMOD request - * @return 0 on success and non-zero value on errors - */ -int createUrlForCHMOD(const char *host, int nnPort, const char *path, - int mode, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a GETFILESTATUS request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the target file - * @param user User name - * @param url Holding the generated URL for GETFILESTATUS request - * @return 0 on success and non-zero value on errors - */ -int createUrlForGetFileStatus(const char *host, int nnPort, - const char *path, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a LISTSTATUS request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the directory for listing - * @param user User name - * @param url Holding the generated URL for LISTSTATUS request - * @return 0 on success and non-zero value on errors - */ -int createUrlForLS(const char *host, int nnPort, - const char *path, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a DELETE request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the file to be deletected - * @param recursive Whether or not to delete in a recursive way - * @param user User name - * @param url Holding the generated URL for DELETE request - * @return 0 on success and non-zero value on errors - */ -int createUrlForDELETE(const char *host, int nnPort, const char *path, - int recursive, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a CHOWN request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the target - * @param owner New owner - * @param group New group - * @param user User name - * @param url Holding the generated URL for CHOWN request - * @return 0 on success and non-zero value on errors - */ -int createUrlForCHOWN(const char *host, int nnPort, const char *path, - const char *owner, const char *group, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a OPEN/READ request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the file to read - * @param user User name - * @param offset Offset for reading (the start position for this read) - * @param length Length of the file to read - * @param url Holding the generated URL for OPEN/READ request - * @return 0 on success and non-zero value on errors - */ -int createUrlForOPEN(const char *host, int nnPort, const char *path, - const char *user, size_t offset, size_t length, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a UTIMES (update time) request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the file for updating time - * @param mTime Modified time to set - * @param aTime Access time to set - * @param user User name - * @param url Holding the generated URL for UTIMES request - * @return 0 on success and non-zero value on errors - */ -int createUrlForUTIMES(const char *host, int nnPort, const char *path, - long unsigned mTime, long unsigned aTime, - const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a WRITE/CREATE request (sent to NameNode) - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the dir to create - * @param user User name - * @param replication Number of replication of the file - * @param blockSize Size of the block for the file - * @param url Holding the generated URL for WRITE request - * @return 0 on success and non-zero value on errors - */ -int createUrlForNnWRITE(const char *host, int nnPort, const char *path, - const char *user, int16_t replication, size_t blockSize, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for an APPEND request (sent to NameNode) - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the file for appending - * @param user User name - * @param url Holding the generated URL for APPEND request - * @return 0 on success and non-zero value on errors - */ -int createUrlForNnAPPEND(const char *host, int nnPort, - const char *path, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a SETREPLICATION request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the target file - * @param replication New replication number - * @param user User name - * @param url Holding the generated URL for SETREPLICATION request - * @return 0 on success and non-zero value on errors - */ -int createUrlForSETREPLICATION(const char *host, int nnPort, const char *path, - int16_t replication, const char *user, - char **url) __attribute__ ((warn_unused_result)); - -/** - * Create the URL for a GET_BLOCK_LOCATIONS request - * - * @param host The hostname of the NameNode - * @param nnPort Port of the NameNode - * @param path Path of the target file - * @param offset The offset in the file - * @param length Length of the file content - * @param user User name - * @param url Holding the generated URL for GET_BLOCK_LOCATIONS request - * @return 0 on success and non-zero value on errors - */ -int createUrlForGetBlockLocations(const char *host, int nnPort, - const char *path, size_t offset, - size_t length, const char *user, - char **url) __attribute__ ((warn_unused_result)); - - -#endif //_HDFS_HTTP_QUERY_H_ http://git-wip-us.apache.org/repos/asf/hadoop/blob/960b19ed/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c deleted file mode 100644 index 178fb9d..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c +++ /dev/null @@ -1,654 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "exception.h" -#include "hdfs.h" /* for hdfsFileInfo */ -#include "hdfs_json_parser.h" - -#include -#include -#include -#include - -static const char * const temporaryRedirectCode = "307 TEMPORARY_REDIRECT"; -static const char * const twoHundredOKCode = "200 OK"; -static const char * const twoHundredOneCreatedCode = "201 Created"; -static const char * const httpHeaderString = "HTTP/1.1"; - -/** - * Exception information after calling JSON operations - */ -struct jsonException { - const char *exception; - const char *javaClassName; - const char *message; -}; - -/** Print out the JSON exception information */ -static int printJsonExceptionV(struct jsonException *exc, int noPrintFlags, - const char *fmt, va_list ap) -{ - char *javaClassName = NULL; - int excErrno = EINTERNAL, shouldPrint = 0; - if (!exc) { - fprintf(stderr, "printJsonExceptionV: the jsonException is NULL\n"); - return EINTERNAL; - } - javaClassName = strdup(exc->javaClassName); - if (!javaClassName) { - fprintf(stderr, "printJsonExceptionV: internal out of memory error\n"); - return EINTERNAL; - } - getExceptionInfo(javaClassName, noPrintFlags, &excErrno, &shouldPrint); - free(javaClassName); - - if (shouldPrint) { - vfprintf(stderr, fmt, ap); - fprintf(stderr, " error:\n"); - fprintf(stderr, "Exception: %s\nJavaClassName: %s\nMessage: %s\n", - exc->exception, exc->javaClassName, exc->message); - } - - free(exc); - return excErrno; -} - -/** - * Print out JSON exception information. - * - * @param exc The exception information to print and free - * @param noPrintFlags Flags which determine which exceptions we should NOT - * print. - * @param fmt Printf-style format list - * @param ... Printf-style varargs - * - * @return The POSIX error number associated with the exception - * object. - */ -static int printJsonException(struct jsonException *exc, int noPrintFlags, - const char *fmt, ...) -{ - va_list ap; - int ret = 0; - - va_start(ap, fmt); - ret = printJsonExceptionV(exc, noPrintFlags, fmt, ap); - va_end(ap); - return ret; -} - -/** Parse the exception information from JSON */ -static struct jsonException *parseJsonException(json_t *jobj) -{ - const char *key = NULL; - json_t *value = NULL; - struct jsonException *exception = NULL; - void *iter = NULL; - - exception = calloc(1, sizeof(*exception)); - if (!exception) { - return NULL; - } - - iter = json_object_iter(jobj); - while (iter) { - key = json_object_iter_key(iter); - value = json_object_iter_value(iter); - - if (!strcmp(key, "exception")) { - exception->exception = json_string_value(value); - } else if (!strcmp(key, "javaClassName")) { - exception->javaClassName = json_string_value(value); - } else if (!strcmp(key, "message")) { - exception->message = json_string_value(value); - } - - iter = json_object_iter_next(jobj, iter); - } - return exception; -} - -/** - * Parse the exception information which is presented in JSON - * - * @param content Exception information in JSON - * @return jsonException for printing out - */ -static struct jsonException *parseException(const char *content) -{ - json_error_t error; - size_t flags = 0; - const char *key = NULL; - json_t *value; - json_t *jobj; - struct jsonException *exception = NULL; - - if (!content) { - return NULL; - } - jobj = json_loads(content, flags, &error); - if (!jobj) { - fprintf(stderr, "JSon parsing error: on line %d: %s\n", - error.line, error.text); - return NULL; - } - void *iter = json_object_iter(jobj); - while(iter) { - key = json_object_iter_key(iter); - value = json_object_iter_value(iter); - - if (!strcmp(key, "RemoteException") && - json_typeof(value) == JSON_OBJECT) { - exception = parseJsonException(value); - break; - } - iter = json_object_iter_next(jobj, iter); - } - - json_decref(jobj); - return exception; -} - -/** - * Parse the response information which uses TRUE/FALSE - * to indicate whether the operation succeeded - * - * @param response Response information - * @return 0 to indicate success - */ -static int parseBoolean(const char *response) -{ - json_t *root, *value; - json_error_t error; - size_t flags = 0; - int result = 0; - - root = json_loads(response, flags, &error); - if (!root) { - fprintf(stderr, "JSon parsing error: on line %d: %s\n", - error.line, error.text); - return EIO; - } - void *iter = json_object_iter(root); - value = json_object_iter_value(iter); - if (json_typeof(value) == JSON_TRUE) { - result = 0; - } else { - result = EIO; // FALSE means error in remote NN/DN - } - json_decref(root); - return result; -} - -int parseMKDIR(const char *response) -{ - return parseBoolean(response); -} - -int parseRENAME(const char *response) -{ - return parseBoolean(response); -} - -int parseDELETE(const char *response) -{ - return parseBoolean(response); -} - -int parseSETREPLICATION(const char *response) -{ - return parseBoolean(response); -} - -/** - * Check the header of response to see if it's 200 OK - * - * @param header Header information for checking - * @param content Stores exception information if there are errors - * @param operation Indicate the operation for exception printing - * @return 0 for success - */ -static int checkHeader(const char *header, const char *content, - const char *operation) -{ - char *result = NULL; - const char delims[] = ":"; - char *savepter; - int ret = 0; - - if (!header || strncmp(header, "HTTP/", strlen("HTTP/"))) { - return EINVAL; - } - if (!(strstr(header, twoHundredOKCode)) || - !(result = strstr(header, "Content-Length"))) { - struct jsonException *exc = parseException(content); - if (exc) { - ret = printJsonException(exc, PRINT_EXC_ALL, - "Calling WEBHDFS (%s)", operation); - } else { - ret = EIO; - } - return ret; - } - result = strtok_r(result, delims, &savepter); - result = strtok_r(NULL, delims, &savepter); - while (isspace(*result)) { - result++; - } - // Content-Length should be equal to 0, - // and the string should be "0\r\nServer" - if (strncmp(result, "0\r\n", 3)) { - ret = EIO; - } - return ret; -} - -int parseCHMOD(const char *header, const char *content) -{ - return checkHeader(header, content, "CHMOD"); -} - -int parseCHOWN(const char *header, const char *content) -{ - return checkHeader(header, content, "CHOWN"); -} - -int parseUTIMES(const char *header, const char *content) -{ - return checkHeader(header, content, "SETTIMES"); -} - -/** - * Check if the header contains correct information - * ("307 TEMPORARY_REDIRECT" and "Location") - * - * @param header Header for parsing - * @param content Contains exception information - * if the remote operation failed - * @param operation Specify the remote operation when printing out exception - * @return 0 for success - */ -static int checkRedirect(const char *header, - const char *content, const char *operation) -{ - const char *locTag = "Location"; - int ret = 0, offset = 0; - - // The header must start with "HTTP/1.1" - if (!header || strncmp(header, httpHeaderString, - strlen(httpHeaderString))) { - return EINVAL; - } - - offset += strlen(httpHeaderString); - while (isspace(header[offset])) { - offset++; - } - // Looking for "307 TEMPORARY_REDIRECT" in header - if (strncmp(header + offset, temporaryRedirectCode, - strlen(temporaryRedirectCode))) { - // Process possible exception information - struct jsonException *exc = parseException(content); - if (exc) { - ret = printJsonException(exc, PRINT_EXC_ALL, - "Calling WEBHDFS (%s)", operation); - } else { - ret = EIO; - } - return ret; - } - // Here we just simply check if header contains "Location" tag, - // detailed processing is in parseDnLoc - if (!(strstr(header, locTag))) { - ret = EIO; - } - return ret; -} - -int parseNnWRITE(const char *header, const char *content) -{ - return checkRedirect(header, content, "Write(NameNode)"); -} - -int parseNnAPPEND(const char *header, const char *content) -{ - return checkRedirect(header, content, "Append(NameNode)"); -} - -/** 0 for success , -1 for out of range, other values for error */ -int parseOPEN(const char *header, const char *content) -{ - int ret = 0, offset = 0; - - if (!header || strncmp(header, httpHeaderString, - strlen(httpHeaderString))) { - return EINVAL; - } - - offset += strlen(httpHeaderString); - while (isspace(header[offset])) { - offset++; - } - if (strncmp(header + offset, temporaryRedirectCode, - strlen(temporaryRedirectCode)) || - !strstr(header, twoHundredOKCode)) { - struct jsonException *exc = parseException(content); - if (exc) { - // If the exception is an IOException and it is because - // the offset is out of the range, do not print out the exception - if (!strcasecmp(exc->exception, "IOException") && - strstr(exc->message, "out of the range")) { - ret = -1; - } else { - ret = printJsonException(exc, PRINT_EXC_ALL, - "Calling WEBHDFS (OPEN)"); - } - } else { - ret = EIO; - } - } - return ret; -} - -int parseDnLoc(char *content, char **dn) -{ - char *url = NULL, *dnLocation = NULL, *savepter, *tempContent; - const char *prefix = "Location: http://"; - const char *prefixToRemove = "Location: "; - const char *delims = "\r\n"; - - tempContent = strdup(content); - if (!tempContent) { - return ENOMEM; - } - - dnLocation = strtok_r(tempContent, delims, &savepter); - while (dnLocation && strncmp(dnLocation, "Location:", - strlen("Location:"))) { - dnLocation = strtok_r(NULL, delims, &savepter); - } - if (!dnLocation) { - return EIO; - } - - while (isspace(*dnLocation)) { - dnLocation++; - } - if (strncmp(dnLocation, prefix, strlen(prefix))) { - return EIO; - } - url = strdup(dnLocation + strlen(prefixToRemove)); - if (!url) { - return ENOMEM; - } - *dn = url; - return 0; -} - -int parseDnWRITE(const char *header, const char *content) -{ - int ret = 0; - if (header == NULL || header[0] == '\0' || - strncmp(header, "HTTP/", strlen("HTTP/"))) { - return EINVAL; - } - if (!(strstr(header, twoHundredOneCreatedCode))) { - struct jsonException *exc = parseException(content); - if (exc) { - ret = printJsonException(exc, PRINT_EXC_ALL, - "Calling WEBHDFS (WRITE(DataNode))"); - } else { - ret = EIO; - } - } - return ret; -} - -int parseDnAPPEND(const char *header, const char *content) -{ - int ret = 0; - - if (header == NULL || header[0] == '\0' || - strncmp(header, "HTTP/", strlen("HTTP/"))) { - return EINVAL; - } - if (!(strstr(header, twoHundredOKCode))) { - struct jsonException *exc = parseException(content); - if (exc) { - ret = printJsonException(exc, PRINT_EXC_ALL, - "Calling WEBHDFS (APPEND(DataNode))"); - } else { - ret = EIO; - } - } - return ret; -} - -/** - * Retrieve file status from the JSON object - * - * @param jobj JSON object for parsing, which contains - * file status information - * @param fileStat hdfsFileInfo handle to hold file status information - * @return 0 on success - */ -static int parseJsonForFileStatus(json_t *jobj, hdfsFileInfo *fileStat) -{ - const char *key, *tempstr; - json_t *value; - void *iter = NULL; - - iter = json_object_iter(jobj); - while (iter) { - key = json_object_iter_key(iter); - value = json_object_iter_value(iter); - - if (!strcmp(key, "accessTime")) { - // json field contains time in milliseconds, - // hdfsFileInfo is counted in seconds - fileStat->mLastAccess = json_integer_value(value) / 1000; - } else if (!strcmp(key, "blockSize")) { - fileStat->mBlockSize = json_integer_value(value); - } else if (!strcmp(key, "length")) { - fileStat->mSize = json_integer_value(value); - } else if (!strcmp(key, "modificationTime")) { - fileStat->mLastMod = json_integer_value(value) / 1000; - } else if (!strcmp(key, "replication")) { - fileStat->mReplication = json_integer_value(value); - } else if (!strcmp(key, "group")) { - fileStat->mGroup = strdup(json_string_value(value)); - if (!fileStat->mGroup) { - return ENOMEM; - } - } else if (!strcmp(key, "owner")) { - fileStat->mOwner = strdup(json_string_value(value)); - if (!fileStat->mOwner) { - return ENOMEM; - } - } else if (!strcmp(key, "pathSuffix")) { - fileStat->mName = strdup(json_string_value(value)); - if (!fileStat->mName) { - return ENOMEM; - } - } else if (!strcmp(key, "permission")) { - tempstr = json_string_value(value); - fileStat->mPermissions = (short) strtol(tempstr, NULL, 8); - } else if (!strcmp(key, "type")) { - tempstr = json_string_value(value); - if (!strcmp(tempstr, "DIRECTORY")) { - fileStat->mKind = kObjectKindDirectory; - } else { - fileStat->mKind = kObjectKindFile; - } - } - // Go to the next key-value pair in the json object - iter = json_object_iter_next(jobj, iter); - } - return 0; -} - -int parseGFS(const char *response, hdfsFileInfo *fileStat, int printError) -{ - int ret = 0, printFlag; - json_error_t error; - size_t flags = 0; - json_t *jobj, *value; - const char *key; - void *iter = NULL; - - if (!response || !fileStat) { - return EIO; - } - jobj = json_loads(response, flags, &error); - if (!jobj) { - fprintf(stderr, "error while parsing json: on line %d: %s\n", - error.line, error.text); - return EIO; - } - iter = json_object_iter(jobj); - key = json_object_iter_key(iter); - value = json_object_iter_value(iter); - if (json_typeof(value) == JSON_OBJECT) { - if (!strcmp(key, "RemoteException")) { - struct jsonException *exception = parseJsonException(value); - if (exception) { - if (printError) { - printFlag = PRINT_EXC_ALL; - } else { - printFlag = NOPRINT_EXC_FILE_NOT_FOUND | - NOPRINT_EXC_ACCESS_CONTROL | - NOPRINT_EXC_PARENT_NOT_DIRECTORY; - } - ret = printJsonException(exception, printFlag, - "Calling WEBHDFS GETFILESTATUS"); - } else { - ret = EIO; - } - } else if (!strcmp(key, "FileStatus")) { - ret = parseJsonForFileStatus(value, fileStat); - } else { - ret = EIO; - } - - } else { - ret = EIO; - } - - json_decref(jobj); - return ret; -} - -/** - * Parse the JSON array. Called to parse the result of - * the LISTSTATUS operation. Thus each element of the JSON array is - * a JSON object with the information of a file entry contained - * in the folder. - * - * @param jobj The JSON array to be parsed - * @param fileStat The hdfsFileInfo handle used to - * store a group of file information - * @param numEntries Capture the number of files in the folder - * @return 0 for success - */ -static int parseJsonArrayForFileStatuses(json_t *jobj, hdfsFileInfo **fileStat, - int *numEntries) -{ - json_t *jvalue = NULL; - int i = 0, ret = 0, arraylen = 0; - hdfsFileInfo *fileInfo = NULL; - - arraylen = (int) json_array_size(jobj); - if (arraylen > 0) { - fileInfo = calloc(arraylen, sizeof(hdfsFileInfo)); - if (!fileInfo) { - return ENOMEM; - } - } - for (i = 0; i < arraylen; i++) { - //Getting the array element at position i - jvalue = json_array_get(jobj, i); - if (json_is_object(jvalue)) { - ret = parseJsonForFileStatus(jvalue, &fileInfo[i]); - if (ret) { - goto done; - } - } else { - ret = EIO; - goto done; - } - } -done: - if (ret) { - free(fileInfo); - } else { - *numEntries = arraylen; - *fileStat = fileInfo; - } - return ret; -} - -int parseLS(const char *response, hdfsFileInfo **fileStats, int *numOfEntries) -{ - int ret = 0; - json_error_t error; - size_t flags = 0; - json_t *jobj, *value; - const char *key; - void *iter = NULL; - - if (!response || response[0] == '\0' || !fileStats) { - return EIO; - } - jobj = json_loads(response, flags, &error); - if (!jobj) { - fprintf(stderr, "error while parsing json: on line %d: %s\n", - error.line, error.text); - return EIO; - } - - iter = json_object_iter(jobj); - key = json_object_iter_key(iter); - value = json_object_iter_value(iter); - if (json_typeof(value) == JSON_OBJECT) { - if (!strcmp(key, "RemoteException")) { - struct jsonException *exception = parseJsonException(value); - if (exception) { - ret = printJsonException(exception, PRINT_EXC_ALL, - "Calling WEBHDFS GETFILESTATUS"); - } else { - ret = EIO; - } - } else if (!strcmp(key, "FileStatuses")) { - iter = json_object_iter(value); - value = json_object_iter_value(iter); - if (json_is_array(value)) { - ret = parseJsonArrayForFileStatuses(value, fileStats, - numOfEntries); - } else { - ret = EIO; - } - } else { - ret = EIO; - } - } else { - ret = EIO; - } - - json_decref(jobj); - return ret; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/960b19ed/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.h b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.h deleted file mode 100644 index c5f2f9c..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.h +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef _HDFS_JSON_PARSER_H_ -#define _HDFS_JSON_PARSER_H_ - -/** - * Parse the response for MKDIR request. The response uses TRUE/FALSE - * to indicate whether the operation succeeded. - * - * @param response The response information to parse. - * @return 0 for success - */ -int parseMKDIR(const char *response); - -/** - * Parse the response for RENAME request. The response uses TRUE/FALSE - * to indicate whether the operation succeeded. - * - * @param response The response information to parse. - * @return 0 for success - */ -int parseRENAME(const char *response); - -/** - * Parse the response for DELETE request. The response uses TRUE/FALSE - * to indicate whether the operation succeeded. - * - * @param response The response information to parse. - * @return 0 for success - */ -int parseDELETE(const char *response); - -/** - * Parse the response for SETREPLICATION request. The response uses TRUE/FALSE - * to indicate whether the operation succeeded. - * - * @param response The response information to parse. - * @return 0 for success - */ -int parseSETREPLICATION(const char *response); - -/** - * Parse the response for OPEN (read) request. A successful operation - * will return "200 OK". - * - * @param response The response information for parsing - * @return 0 for success , -1 for out of range, other values for error - */ -int parseOPEN(const char *header, const char *content); - -/** - * Parse the response for WRITE (from NameNode) request. - * A successful operation should return "307 TEMPORARY_REDIRECT" in its header. - * - * @param header The header of the http response - * @param content If failing, the exception message - * sent from NameNode is stored in content - * @return 0 for success - */ -int parseNnWRITE(const char *header, const char *content); - -/** - * Parse the response for WRITE (from DataNode) request. - * A successful operation should return "201 Created" in its header. - * - * @param header The header of the http response - * @param content If failing, the exception message - * sent from DataNode is stored in content - * @return 0 for success - */ -int parseDnWRITE(const char *header, const char *content); - -/** - * Parse the response for APPEND (sent from NameNode) request. - * A successful operation should return "307 TEMPORARY_REDIRECT" in its header. - * - * @param header The header of the http response - * @param content If failing, the exception message - * sent from NameNode is stored in content - * @return 0 for success - */ -int parseNnAPPEND(const char *header, const char *content); - -/** - * Parse the response for APPEND (from DataNode) request. - * A successful operation should return "200 OK" in its header. - * - * @param header The header of the http response - * @param content If failing, the exception message - * sent from DataNode is stored in content - * @return 0 for success - */ -int parseDnAPPEND(const char *header, const char *content); - -/** - * Parse the response (from NameNode) to get the location information - * of the DataNode that should be contacted for the following write operation. - * - * @param content Content of the http header - * @param dn To store the location of the DataNode for writing - * @return 0 for success - */ -int parseDnLoc(char *content, char **dn) __attribute__ ((warn_unused_result)); - -/** - * Parse the response for GETFILESTATUS operation. - * - * @param response Response to parse. Its detailed format is specified in - * "http://hadoop.apache.org/docs/stable/webhdfs.html#GETFILESTATUS" - * @param fileStat A hdfsFileInfo handle for holding file information - * @param printError Whether or not print out exception - * when file does not exist - * @return 0 for success, non-zero value to indicate error - */ -int parseGFS(const char *response, hdfsFileInfo *fileStat, int printError); - -/** - * Parse the response for LISTSTATUS operation. - * - * @param response Response to parse. Its detailed format is specified in - * "http://hadoop.apache.org/docs/r1.0.3/webhdfs.html#LISTSTATUS" - * @param fileStats Pointer pointing to a list of hdfsFileInfo handles - * holding file/dir information in the directory - * @param numEntries After parsing, the value of this parameter indicates - * the number of file entries. - * @return 0 for success, non-zero value to indicate error - */ -int parseLS(const char *response, hdfsFileInfo **fileStats, int *numOfEntries); - -/** - * Parse the response for CHOWN request. - * A successful operation should contains "200 OK" in its header, - * and the Content-Length should be 0. - * - * @param header The header of the http response - * @param content If failing, the exception message is stored in content - * @return 0 for success - */ -int parseCHOWN(const char *header, const char *content); - -/** - * Parse the response for CHMOD request. - * A successful operation should contains "200 OK" in its header, - * and the Content-Length should be 0. - * - * @param header The header of the http response - * @param content If failing, the exception message is stored in content - * @return 0 for success - */ -int parseCHMOD(const char *header, const char *content); - -/** - * Parse the response for SETTIMES request. - * A successful operation should contains "200 OK" in its header, - * and the Content-Length should be 0. - * - * @param header The header of the http response - * @param content If failing, the exception message is stored in content - * @return 0 for success - */ -int parseUTIMES(const char *header, const char *content); - -#endif //_HDFS_JSON_PARSER_H_