Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E101FD80C for ; Thu, 8 Nov 2012 19:10:50 +0000 (UTC) Received: (qmail 74126 invoked by uid 500); 8 Nov 2012 19:10:50 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 74080 invoked by uid 500); 8 Nov 2012 19:10:50 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 74069 invoked by uid 99); 8 Nov 2012 19:10:50 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 08 Nov 2012 19:10:50 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 08 Nov 2012 19:10:47 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 6711E2388B9B; Thu, 8 Nov 2012 19:10:10 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1407217 [2/7] - in /hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs: ./ src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/ src/contrib/bkjournal/src/main/proto/ src/contrib/bkjournal/src/test/j... Date: Thu, 08 Nov 2012 19:10:04 -0000 To: hdfs-commits@hadoop.apache.org From: suresh@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121108191010.6711E2388B9B@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c?rev=1407217&r1=1407216&r2=1407217&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.c Thu Nov 8 19:09:46 2012 @@ -22,233 +22,381 @@ #include #include -#define NUM_OF_PERMISSION_BITS 4 -#define NUM_OF_PORT_BITS 6 -#define NUM_OF_REPLICATION_BITS 6 - -static char *prepareQUERY(const char *host, int nnPort, const char *srcpath, const char *OP, const char *user) { - size_t length; - char *url; - const char *const protocol = "http://"; - const char *const prefix = "/webhdfs/v1"; - char *temp; - char *port; - port= (char*) malloc(NUM_OF_PORT_BITS); - if (!port) { - return NULL; - } - sprintf(port,"%d",nnPort); - if (user != NULL) { - length = strlen(protocol) + strlen(host) + strlen(":") + strlen(port) + strlen(prefix) + strlen(srcpath) + strlen ("?op=") + strlen(OP) + strlen("&user.name=") + strlen(user); - } else { - length = strlen(protocol) + strlen(host) + strlen(":") + strlen(port) + strlen(prefix) + strlen(srcpath) + strlen ("?op=") + strlen(OP); +#define PERM_STR_LEN 4 // "644" + one byte for NUL +#define SHORT_STR_LEN 6 // 65535 + NUL +#define LONG_STR_LEN 21 // 2^64-1 = 18446744073709551615 + NUL + +/** + * Create query based on NameNode hostname, + * NameNode port, path, operation and other parameters + * + * @param host NameNode hostName + * @param nnPort Port of NameNode + * @param path Absolute path for the corresponding file + * @param op Operations + * @param paraNum Number of remaining parameters + * @param paraNames Names of remaining parameters + * @param paraValues Values of remaining parameters + * @param url Holding the created URL + * @return 0 on success and non-zero value to indicate error + */ +static int createQueryURL(const char *host, unsigned int nnPort, + const char *path, const char *op, int paraNum, + const char **paraNames, const char **paraValues, + char **queryUrl) +{ + size_t length = 0; + int i = 0, offset = 0, ret = 0; + char *url = NULL; + const char *protocol = "http://"; + const char *prefix = "/webhdfs/v1"; + + if (!paraNames || !paraValues) { + return EINVAL; + } + length = strlen(protocol) + strlen(host) + strlen(":") + + SHORT_STR_LEN + strlen(prefix) + strlen(path) + + strlen ("?op=") + strlen(op); + for (i = 0; i < paraNum; i++) { + if (paraNames[i] && paraValues[i]) { + length += 2 + strlen(paraNames[i]) + strlen(paraValues[i]); + } + } + url = malloc(length); // The '\0' has already been included + // when using SHORT_STR_LEN + if (!url) { + return ENOMEM; } - temp = (char*) malloc(length + 1); - if (!temp) { - return NULL; + offset = snprintf(url, length, "%s%s:%d%s%s?op=%s", + protocol, host, nnPort, prefix, path, op); + if (offset >= length || offset < 0) { + ret = EIO; + goto done; + } + for (i = 0; i < paraNum; i++) { + if (!paraNames[i] || !paraValues[i] || paraNames[i][0] == '\0' || + paraValues[i][0] == '\0') { + continue; + } + offset += snprintf(url + offset, length - offset, + "&%s=%s", paraNames[i], paraValues[i]); + if (offset >= length || offset < 0) { + ret = EIO; + goto done; + } } - strcpy(temp,protocol); - temp = strcat(temp,host); - temp = strcat(temp,":"); - temp = strcat(temp,port); - temp = strcat(temp,prefix); - temp = strcat(temp,srcpath); - temp = strcat(temp,"?op="); - temp = strcat(temp,OP); - if (user) { - temp = strcat(temp,"&user.name="); - temp = strcat(temp,user); +done: + if (ret) { + free(url); + return ret; } - url = temp; - return url; + *queryUrl = url; + return 0; } - -static int decToOctal(int decNo) { - int octNo=0; - int expo =0; - while (decNo != 0) { - octNo = ((decNo % 8) * pow(10,expo)) + octNo; - decNo = decNo / 8; - expo++; - } - return octNo; +int createUrlForMKDIR(const char *host, int nnPort, + const char *path, const char *user, char **url) +{ + const char *userPara = "user.name"; + return createQueryURL(host, nnPort, path, "MKDIRS", 1, + &userPara, &user, url); } - -char *prepareMKDIR(const char *host, int nnPort, const char *dirsubpath, const char *user) { - return prepareQUERY(host, nnPort, dirsubpath, "MKDIRS", user); +int createUrlForGetFileStatus(const char *host, int nnPort, const char *path, + const char *user, char **url) +{ + const char *userPara = "user.name"; + return createQueryURL(host, nnPort, path, "GETFILESTATUS", 1, + &userPara, &user, url); } - -char *prepareMKDIRwithMode(const char *host, int nnPort, const char *dirsubpath, int mode, const char *user) { - char *url; - char *permission; - permission = (char*) malloc(NUM_OF_PERMISSION_BITS); - if (!permission) { - return NULL; - } - mode = decToOctal(mode); - sprintf(permission,"%d",mode); - url = prepareMKDIR(host, nnPort, dirsubpath, user); - url = realloc(url,(strlen(url) + strlen("&permission=") + strlen(permission) + 1)); - if (!url) { - return NULL; - } - url = strcat(url,"&permission="); - url = strcat(url,permission); - return url; +int createUrlForLS(const char *host, int nnPort, const char *path, + const char *user, char **url) +{ + const char *userPara = "user.name"; + return createQueryURL(host, nnPort, path, "LISTSTATUS", + 1, &userPara, &user, url); } - -char *prepareRENAME(const char *host, int nnPort, const char *srcpath, const char *destpath, const char *user) { - char *url; - url = prepareQUERY(host, nnPort, srcpath, "RENAME", user); - url = realloc(url,(strlen(url) + strlen("&destination=") + strlen(destpath) + 1)); - if (!url) { - return NULL; - } - url = strcat(url,"&destination="); - url = strcat(url,destpath); - return url; +int createUrlForNnAPPEND(const char *host, int nnPort, const char *path, + const char *user, char **url) +{ + const char *userPara = "user.name"; + return createQueryURL(host, nnPort, path, "APPEND", + 1, &userPara, &user, url); } -char *prepareGFS(const char *host, int nnPort, const char *dirsubpath, const char *user) { - return (prepareQUERY(host, nnPort, dirsubpath, "GETFILESTATUS", user)); +int createUrlForMKDIRwithMode(const char *host, int nnPort, const char *path, + int mode, const char *user, char **url) +{ + int strlength; + char permission[PERM_STR_LEN]; + const char *paraNames[2], *paraValues[2]; + + paraNames[0] = "permission"; + paraNames[1] = "user.name"; + memset(permission, 0, PERM_STR_LEN); + strlength = snprintf(permission, PERM_STR_LEN, "%o", mode); + if (strlength < 0 || strlength >= PERM_STR_LEN) { + return EIO; + } + paraValues[0] = permission; + paraValues[1] = user; + + return createQueryURL(host, nnPort, path, "MKDIRS", 2, + paraNames, paraValues, url); } -char *prepareLS(const char *host, int nnPort, const char *dirsubpath, const char *user) { - return (prepareQUERY(host, nnPort, dirsubpath, "LISTSTATUS", user)); +int createUrlForRENAME(const char *host, int nnPort, const char *srcpath, + const char *destpath, const char *user, char **url) +{ + const char *paraNames[2], *paraValues[2]; + paraNames[0] = "destination"; + paraNames[1] = "user.name"; + paraValues[0] = destpath; + paraValues[1] = user; + + return createQueryURL(host, nnPort, srcpath, + "RENAME", 2, paraNames, paraValues, url); } -char *prepareCHMOD(const char *host, int nnPort, const char *dirsubpath, int mode, const char *user) { - char *url; - char *permission; - permission = (char*) malloc(NUM_OF_PERMISSION_BITS); - if (!permission) { - return NULL; - } - mode &= 0x3FFF; - mode = decToOctal(mode); - sprintf(permission,"%d",mode); - url = prepareQUERY(host, nnPort, dirsubpath, "SETPERMISSION", user); - url = realloc(url,(strlen(url) + strlen("&permission=") + strlen(permission) + 1)); - if (!url) { - return NULL; +int createUrlForCHMOD(const char *host, int nnPort, const char *path, + int mode, const char *user, char **url) +{ + int strlength; + char permission[PERM_STR_LEN]; + const char *paraNames[2], *paraValues[2]; + + paraNames[0] = "permission"; + paraNames[1] = "user.name"; + memset(permission, 0, PERM_STR_LEN); + strlength = snprintf(permission, PERM_STR_LEN, "%o", mode); + if (strlength < 0 || strlength >= PERM_STR_LEN) { + return EIO; } - url = strcat(url,"&permission="); - url = strcat(url,permission); - return url; + paraValues[0] = permission; + paraValues[1] = user; + + return createQueryURL(host, nnPort, path, "SETPERMISSION", + 2, paraNames, paraValues, url); } -char *prepareDELETE(const char *host, int nnPort, const char *dirsubpath, int recursive, const char *user) { - char *url = (prepareQUERY(host, nnPort, dirsubpath, "DELETE", user)); - char *recursiveFlag = (char *)malloc(6); - if (!recursive) { - strcpy(recursiveFlag, "false"); +int createUrlForDELETE(const char *host, int nnPort, const char *path, + int recursive, const char *user, char **url) +{ + const char *paraNames[2], *paraValues[2]; + paraNames[0] = "recursive"; + paraNames[1] = "user.name"; + if (recursive) { + paraValues[0] = "true"; } else { - strcpy(recursiveFlag, "true"); - } - url = (char *) realloc(url, strlen(url) + strlen("&recursive=") + strlen(recursiveFlag) + 1); - if (!url) { - return NULL; + paraValues[0] = "false"; } + paraValues[1] = user; - strcat(url, "&recursive="); - strcat(url, recursiveFlag); - return url; + return createQueryURL(host, nnPort, path, "DELETE", + 2, paraNames, paraValues, url); } -char *prepareCHOWN(const char *host, int nnPort, const char *dirsubpath, const char *owner, const char *group, const char *user) { - char *url; - url = prepareQUERY(host, nnPort, dirsubpath, "SETOWNER", user); - if (!url) { - return NULL; - } - if(owner != NULL) { - url = realloc(url,(strlen(url) + strlen("&owner=") + strlen(owner) + 1)); - url = strcat(url,"&owner="); - url = strcat(url,owner); - } - if (group != NULL) { - url = realloc(url,(strlen(url) + strlen("&group=") + strlen(group) + 1)); - url = strcat(url,"&group="); - url = strcat(url,group); - } - return url; +int createUrlForCHOWN(const char *host, int nnPort, const char *path, + const char *owner, const char *group, + const char *user, char **url) +{ + const char *paraNames[3], *paraValues[3]; + paraNames[0] = "owner"; + paraNames[1] = "group"; + paraNames[2] = "user.name"; + paraValues[0] = owner; + paraValues[1] = group; + paraValues[2] = user; + + return createQueryURL(host, nnPort, path, "SETOWNER", + 3, paraNames, paraValues, url); } -char *prepareOPEN(const char *host, int nnPort, const char *dirsubpath, const char *user, size_t offset, size_t length) { - char *base_url = prepareQUERY(host, nnPort, dirsubpath, "OPEN", user); - char *url = (char *) malloc(strlen(base_url) + strlen("&offset=") + 15 + strlen("&length=") + 15); - if (!url) { - return NULL; - } - sprintf(url, "%s&offset=%ld&length=%ld", base_url, offset, length); - return url; +int createUrlForOPEN(const char *host, int nnPort, const char *path, + const char *user, size_t offset, size_t length, char **url) +{ + int strlength; + char offsetStr[LONG_STR_LEN], lengthStr[LONG_STR_LEN]; + const char *paraNames[3], *paraValues[3]; + + paraNames[0] = "offset"; + paraNames[1] = "length"; + paraNames[2] = "user.name"; + memset(offsetStr, 0, LONG_STR_LEN); + memset(lengthStr, 0, LONG_STR_LEN); + strlength = snprintf(offsetStr, LONG_STR_LEN, "%lu", offset); + if (strlength < 0 || strlength >= LONG_STR_LEN) { + return EIO; + } + strlength = snprintf(lengthStr, LONG_STR_LEN, "%lu", length); + if (strlength < 0 || strlength >= LONG_STR_LEN) { + return EIO; + } + paraValues[0] = offsetStr; + paraValues[1] = lengthStr; + paraValues[2] = user; + + return createQueryURL(host, nnPort, path, "OPEN", + 3, paraNames, paraValues, url); } -char *prepareUTIMES(const char *host, int nnPort, const char *dirsubpath, long unsigned mTime, long unsigned aTime, const char *user) { - char *url; - char *modTime; - char *acsTime; - modTime = (char*) malloc(12); - acsTime = (char*) malloc(12); - url = prepareQUERY(host, nnPort, dirsubpath, "SETTIMES", user); - sprintf(modTime,"%lu",mTime); - sprintf(acsTime,"%lu",aTime); - url = realloc(url,(strlen(url) + strlen("&modificationtime=") + strlen(modTime) + strlen("&accesstime=") + strlen(acsTime) + 1)); - if (!url) { - return NULL; - } - url = strcat(url, "&modificationtime="); - url = strcat(url, modTime); - url = strcat(url,"&accesstime="); - url = strcat(url, acsTime); - return url; +int createUrlForUTIMES(const char *host, int nnPort, const char *path, + long unsigned mTime, long unsigned aTime, + const char *user, char **url) +{ + int strlength; + char modTime[LONG_STR_LEN], acsTime[LONG_STR_LEN]; + const char *paraNames[3], *paraValues[3]; + + memset(modTime, 0, LONG_STR_LEN); + memset(acsTime, 0, LONG_STR_LEN); + strlength = snprintf(modTime, LONG_STR_LEN, "%lu", mTime); + if (strlength < 0 || strlength >= LONG_STR_LEN) { + return EIO; + } + strlength = snprintf(acsTime, LONG_STR_LEN, "%lu", aTime); + if (strlength < 0 || strlength >= LONG_STR_LEN) { + return EIO; + } + paraNames[0] = "modificationtime"; + paraNames[1] = "accesstime"; + paraNames[2] = "user.name"; + paraValues[0] = modTime; + paraValues[1] = acsTime; + paraValues[2] = user; + + return createQueryURL(host, nnPort, path, "SETTIMES", + 3, paraNames, paraValues, url); } -char *prepareNnWRITE(const char *host, int nnPort, const char *dirsubpath, const char *user, int16_t replication, size_t blockSize) { - char *url; - url = prepareQUERY(host, nnPort, dirsubpath, "CREATE", user); - url = realloc(url, (strlen(url) + strlen("&overwrite=true") + 1)); - if (!url) { - return NULL; - } - url = strcat(url, "&overwrite=true"); +int createUrlForNnWRITE(const char *host, int nnPort, + const char *path, const char *user, + int16_t replication, size_t blockSize, char **url) +{ + int strlength; + char repStr[SHORT_STR_LEN], blockSizeStr[LONG_STR_LEN]; + const char *paraNames[4], *paraValues[4]; + + memset(repStr, 0, SHORT_STR_LEN); + memset(blockSizeStr, 0, LONG_STR_LEN); if (replication > 0) { - url = realloc(url, (strlen(url) + strlen("&replication=") + 6)); - if (!url) { - return NULL; + strlength = snprintf(repStr, SHORT_STR_LEN, "%u", replication); + if (strlength < 0 || strlength >= SHORT_STR_LEN) { + return EIO; } - sprintf(url, "%s&replication=%d", url, replication); } if (blockSize > 0) { - url = realloc(url, (strlen(url) + strlen("&blocksize=") + 16)); - if (!url) { - return NULL; + strlength = snprintf(blockSizeStr, LONG_STR_LEN, "%lu", blockSize); + if (strlength < 0 || strlength >= LONG_STR_LEN) { + return EIO; } - sprintf(url, "%s&blocksize=%ld", url, blockSize); } - return url; + paraNames[0] = "overwrite"; + paraNames[1] = "replication"; + paraNames[2] = "blocksize"; + paraNames[3] = "user.name"; + paraValues[0] = "true"; + paraValues[1] = repStr; + paraValues[2] = blockSizeStr; + paraValues[3] = user; + + return createQueryURL(host, nnPort, path, "CREATE", + 4, paraNames, paraValues, url); } -char *prepareNnAPPEND(const char *host, int nnPort, const char *dirsubpath, const char *user) { - return (prepareQUERY(host, nnPort, dirsubpath, "APPEND", user)); +int createUrlForSETREPLICATION(const char *host, int nnPort, + const char *path, int16_t replication, + const char *user, char **url) +{ + char repStr[SHORT_STR_LEN]; + const char *paraNames[2], *paraValues[2]; + int strlength; + + memset(repStr, 0, SHORT_STR_LEN); + if (replication > 0) { + strlength = snprintf(repStr, SHORT_STR_LEN, "%u", replication); + if (strlength < 0 || strlength >= SHORT_STR_LEN) { + return EIO; + } + } + paraNames[0] = "replication"; + paraNames[1] = "user.name"; + paraValues[0] = repStr; + paraValues[1] = user; + + return createQueryURL(host, nnPort, path, "SETREPLICATION", + 2, paraNames, paraValues, url); } -char *prepareSETREPLICATION(const char *host, int nnPort, const char *path, int16_t replication, const char *user) +int createUrlForGetBlockLocations(const char *host, int nnPort, + const char *path, size_t offset, + size_t length, const char *user, char **url) { - char *url = prepareQUERY(host, nnPort, path, "SETREPLICATION", user); - char *replicationNum = (char *) malloc(NUM_OF_REPLICATION_BITS); - sprintf(replicationNum, "%u", replication); - url = realloc(url, strlen(url) + strlen("&replication=") + strlen(replicationNum)+ 1); - if (!url) { - return NULL; + char offsetStr[LONG_STR_LEN], lengthStr[LONG_STR_LEN]; + const char *paraNames[3], *paraValues[3]; + int strlength; + + memset(offsetStr, 0, LONG_STR_LEN); + memset(lengthStr, 0, LONG_STR_LEN); + if (offset > 0) { + strlength = snprintf(offsetStr, LONG_STR_LEN, "%lu", offset); + if (strlength < 0 || strlength >= LONG_STR_LEN) { + return EIO; + } } + if (length > 0) { + strlength = snprintf(lengthStr, LONG_STR_LEN, "%lu", length); + if (strlength < 0 || strlength >= LONG_STR_LEN) { + return EIO; + } + } + paraNames[0] = "offset"; + paraNames[1] = "length"; + paraNames[2] = "user.name"; + paraValues[0] = offsetStr; + paraValues[1] = lengthStr; + paraValues[2] = user; + + return createQueryURL(host, nnPort, path, "GET_BLOCK_LOCATIONS", + 3, paraNames, paraValues, url); +} + +int createUrlForReadFromDatanode(const char *dnHost, int dnPort, + const char *path, size_t offset, + size_t length, const char *user, + const char *namenodeRpcAddr, char **url) +{ + char offsetStr[LONG_STR_LEN], lengthStr[LONG_STR_LEN]; + const char *paraNames[4], *paraValues[4]; + int strlength; + + memset(offsetStr, 0, LONG_STR_LEN); + memset(lengthStr, 0, LONG_STR_LEN); + if (offset > 0) { + strlength = snprintf(offsetStr, LONG_STR_LEN, "%lu", offset); + if (strlength < 0 || strlength >= LONG_STR_LEN) { + return EIO; + } + } + if (length > 0) { + strlength = snprintf(lengthStr, LONG_STR_LEN, "%lu", length); + if (strlength < 0 || strlength >= LONG_STR_LEN) { + return EIO; + } + } + + paraNames[0] = "offset"; + paraNames[1] = "length"; + paraNames[2] = "user.name"; + paraNames[3] = "namenoderpcaddress"; + paraValues[0] = offsetStr; + paraValues[1] = lengthStr; + paraValues[2] = user; + paraValues[3] = namenodeRpcAddr; - url = strcat(url, "&replication="); - url = strcat(url, replicationNum); - return url; + return createQueryURL(dnHost, dnPort, path, "OPEN", + 4, paraNames, paraValues, url); } \ No newline at end of file Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h?rev=1407217&r1=1407216&r2=1407217&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_http_query.h Thu Nov 8 19:09:46 2012 @@ -20,22 +20,221 @@ #ifndef _HDFS_HTTP_QUERY_H_ #define _HDFS_HTTP_QUERY_H_ -#include -#include +#include /* for size_t */ +#include /* for int16_t */ -char *prepareMKDIR(const char *host, int nnPort, const char *dirsubpath, const char *user); -char *prepareMKDIRwithMode(const char *host, int nnPort, const char *dirsubpath, int mode, const char *user); -char *prepareRENAME(const char *host, int nnPort, const char *srcpath, const char *destpath, const char *user); -char *prepareCHMOD(const char *host, int nnPort, const char *dirsubpath, int mode, const char *user); -char *prepareGFS(const char *host, int nnPort, const char *dirsubpath, const char *user); -char *prepareLS(const char *host, int nnPort, const char *dirsubpath, const char *user); -char *prepareDELETE(const char *host, int nnPort, const char *dirsubpath, int recursive, const char *user); -char *prepareCHOWN(const char *host, int nnPort, const char *dirsubpath, const char *owner, const char *group, const char *user); -char *prepareOPEN(const char *host, int nnPort, const char *dirsubpath, const char *user, size_t offset, size_t length); -char *prepareUTIMES(const char *host, int nnPort, const char *dirsubpath, long unsigned mTime, long unsigned aTime, const char *user); -char *prepareNnWRITE(const char *host, int nnPort, const char *dirsubpath, const char *user, int16_t replication, size_t blockSize); -char *prepareNnAPPEND(const char *host, int nnPort, const char *dirsubpath, const char *user); -char *prepareSETREPLICATION(const char *host, int nnPort, const char *path, int16_t replication, const char *user); +/** + * Create the URL for a MKDIR request + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Path of the dir to create + * @param user User name + * @param url Holding the generated URL for MKDIR request + * @return 0 on success and non-zero value on errors + */ +int createUrlForMKDIR(const char *host, int nnPort, + const char *path, const char *user, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for a MKDIR (with mode) request + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Path of the dir to create + * @param mode Mode of MKDIR + * @param user User name + * @param url Holding the generated URL for MKDIR request + * @return 0 on success and non-zero value on errors + */ +int createUrlForMKDIRwithMode(const char *host, int nnPort, const char *path, + int mode, const char *user, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for a RENAME request + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param srcpath Source path + * @param dstpath Destination path + * @param user User name + * @param url Holding the generated URL for RENAME request + * @return 0 on success and non-zero value on errors + */ +int createUrlForRENAME(const char *host, int nnPort, const char *srcpath, + const char *dstpath, const char *user, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for a CHMOD request + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Target path + * @param mode New mode for the file + * @param user User name + * @param url Holding the generated URL for CHMOD request + * @return 0 on success and non-zero value on errors + */ +int createUrlForCHMOD(const char *host, int nnPort, const char *path, + int mode, const char *user, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for a GETFILESTATUS request + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Path of the target file + * @param user User name + * @param url Holding the generated URL for GETFILESTATUS request + * @return 0 on success and non-zero value on errors + */ +int createUrlForGetFileStatus(const char *host, int nnPort, + const char *path, const char *user, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for a LISTSTATUS request + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Path of the directory for listing + * @param user User name + * @param url Holding the generated URL for LISTSTATUS request + * @return 0 on success and non-zero value on errors + */ +int createUrlForLS(const char *host, int nnPort, + const char *path, const char *user, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for a DELETE request + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Path of the file to be deletected + * @param recursive Whether or not to delete in a recursive way + * @param user User name + * @param url Holding the generated URL for DELETE request + * @return 0 on success and non-zero value on errors + */ +int createUrlForDELETE(const char *host, int nnPort, const char *path, + int recursive, const char *user, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for a CHOWN request + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Path of the target + * @param owner New owner + * @param group New group + * @param user User name + * @param url Holding the generated URL for CHOWN request + * @return 0 on success and non-zero value on errors + */ +int createUrlForCHOWN(const char *host, int nnPort, const char *path, + const char *owner, const char *group, const char *user, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for a OPEN/READ request + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Path of the file to read + * @param user User name + * @param offset Offset for reading (the start position for this read) + * @param length Length of the file to read + * @param url Holding the generated URL for OPEN/READ request + * @return 0 on success and non-zero value on errors + */ +int createUrlForOPEN(const char *host, int nnPort, const char *path, + const char *user, size_t offset, size_t length, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for a UTIMES (update time) request + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Path of the file for updating time + * @param mTime Modified time to set + * @param aTime Access time to set + * @param user User name + * @param url Holding the generated URL for UTIMES request + * @return 0 on success and non-zero value on errors + */ +int createUrlForUTIMES(const char *host, int nnPort, const char *path, + long unsigned mTime, long unsigned aTime, + const char *user, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for a WRITE/CREATE request (sent to NameNode) + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Path of the dir to create + * @param user User name + * @param replication Number of replication of the file + * @param blockSize Size of the block for the file + * @param url Holding the generated URL for WRITE request + * @return 0 on success and non-zero value on errors + */ +int createUrlForNnWRITE(const char *host, int nnPort, const char *path, + const char *user, int16_t replication, size_t blockSize, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for an APPEND request (sent to NameNode) + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Path of the file for appending + * @param user User name + * @param url Holding the generated URL for APPEND request + * @return 0 on success and non-zero value on errors + */ +int createUrlForNnAPPEND(const char *host, int nnPort, + const char *path, const char *user, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for a SETREPLICATION request + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Path of the target file + * @param replication New replication number + * @param user User name + * @param url Holding the generated URL for SETREPLICATION request + * @return 0 on success and non-zero value on errors + */ +int createUrlForSETREPLICATION(const char *host, int nnPort, const char *path, + int16_t replication, const char *user, + char **url) __attribute__ ((warn_unused_result)); + +/** + * Create the URL for a GET_BLOCK_LOCATIONS request + * + * @param host The hostname of the NameNode + * @param nnPort Port of the NameNode + * @param path Path of the target file + * @param offset The offset in the file + * @param length Length of the file content + * @param user User name + * @param url Holding the generated URL for GET_BLOCK_LOCATIONS request + * @return 0 on success and non-zero value on errors + */ +int createUrlForGetBlockLocations(const char *host, int nnPort, + const char *path, size_t offset, + size_t length, const char *user, + char **url) __attribute__ ((warn_unused_result)); #endif //_HDFS_HTTP_QUERY_H_ Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c?rev=1407217&r1=1407216&r2=1407217&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.c Thu Nov 8 19:09:46 2012 @@ -25,6 +25,11 @@ #include #include +static const char * const temporaryRedirectCode = "307 TEMPORARY_REDIRECT"; +static const char * const twoHundredOKCode = "200 OK"; +static const char * const twoHundredOneCreatedCode = "201 Created"; +static const char * const httpHeaderString = "HTTP/1.1"; + /** * Exception information after calling JSON operations */ @@ -34,9 +39,6 @@ struct jsonException { const char *message; }; -static hdfsFileInfo *parseJsonGFS(json_t *jobj, hdfsFileInfo *fileStat, - int *numEntries, const char *operation); - static void dotsToSlashes(char *str) { for (; *str != '\0'; str++) { @@ -45,8 +47,9 @@ static void dotsToSlashes(char *str) } } -int printJsonExceptionV(struct jsonException *exc, int noPrintFlags, - const char *fmt, va_list ap) +/** Print out the JSON exception information */ +static int printJsonExceptionV(struct jsonException *exc, int noPrintFlags, + const char *fmt, va_list ap) { char *javaClassName = NULL; int excErrno = EINTERNAL, shouldPrint = 0; @@ -74,11 +77,23 @@ int printJsonExceptionV(struct jsonExcep return excErrno; } -int printJsonException(struct jsonException *exc, int noPrintFlags, - const char *fmt, ...) +/** + * Print out JSON exception information. + * + * @param exc The exception information to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +static int printJsonException(struct jsonException *exc, int noPrintFlags, + const char *fmt, ...) { va_list ap; - int ret; + int ret = 0; va_start(ap, fmt); ret = printJsonExceptionV(exc, noPrintFlags, fmt, ap); @@ -86,81 +101,20 @@ int printJsonException(struct jsonExcept return ret; } -static hdfsFileInfo *json_parse_array(json_t *jobj, char *key, hdfsFileInfo *fileStat, int *numEntries, const char *operation) { - int arraylen = json_array_size(jobj); //Getting the length of the array - *numEntries = arraylen; - if (!key) { - return NULL; - } - if(arraylen > 0) { - fileStat = (hdfsFileInfo *)realloc(fileStat,sizeof(hdfsFileInfo)*arraylen); - } - json_t *jvalue; - int i; - for (i=0; i< arraylen; i++) { - jvalue = json_array_get(jobj, i); //Getting the array element at position i - if (json_is_array(jvalue)) { // array within an array - program should never come here for now - json_parse_array(jvalue, NULL, &fileStat[i], numEntries, operation); - } - else if (json_is_object(jvalue)) { // program will definitely come over here - parseJsonGFS(jvalue, &fileStat[i], numEntries, operation); - } - else { - return NULL; // program will never come over here for now - } - } - *numEntries = arraylen; - return fileStat; -} - -int parseBoolean(char *response) { - json_t *root; - json_error_t error; - size_t flags = 0; - int result = 0; - const char *key; - json_t *value; - root = json_loads(response, flags, &error); - void *iter = json_object_iter(root); - while(iter) { - key = json_object_iter_key(iter); - value = json_object_iter_value(iter); - switch (json_typeof(value)) { - case JSON_TRUE: - result = 1; - break; - default: - result = 0; - break; - } - iter = json_object_iter_next(root, iter); - } - return result; -} - -int parseMKDIR(char *response) { - return (parseBoolean(response)); -} - -int parseRENAME(char *response) { - return (parseBoolean(response)); -} - -int parseDELETE(char *response) { - return (parseBoolean(response)); -} - -struct jsonException *parseJsonException(json_t *jobj) { - const char *key; - json_t *value; +/** Parse the exception information from JSON */ +static struct jsonException *parseJsonException(json_t *jobj) +{ + const char *key = NULL; + json_t *value = NULL; struct jsonException *exception = NULL; + void *iter = NULL; exception = calloc(1, sizeof(*exception)); if (!exception) { return NULL; } - void *iter = json_object_iter(jobj); + iter = json_object_iter(jobj); while (iter) { key = json_object_iter_key(iter); value = json_object_iter_value(iter); @@ -175,23 +129,31 @@ struct jsonException *parseJsonException iter = json_object_iter_next(jobj, iter); } - return exception; } -struct jsonException *parseException(const char *content) { - if (!content) { - return NULL; - } - +/** + * Parse the exception information which is presented in JSON + * + * @param content Exception information in JSON + * @return jsonException for printing out + */ +static struct jsonException *parseException(const char *content) +{ json_error_t error; size_t flags = 0; - const char *key; + const char *key = NULL; json_t *value; - json_t *jobj = json_loads(content, flags, &error); + json_t *jobj; + struct jsonException *exception = NULL; + if (!content) { + return NULL; + } + jobj = json_loads(content, flags, &error); if (!jobj) { - fprintf(stderr, "JSon parsing failed\n"); + fprintf(stderr, "JSon parsing error: on line %d: %s\n", + error.line, error.text); return NULL; } void *iter = json_object_iter(jobj); @@ -199,254 +161,503 @@ struct jsonException *parseException(con key = json_object_iter_key(iter); value = json_object_iter_value(iter); - if (!strcmp(key, "RemoteException") && json_typeof(value) == JSON_OBJECT) { - return parseJsonException(value); + if (!strcmp(key, "RemoteException") && + json_typeof(value) == JSON_OBJECT) { + exception = parseJsonException(value); + break; } iter = json_object_iter_next(jobj, iter); } - return NULL; + + json_decref(jobj); + return exception; } -static hdfsFileInfo *parseJsonGFS(json_t *jobj, hdfsFileInfo *fileStat, - int *numEntries, const char *operation) +/** + * Parse the response information which uses TRUE/FALSE + * to indicate whether the operation succeeded + * + * @param response Response information + * @return 0 to indicate success + */ +static int parseBoolean(const char *response) { - const char *tempstr; - const char *key; - json_t *value; - void *iter = json_object_iter(jobj); - while(iter) { - key = json_object_iter_key(iter); - value = json_object_iter_value(iter); - - switch (json_typeof(value)) { - case JSON_INTEGER: - if(!strcmp(key,"accessTime")) { - fileStat->mLastAccess = (time_t)(json_integer_value(value)/1000); - } else if (!strcmp(key,"blockSize")) { - fileStat->mBlockSize = (tOffset)json_integer_value(value); - } else if (!strcmp(key,"length")) { - fileStat->mSize = (tOffset)json_integer_value(value); - } else if(!strcmp(key,"modificationTime")) { - fileStat->mLastMod = (time_t)(json_integer_value(value)/1000); - } else if (!strcmp(key,"replication")) { - fileStat->mReplication = (short)json_integer_value(value); - } - break; - - case JSON_STRING: - if(!strcmp(key,"group")) { - fileStat->mGroup=(char *)json_string_value(value); - } else if (!strcmp(key,"owner")) { - fileStat->mOwner=(char *)json_string_value(value); - } else if (!strcmp(key,"pathSuffix")) { - fileStat->mName=(char *)json_string_value(value); - } else if (!strcmp(key,"permission")) { - tempstr=(char *)json_string_value(value); - fileStat->mPermissions = (short)strtol(tempstr,(char **)NULL,8); - } else if (!strcmp(key,"type")) { - char *cvalue = (char *)json_string_value(value); - if (!strcmp(cvalue, "DIRECTORY")) { - fileStat->mKind = kObjectKindDirectory; - } else { - fileStat->mKind = kObjectKindFile; - } - } - break; - - case JSON_OBJECT: - if(!strcmp(key,"FileStatus")) { - parseJsonGFS(value, fileStat, numEntries, operation); - } else if (!strcmp(key,"FileStatuses")) { - fileStat = parseJsonGFS(value, &fileStat[0], numEntries, operation); - } else if (!strcmp(key,"RemoteException")) { - //Besides returning NULL, we also need to print the exception information - struct jsonException *exception = parseJsonException(value); - if (exception) { - errno = printJsonException(exception, PRINT_EXC_ALL, "Calling WEBHDFS (%s)", operation); - } - - if(fileStat != NULL) { - free(fileStat); - fileStat = NULL; - } - } - break; - - case JSON_ARRAY: - if (!strcmp(key,"FileStatus")) { - fileStat = json_parse_array(value,(char *) key,fileStat,numEntries, operation); - } - break; - - default: - if(fileStat != NULL) { - free(fileStat); - fileStat = NULL; - } - } - iter = json_object_iter_next(jobj, iter); + json_t *root, *value; + json_error_t error; + size_t flags = 0; + int result = 0; + + root = json_loads(response, flags, &error); + if (!root) { + fprintf(stderr, "JSon parsing error: on line %d: %s\n", + error.line, error.text); + return EIO; + } + void *iter = json_object_iter(root); + value = json_object_iter_value(iter); + if (json_typeof(value) == JSON_TRUE) { + result = 0; + } else { + result = EIO; // FALSE means error in remote NN/DN } - return fileStat; + json_decref(root); + return result; } +int parseMKDIR(const char *response) +{ + return parseBoolean(response); +} + +int parseRENAME(const char *response) +{ + return parseBoolean(response); +} + +int parseDELETE(const char *response) +{ + return parseBoolean(response); +} -int checkHeader(char *header, const char *content, const char *operation) { +int parseSETREPLICATION(const char *response) +{ + return parseBoolean(response); +} + +/** + * Check the header of response to see if it's 200 OK + * + * @param header Header information for checking + * @param content Stores exception information if there are errors + * @param operation Indicate the operation for exception printing + * @return 0 for success + */ +static int checkHeader(const char *header, const char *content, + const char *operation) +{ char *result = NULL; - char delims[] = ":"; - char *responseCode= "200 OK"; - if(header == '\0' || strncmp(header, "HTTP/", strlen("HTTP/"))) { - return 0; + const char delims[] = ":"; + char *savepter; + int ret = 0; + + if (!header || strncmp(header, "HTTP/", strlen("HTTP/"))) { + return EINVAL; } - if(!(strstr(header, responseCode)) || !(header = strstr(header, "Content-Length"))) { + if (!(strstr(header, twoHundredOKCode)) || + !(result = strstr(header, "Content-Length"))) { struct jsonException *exc = parseException(content); if (exc) { - errno = printJsonException(exc, PRINT_EXC_ALL, "Calling WEBHDFS (%s)", operation); + ret = printJsonException(exc, PRINT_EXC_ALL, + "Calling WEBHDFS (%s)", operation); + } else { + ret = EIO; } - return 0; + return ret; } - result = strtok(header, delims); - result = strtok(NULL,delims); + result = strtok_r(result, delims, &savepter); + result = strtok_r(NULL, delims, &savepter); while (isspace(*result)) { result++; } - if(strcmp(result,"0")) { //Content-Length should be equal to 0 - return 1; - } else { - return 0; - } -} - -int parseOPEN(const char *header, const char *content) { - const char *responseCode1 = "307 TEMPORARY_REDIRECT"; - const char *responseCode2 = "200 OK"; - if(header == '\0' || strncmp(header,"HTTP/",strlen("HTTP/"))) { - return -1; - } - if(!(strstr(header,responseCode1) && strstr(header, responseCode2))) { - struct jsonException *exc = parseException(content); - if (exc) { - //if the exception is an IOException and it is because the offset is out of the range - //do not print out the exception - if (!strcasecmp(exc->exception, "IOException") && strstr(exc->message, "out of the range")) { - return 0; - } - errno = printJsonException(exc, PRINT_EXC_ALL, "Calling WEBHDFS (OPEN)"); - } - return -1; + // Content-Length should be equal to 0, + // and the string should be "0\r\nServer" + if (strncmp(result, "0\r\n", 3)) { + ret = EIO; } - - return 1; + return ret; } -int parseCHMOD(char *header, const char *content) { +int parseCHMOD(const char *header, const char *content) +{ return checkHeader(header, content, "CHMOD"); } - -int parseCHOWN(char *header, const char *content) { +int parseCHOWN(const char *header, const char *content) +{ return checkHeader(header, content, "CHOWN"); } -int parseUTIMES(char *header, const char *content) { - return checkHeader(header, content, "UTIMES"); +int parseUTIMES(const char *header, const char *content) +{ + return checkHeader(header, content, "SETTIMES"); } - -int checkIfRedirect(const char *const headerstr, const char *content, const char *operation) { - char *responseCode = "307 TEMPORARY_REDIRECT"; - char * locTag = "Location"; - char * tempHeader; - if(headerstr == '\0' || strncmp(headerstr,"HTTP/", 5)) { - return 0; +/** + * Check if the header contains correct information + * ("307 TEMPORARY_REDIRECT" and "Location") + * + * @param header Header for parsing + * @param content Contains exception information + * if the remote operation failed + * @param operation Specify the remote operation when printing out exception + * @return 0 for success + */ +static int checkRedirect(const char *header, + const char *content, const char *operation) +{ + const char *locTag = "Location"; + int ret = 0, offset = 0; + + // The header must start with "HTTP/1.1" + if (!header || strncmp(header, httpHeaderString, + strlen(httpHeaderString))) { + return EINVAL; } - if(!(tempHeader = strstr(headerstr,responseCode))) { - //process possible exception information + + offset += strlen(httpHeaderString); + while (isspace(header[offset])) { + offset++; + } + // Looking for "307 TEMPORARY_REDIRECT" in header + if (strncmp(header + offset, temporaryRedirectCode, + strlen(temporaryRedirectCode))) { + // Process possible exception information struct jsonException *exc = parseException(content); if (exc) { - errno = printJsonException(exc, PRINT_EXC_ALL, "Calling WEBHDFS (%s)", operation); - } - return 0; - } - if(!(strstr(tempHeader,locTag))) { - return 0; + ret = printJsonException(exc, PRINT_EXC_ALL, + "Calling WEBHDFS (%s)", operation); + } else { + ret = EIO; + } + return ret; + } + // Here we just simply check if header contains "Location" tag, + // detailed processing is in parseDnLoc + if (!(strstr(header, locTag))) { + ret = EIO; } - return 1; + return ret; } - -int parseNnWRITE(const char *header, const char *content) { - return checkIfRedirect(header, content, "Write(NameNode)"); +int parseNnWRITE(const char *header, const char *content) +{ + return checkRedirect(header, content, "Write(NameNode)"); } +int parseNnAPPEND(const char *header, const char *content) +{ + return checkRedirect(header, content, "Append(NameNode)"); +} -int parseNnAPPEND(const char *header, const char *content) { - return checkIfRedirect(header, content, "Append(NameNode)"); +/** 0 for success , -1 for out of range, other values for error */ +int parseOPEN(const char *header, const char *content) +{ + int ret = 0, offset = 0; + + if (!header || strncmp(header, httpHeaderString, + strlen(httpHeaderString))) { + return EINVAL; + } + + offset += strlen(httpHeaderString); + while (isspace(header[offset])) { + offset++; + } + if (strncmp(header + offset, temporaryRedirectCode, + strlen(temporaryRedirectCode)) || + !strstr(header, twoHundredOKCode)) { + struct jsonException *exc = parseException(content); + if (exc) { + // If the exception is an IOException and it is because + // the offset is out of the range, do not print out the exception + if (!strcasecmp(exc->exception, "IOException") && + strstr(exc->message, "out of the range")) { + ret = -1; + } else { + ret = printJsonException(exc, PRINT_EXC_ALL, + "Calling WEBHDFS (OPEN)"); + } + } else { + ret = EIO; + } + } + return ret; } -char *parseDnLoc(char *content) { - char delims[] = "\r\n"; - char *url = NULL; - char *DnLocation = NULL; - char *savepter; - DnLocation = strtok_r(content, delims, &savepter); - while (DnLocation && strncmp(DnLocation, "Location:", strlen("Location:"))) { - DnLocation = strtok_r(NULL, delims, &savepter); +int parseDnLoc(char *content, char **dn) +{ + char *url = NULL, *dnLocation = NULL, *savepter, *tempContent; + const char *prefix = "Location: http://"; + const char *prefixToRemove = "Location: "; + const char *delims = "\r\n"; + + tempContent = strdup(content); + if (!tempContent) { + return ENOMEM; } - if (!DnLocation) { - return NULL; + + dnLocation = strtok_r(tempContent, delims, &savepter); + while (dnLocation && strncmp(dnLocation, "Location:", + strlen("Location:"))) { + dnLocation = strtok_r(NULL, delims, &savepter); } - DnLocation = strstr(DnLocation, "http"); - if (!DnLocation) { - return NULL; + if (!dnLocation) { + return EIO; + } + + while (isspace(*dnLocation)) { + dnLocation++; } - url = malloc(strlen(DnLocation) + 1); + if (strncmp(dnLocation, prefix, strlen(prefix))) { + return EIO; + } + url = strdup(dnLocation + strlen(prefixToRemove)); if (!url) { - return NULL; + return ENOMEM; } - strcpy(url, DnLocation); - return url; + *dn = url; + return 0; } -int parseDnWRITE(const char *header, const char *content) { - char *responseCode = "201 Created"; - fprintf(stderr, "\nheaderstr is: %s\n", header); - if(header == '\0' || strncmp(header,"HTTP/",strlen("HTTP/"))) { - return 0; +int parseDnWRITE(const char *header, const char *content) +{ + int ret = 0; + if (header == NULL || header[0] == '\0' || + strncmp(header, "HTTP/", strlen("HTTP/"))) { + return EINVAL; } - if(!(strstr(header,responseCode))) { + if (!(strstr(header, twoHundredOneCreatedCode))) { struct jsonException *exc = parseException(content); if (exc) { - errno = printJsonException(exc, PRINT_EXC_ALL, "Calling WEBHDFS (WRITE(DataNode))"); + ret = printJsonException(exc, PRINT_EXC_ALL, + "Calling WEBHDFS (WRITE(DataNode))"); + } else { + ret = EIO; } - return 0; } - return 1; + return ret; } -int parseDnAPPEND(const char *header, const char *content) { - char *responseCode = "200 OK"; - if(header == '\0' || strncmp(header, "HTTP/", strlen("HTTP/"))) { - return 0; +int parseDnAPPEND(const char *header, const char *content) +{ + int ret = 0; + + if (header == NULL || header[0] == '\0' || + strncmp(header, "HTTP/", strlen("HTTP/"))) { + return EINVAL; } - if(!(strstr(header, responseCode))) { + if (!(strstr(header, twoHundredOKCode))) { struct jsonException *exc = parseException(content); if (exc) { - errno = printJsonException(exc, PRINT_EXC_ALL, "Calling WEBHDFS (APPEND(DataNode))"); + ret = printJsonException(exc, PRINT_EXC_ALL, + "Calling WEBHDFS (APPEND(DataNode))"); + } else { + ret = EIO; } - return 0; } - return 1; + return ret; } -hdfsFileInfo *parseGFS(char *str, hdfsFileInfo *fileStat, int *numEntries) { +/** + * Retrieve file status from the JSON object + * + * @param jobj JSON object for parsing, which contains + * file status information + * @param fileStat hdfsFileInfo handle to hold file status information + * @return 0 on success + */ +static int parseJsonForFileStatus(json_t *jobj, hdfsFileInfo *fileStat) +{ + const char *key, *tempstr; + json_t *value; + void *iter = NULL; + + iter = json_object_iter(jobj); + while (iter) { + key = json_object_iter_key(iter); + value = json_object_iter_value(iter); + + if (!strcmp(key, "accessTime")) { + // json field contains time in milliseconds, + // hdfsFileInfo is counted in seconds + fileStat->mLastAccess = json_integer_value(value) / 1000; + } else if (!strcmp(key, "blockSize")) { + fileStat->mBlockSize = json_integer_value(value); + } else if (!strcmp(key, "length")) { + fileStat->mSize = json_integer_value(value); + } else if (!strcmp(key, "modificationTime")) { + fileStat->mLastMod = json_integer_value(value) / 1000; + } else if (!strcmp(key, "replication")) { + fileStat->mReplication = json_integer_value(value); + } else if (!strcmp(key, "group")) { + fileStat->mGroup = strdup(json_string_value(value)); + if (!fileStat->mGroup) { + return ENOMEM; + } + } else if (!strcmp(key, "owner")) { + fileStat->mOwner = strdup(json_string_value(value)); + if (!fileStat->mOwner) { + return ENOMEM; + } + } else if (!strcmp(key, "pathSuffix")) { + fileStat->mName = strdup(json_string_value(value)); + if (!fileStat->mName) { + return ENOMEM; + } + } else if (!strcmp(key, "permission")) { + tempstr = json_string_value(value); + fileStat->mPermissions = (short) strtol(tempstr, NULL, 8); + } else if (!strcmp(key, "type")) { + tempstr = json_string_value(value); + if (!strcmp(tempstr, "DIRECTORY")) { + fileStat->mKind = kObjectKindDirectory; + } else { + fileStat->mKind = kObjectKindFile; + } + } + // Go to the next key-value pair in the json object + iter = json_object_iter_next(jobj, iter); + } + return 0; +} + +int parseGFS(const char *response, hdfsFileInfo *fileStat, int printError) +{ + int ret = 0, printFlag; json_error_t error; size_t flags = 0; - json_t *jobj = json_loads(str, flags, &error); - fileStat = parseJsonGFS(jobj, fileStat, numEntries, "GETPATHSTATUS/LISTSTATUS"); - return fileStat; + json_t *jobj, *value; + const char *key; + void *iter = NULL; + + if (!response || !fileStat) { + return EIO; + } + jobj = json_loads(response, flags, &error); + if (!jobj) { + fprintf(stderr, "error while parsing json: on line %d: %s\n", + error.line, error.text); + return EIO; + } + iter = json_object_iter(jobj); + key = json_object_iter_key(iter); + value = json_object_iter_value(iter); + if (json_typeof(value) == JSON_OBJECT) { + if (!strcmp(key, "RemoteException")) { + struct jsonException *exception = parseJsonException(value); + if (exception) { + if (printError) { + printFlag = PRINT_EXC_ALL; + } else { + printFlag = NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | + NOPRINT_EXC_PARENT_NOT_DIRECTORY; + } + ret = printJsonException(exception, printFlag, + "Calling WEBHDFS GETFILESTATUS"); + } else { + ret = EIO; + } + } else if (!strcmp(key, "FileStatus")) { + ret = parseJsonForFileStatus(value, fileStat); + } else { + ret = EIO; + } + + } else { + ret = EIO; + } + + json_decref(jobj); + return ret; } -int parseSETREPLICATION(char *response) { - return (parseBoolean(response)); +/** + * Parse the JSON array. Called to parse the result of + * the LISTSTATUS operation. Thus each element of the JSON array is + * a JSON object with the information of a file entry contained + * in the folder. + * + * @param jobj The JSON array to be parsed + * @param fileStat The hdfsFileInfo handle used to + * store a group of file information + * @param numEntries Capture the number of files in the folder + * @return 0 for success + */ +static int parseJsonArrayForFileStatuses(json_t *jobj, hdfsFileInfo **fileStat, + int *numEntries) +{ + json_t *jvalue = NULL; + int i = 0, ret = 0, arraylen = 0; + hdfsFileInfo *fileInfo = NULL; + + arraylen = (int) json_array_size(jobj); + if (arraylen > 0) { + fileInfo = calloc(arraylen, sizeof(hdfsFileInfo)); + if (!fileInfo) { + return ENOMEM; + } + } + for (i = 0; i < arraylen; i++) { + //Getting the array element at position i + jvalue = json_array_get(jobj, i); + if (json_is_object(jvalue)) { + ret = parseJsonForFileStatus(jvalue, &fileInfo[i]); + if (ret) { + goto done; + } + } else { + ret = EIO; + goto done; + } + } +done: + if (ret) { + free(fileInfo); + } else { + *numEntries = arraylen; + *fileStat = fileInfo; + } + return ret; } +int parseLS(const char *response, hdfsFileInfo **fileStats, int *numOfEntries) +{ + int ret = 0; + json_error_t error; + size_t flags = 0; + json_t *jobj, *value; + const char *key; + void *iter = NULL; + + if (!response || response[0] == '\0' || !fileStats) { + return EIO; + } + jobj = json_loads(response, flags, &error); + if (!jobj) { + fprintf(stderr, "error while parsing json: on line %d: %s\n", + error.line, error.text); + return EIO; + } + + iter = json_object_iter(jobj); + key = json_object_iter_key(iter); + value = json_object_iter_value(iter); + if (json_typeof(value) == JSON_OBJECT) { + if (!strcmp(key, "RemoteException")) { + struct jsonException *exception = parseJsonException(value); + if (exception) { + ret = printJsonException(exception, PRINT_EXC_ALL, + "Calling WEBHDFS GETFILESTATUS"); + } else { + ret = EIO; + } + } else if (!strcmp(key, "FileStatuses")) { + iter = json_object_iter(value); + value = json_object_iter_value(iter); + if (json_is_array(value)) { + ret = parseJsonArrayForFileStatuses(value, fileStats, + numOfEntries); + } else { + ret = EIO; + } + } else { + ret = EIO; + } + } else { + ret = EIO; + } + + json_decref(jobj); + return ret; +} Modified: hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.h URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.h?rev=1407217&r1=1407216&r2=1407217&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.h (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/src/hdfs_json_parser.h Thu Nov 8 19:09:46 2012 @@ -18,41 +18,161 @@ #ifndef _HDFS_JSON_PARSER_H_ #define _HDFS_JSON_PARSER_H_ -struct jsonException; +/** + * Parse the response for MKDIR request. The response uses TRUE/FALSE + * to indicate whether the operation succeeded. + * + * @param response The response information to parse. + * @return 0 for success + */ +int parseMKDIR(const char *response); /** - * Print out JSON exception information. + * Parse the response for RENAME request. The response uses TRUE/FALSE + * to indicate whether the operation succeeded. * - * @param exc The exception information to print and free - * @param noPrintFlags Flags which determine which exceptions we should NOT - * print. - * @param fmt Printf-style format list - * @param ... Printf-style varargs + * @param response The response information to parse. + * @return 0 for success + */ +int parseRENAME(const char *response); + +/** + * Parse the response for DELETE request. The response uses TRUE/FALSE + * to indicate whether the operation succeeded. * - * @return The POSIX error number associated with the exception - * object. + * @param response The response information to parse. + * @return 0 for success */ -int printJsonException(struct jsonException *exc, int noPrintFlags, - const char *fmt, ...); +int parseDELETE(const char *response); -int parseMKDIR(char *response); -int parseRENAME(char *response); -int parseDELETE (char *response); -int parseSETREPLICATION(char *response); +/** + * Parse the response for SETREPLICATION request. The response uses TRUE/FALSE + * to indicate whether the operation succeeded. + * + * @param response The response information to parse. + * @return 0 for success + */ +int parseSETREPLICATION(const char *response); +/** + * Parse the response for OPEN (read) request. A successful operation + * will return "200 OK". + * + * @param response The response information for parsing + * @return 0 for success , -1 for out of range, other values for error + */ int parseOPEN(const char *header, const char *content); +/** + * Parse the response for WRITE (from NameNode) request. + * A successful operation should return "307 TEMPORARY_REDIRECT" in its header. + * + * @param header The header of the http response + * @param content If failing, the exception message + * sent from NameNode is stored in content + * @return 0 for success + */ int parseNnWRITE(const char *header, const char *content); + +/** + * Parse the response for WRITE (from DataNode) request. + * A successful operation should return "201 Created" in its header. + * + * @param header The header of the http response + * @param content If failing, the exception message + * sent from DataNode is stored in content + * @return 0 for success + */ int parseDnWRITE(const char *header, const char *content); + +/** + * Parse the response for APPEND (sent from NameNode) request. + * A successful operation should return "307 TEMPORARY_REDIRECT" in its header. + * + * @param header The header of the http response + * @param content If failing, the exception message + * sent from NameNode is stored in content + * @return 0 for success + */ int parseNnAPPEND(const char *header, const char *content); + +/** + * Parse the response for APPEND (from DataNode) request. + * A successful operation should return "200 OK" in its header. + * + * @param header The header of the http response + * @param content If failing, the exception message + * sent from DataNode is stored in content + * @return 0 for success + */ int parseDnAPPEND(const char *header, const char *content); -char* parseDnLoc(char *content); +/** + * Parse the response (from NameNode) to get the location information + * of the DataNode that should be contacted for the following write operation. + * + * @param content Content of the http header + * @param dn To store the location of the DataNode for writing + * @return 0 for success + */ +int parseDnLoc(char *content, char **dn) __attribute__ ((warn_unused_result)); + +/** + * Parse the response for GETFILESTATUS operation. + * + * @param response Response to parse. Its detailed format is specified in + * "http://hadoop.apache.org/docs/stable/webhdfs.html#GETFILESTATUS" + * @param fileStat A hdfsFileInfo handle for holding file information + * @param printError Whether or not print out exception + * when file does not exist + * @return 0 for success, non-zero value to indicate error + */ +int parseGFS(const char *response, hdfsFileInfo *fileStat, int printError); -hdfsFileInfo *parseGFS(char *response, hdfsFileInfo *fileStat, int *numEntries); +/** + * Parse the response for LISTSTATUS operation. + * + * @param response Response to parse. Its detailed format is specified in + * "http://hadoop.apache.org/docs/r1.0.3/webhdfs.html#LISTSTATUS" + * @param fileStats Pointer pointing to a list of hdfsFileInfo handles + * holding file/dir information in the directory + * @param numEntries After parsing, the value of this parameter indicates + * the number of file entries. + * @return 0 for success, non-zero value to indicate error + */ +int parseLS(const char *response, hdfsFileInfo **fileStats, int *numOfEntries); -int parseCHOWN (char *header, const char *content); -int parseCHMOD (char *header, const char *content); -int parseUTIMES(char *header, const char *content); +/** + * Parse the response for CHOWN request. + * A successful operation should contains "200 OK" in its header, + * and the Content-Length should be 0. + * + * @param header The header of the http response + * @param content If failing, the exception message is stored in content + * @return 0 for success + */ +int parseCHOWN(const char *header, const char *content); + +/** + * Parse the response for CHMOD request. + * A successful operation should contains "200 OK" in its header, + * and the Content-Length should be 0. + * + * @param header The header of the http response + * @param content If failing, the exception message is stored in content + * @return 0 for success + */ +int parseCHMOD(const char *header, const char *content); + +/** + * Parse the response for SETTIMES request. + * A successful operation should contains "200 OK" in its header, + * and the Content-Length should be 0. + * + * @param header The header of the http response + * @param content If failing, the exception message is stored in content + * @return 0 for success + */ +int parseUTIMES(const char *header, const char *content); -#endif //_FUSE_JSON_PARSER_H +#endif //_HDFS_JSON_PARSER_H_