Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 970BD18C9F for ; Wed, 7 Oct 2015 07:16:25 +0000 (UTC) Received: (qmail 72966 invoked by uid 500); 7 Oct 2015 07:16:11 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 72835 invoked by uid 500); 7 Oct 2015 07:16:11 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 72155 invoked by uid 99); 7 Oct 2015 07:16:11 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 07 Oct 2015 07:16:11 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id CEDB8E0ADA; Wed, 7 Oct 2015 07:16:10 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wheat9@apache.org To: common-commits@hadoop.apache.org Date: Wed, 07 Oct 2015 07:16:24 -0000 Message-Id: In-Reply-To: <43741109eb13465bbbe1298c7acc189a@git.apache.org> References: <43741109eb13465bbbe1298c7acc189a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [16/19] hadoop git commit: HDFS-9170. Move libhdfs / fuse-dfs / libwebhdfs to hdfs-client. Contributed by Haohui Mai. http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_chmod.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_chmod.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_chmod.c new file mode 100644 index 0000000..8c25f53 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_chmod.c @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_users.h" +#include "fuse_connect.h" + +int dfs_chmod(const char *path, mode_t mode) +{ + struct hdfsConn *conn = NULL; + hdfsFS fs; + TRACE1("chmod", path) + int ret = 0; + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + + assert(path); + assert(dfs); + assert('/' == *path); + + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; + } + fs = hdfsConnGetFs(conn); + + if (hdfsChmod(fs, path, (short)mode)) { + ERROR("Could not chmod %s to %d", path, (int)mode); + ret = (errno > 0) ? -errno : -EIO; + goto cleanup; + } + +cleanup: + if (conn) { + hdfsConnRelease(conn); + } + + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_chown.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_chown.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_chown.c new file mode 100644 index 0000000..2a6b61c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_chown.c @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_users.h" +#include "fuse_impls.h" +#include "fuse_connect.h" + +#include + +int dfs_chown(const char *path, uid_t uid, gid_t gid) +{ + struct hdfsConn *conn = NULL; + int ret = 0; + char *user = NULL; + char *group = NULL; + + TRACE1("chown", path) + + // retrieve dfs specific data + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + + // check params and the context var + assert(path); + assert(dfs); + assert('/' == *path); + + if ((uid == -1) && (gid == -1)) { + ret = 0; + goto cleanup; + } + if (uid != -1) { + user = getUsername(uid); + if (NULL == user) { + ERROR("Could not lookup the user id string %d",(int)uid); + ret = -EIO; + goto cleanup; + } + } + if (gid != -1) { + group = getGroup(gid); + if (group == NULL) { + ERROR("Could not lookup the group id string %d",(int)gid); + ret = -EIO; + goto cleanup; + } + } + + ret = fuseConnect(user, fuse_get_context(), &conn); + if (ret) { + fprintf(stderr, "fuseConnect: failed to open a libhdfs connection! " + "error %d.\n", ret); + ret = -EIO; + goto cleanup; + } + + if (hdfsChown(hdfsConnGetFs(conn), path, user, group)) { + ret = errno; + ERROR("Could not chown %s to %d:%d: error %d", path, (int)uid, gid, ret); + ret = (ret > 0) ? -ret : -EIO; + goto cleanup; + } + +cleanup: + if (conn) { + hdfsConnRelease(conn); + } + free(user); + free(group); + + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_create.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_create.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_create.c new file mode 100644 index 0000000..256e383 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_create.c @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" + +int dfs_create(const char *path, mode_t mode, struct fuse_file_info *fi) +{ + TRACE1("create", path) + fi->flags |= mode; + return dfs_open(path, fi); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_flush.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_flush.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_flush.c new file mode 100644 index 0000000..adb065b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_flush.c @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_connect.h" +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_file_handle.h" + +int dfs_flush(const char *path, struct fuse_file_info *fi) { + TRACE1("flush", path) + + // retrieve dfs specific data + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + + // check params and the context var + assert(path); + assert(dfs); + assert('/' == *path); + assert(fi); + + if (NULL == (void*)fi->fh) { + return 0; + } + + // note that fuse calls flush on RO files too and hdfs does not like that and will return an error + if (fi->flags & O_WRONLY) { + + dfs_fh *fh = (dfs_fh*)fi->fh; + assert(fh); + hdfsFile file_handle = (hdfsFile)fh->hdfsFH; + assert(file_handle); + if (hdfsFlush(hdfsConnGetFs(fh->conn), file_handle) != 0) { + ERROR("Could not flush %lx for %s\n",(long)file_handle, path); + return -EIO; + } + } + + return 0; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_getattr.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_getattr.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_getattr.c new file mode 100644 index 0000000..2e43518 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_getattr.c @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_stat_struct.h" +#include "fuse_connect.h" + +int dfs_getattr(const char *path, struct stat *st) +{ + struct hdfsConn *conn = NULL; + hdfsFS fs; + int ret; + hdfsFileInfo *info; + + TRACE1("getattr", path) + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + assert(dfs); + assert(path); + assert(st); + + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; + } + fs = hdfsConnGetFs(conn); + + info = hdfsGetPathInfo(fs,path); + if (NULL == info) { + ret = -ENOENT; + goto cleanup; + } + fill_stat_structure(&info[0], st); + + // setup hard link info - for a file it is 1 else num entries in a dir + 2 (for . and ..) + if (info[0].mKind == kObjectKindDirectory) { + int numEntries = 0; + hdfsFileInfo *info = hdfsListDirectory(fs,path,&numEntries); + + if (info) { + hdfsFreeFileInfo(info,numEntries); + } + st->st_nlink = numEntries + 2; + } else { + // not a directory + st->st_nlink = 1; + } + + // free the info pointer + hdfsFreeFileInfo(info,1); + +cleanup: + if (conn) { + hdfsConnRelease(conn); + } + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_mkdir.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_mkdir.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_mkdir.c new file mode 100644 index 0000000..b05551f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_mkdir.c @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_trash.h" +#include "fuse_connect.h" + +int dfs_mkdir(const char *path, mode_t mode) +{ + struct hdfsConn *conn = NULL; + hdfsFS fs; + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + int ret; + + TRACE1("mkdir", path) + + assert(path); + assert(dfs); + assert('/' == *path); + + if (is_protected(path)) { + ERROR("HDFS trying to create directory %s", path); + return -EACCES; + } + + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; + } + fs = hdfsConnGetFs(conn); + + // In theory the create and chmod should be atomic. + + if (hdfsCreateDirectory(fs, path)) { + ERROR("HDFS could not create directory %s", path); + ret = (errno > 0) ? -errno : -EIO; + goto cleanup; + } + + if (hdfsChmod(fs, path, (short)mode)) { + ERROR("Could not chmod %s to %d", path, (int)mode); + ret = (errno > 0) ? -errno : -EIO; + } + ret = 0; + +cleanup: + if (conn) { + hdfsConnRelease(conn); + } + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_mknod.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_mknod.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_mknod.c new file mode 100644 index 0000000..c745cf1 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_mknod.c @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" + +int dfs_mknod(const char *path, mode_t mode, dev_t rdev) +{ + TRACE1("mknod", path); + DEBUG("dfs_mknod"); + return 0; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_open.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_open.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_open.c new file mode 100644 index 0000000..ca670ce --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_open.c @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_connect.h" +#include "fuse_file_handle.h" + +#include +#include + +/** + * Given a set of FUSE flags, determine the libhdfs flags we need. + * + * This is complicated by two things: + * 1. libhdfs doesn't support O_RDWR at all; + * 2. when given O_WRONLY, libhdfs will truncate the file unless O_APPEND is + * also given. In other words, there is an implicit O_TRUNC. + * + * Probably the next iteration of the libhdfs interface should not use the POSIX + * flags at all, since, as you can see, they don't really match up very closely + * to the POSIX meaning. However, for the time being, this is the API. + * + * @param fs The libhdfs object + * @param path The path we're opening + * @param flags The FUSE flags + * + * @return negative error code on failure; flags otherwise. + */ +static int64_t get_hdfs_open_flags(hdfsFS fs, const char *path, int flags) +{ + int64_t ret; + hdfsFileInfo *info; + + if ((flags & O_ACCMODE) == O_RDONLY) { + return O_RDONLY; + } + if (flags & O_TRUNC) { + /* If we're opening for write or read/write, O_TRUNC means we should blow + * away the file which is there and create our own file. + * */ + return O_WRONLY; + } + info = hdfsGetPathInfo(fs, path); + if (info) { + if (info->mSize == 0) { + // If the file has zero length, we shouldn't feel bad about blowing it + // away. + ret = O_WRONLY; + } else if ((flags & O_ACCMODE) == O_RDWR) { + // HACK: translate O_RDWR requests into O_RDONLY if the file already + // exists and has non-zero length. + ret = O_RDONLY; + } else { // O_WRONLY + // HACK: translate O_WRONLY requests into append if the file already + // exists. + ret = O_WRONLY | O_APPEND; + } + } else { // !info + if (flags & O_CREAT) { + ret = O_WRONLY; + } else { + ret = -ENOENT; + } + } + if (info) { + hdfsFreeFileInfo(info, 1); + } + return ret; +} + +int dfs_open(const char *path, struct fuse_file_info *fi) +{ + hdfsFS fs = NULL; + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + dfs_fh *fh = NULL; + int mutexInit = 0, ret, flags = 0; + int64_t flagRet; + + TRACE1("open", path) + + // check params and the context var + assert(path); + assert('/' == *path); + assert(dfs); + + // retrieve dfs specific data + fh = (dfs_fh*)calloc(1, sizeof (dfs_fh)); + if (!fh) { + ERROR("Malloc of new file handle failed"); + ret = -EIO; + goto error; + } + ret = fuseConnectAsThreadUid(&fh->conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto error; + } + fs = hdfsConnGetFs(fh->conn); + flagRet = get_hdfs_open_flags(fs, path, fi->flags); + if (flagRet < 0) { + ret = -flagRet; + goto error; + } + flags = flagRet; + if ((fh->hdfsFH = hdfsOpenFile(fs, path, flags, 0, 0, 0)) == NULL) { + ERROR("Could not open file %s (errno=%d)", path, errno); + if (errno == 0 || errno == EINTERNAL) { + ret = -EIO; + goto error; + } + ret = -errno; + goto error; + } + + ret = pthread_mutex_init(&fh->mutex, NULL); + if (ret) { + fprintf(stderr, "dfs_open: error initializing mutex: error %d\n", ret); + ret = -EIO; + goto error; + } + mutexInit = 1; + + if ((flags & O_ACCMODE) == O_WRONLY) { + fh->buf = NULL; + } else { + assert(dfs->rdbuffer_size > 0); + fh->buf = (char*)malloc(dfs->rdbuffer_size * sizeof(char)); + if (NULL == fh->buf) { + ERROR("Could not allocate memory for a read for file %s\n", path); + ret = -EIO; + goto error; + } + fh->buffersStartOffset = 0; + fh->bufferSize = 0; + } + fi->fh = (uint64_t)fh; + return 0; + +error: + if (fh) { + if (mutexInit) { + pthread_mutex_destroy(&fh->mutex); + } + free(fh->buf); + if (fh->hdfsFH) { + hdfsCloseFile(fs, fh->hdfsFH); + } + if (fh->conn) { + hdfsConnRelease(fh->conn); + } + free(fh); + } + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_read.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_read.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_read.c new file mode 100644 index 0000000..feade45 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_read.c @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_connect.h" +#include "fuse_dfs.h" +#include "fuse_file_handle.h" +#include "fuse_impls.h" + +static size_t min(const size_t x, const size_t y) { + return x < y ? x : y; +} + +/** + * dfs_read + * + * Reads from dfs or the open file's buffer. Note that fuse requires that + * either the entire read be satisfied or the EOF is hit or direct_io is enabled + * + */ +int dfs_read(const char *path, char *buf, size_t size, off_t offset, + struct fuse_file_info *fi) +{ + TRACE1("read",path) + + // retrieve dfs specific data + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + + // check params and the context var + assert(dfs); + assert(path); + assert(buf); + assert(offset >= 0); + assert(size >= 0); + assert(fi); + + dfs_fh *fh = (dfs_fh*)fi->fh; + hdfsFS fs = hdfsConnGetFs(fh->conn); + + assert(fh != NULL); + assert(fh->hdfsFH != NULL); + + // special case this as simplifies the rest of the logic to know the caller wanted > 0 bytes + if (size == 0) + return 0; + + // If size is bigger than the read buffer, then just read right into the user supplied buffer + if ( size >= dfs->rdbuffer_size) { + int num_read; + size_t total_read = 0; + while (size - total_read > 0 && (num_read = hdfsPread(fs, fh->hdfsFH, offset + total_read, buf + total_read, size - total_read)) > 0) { + total_read += num_read; + } + // if there was an error before satisfying the current read, this logic declares it an error + // and does not try to return any of the bytes read. Don't think it matters, so the code + // is just being conservative. + if (total_read < size && num_read < 0) { + total_read = -EIO; + } + return total_read; + } + + // + // Critical section - protect from multiple reads in different threads accessing the read buffer + // (no returns until end) + // + + pthread_mutex_lock(&fh->mutex); + + // used only to check the postcondition of this function - namely that we satisfy + // the entire read or EOF is hit. + int isEOF = 0; + int ret = 0; + + // check if the buffer is empty or + // the read starts before the buffer starts or + // the read ends after the buffer ends + + if (fh->bufferSize == 0 || + offset < fh->buffersStartOffset || + offset + size > fh->buffersStartOffset + fh->bufferSize) + { + // Read into the buffer from DFS + int num_read = 0; + size_t total_read = 0; + + while (dfs->rdbuffer_size - total_read > 0 && + (num_read = hdfsPread(fs, fh->hdfsFH, offset + total_read, fh->buf + total_read, dfs->rdbuffer_size - total_read)) > 0) { + total_read += num_read; + } + + // if there was an error before satisfying the current read, this logic declares it an error + // and does not try to return any of the bytes read. Don't think it matters, so the code + // is just being conservative. + if (total_read < size && num_read < 0) { + // invalidate the buffer + fh->bufferSize = 0; + ERROR("pread failed for %s with return code %d", path, (int)num_read); + ret = -EIO; + } else { + // Either EOF, all read or read beyond size, but then there was an error + fh->bufferSize = total_read; + fh->buffersStartOffset = offset; + + if (dfs->rdbuffer_size - total_read > 0) { + // assert(num_read == 0); this should be true since if num_read < 0 handled above. + isEOF = 1; + } + } + } + + // + // NOTE on EOF, fh->bufferSize == 0 and ret = 0 ,so the logic for copying data into the caller's buffer is bypassed, and + // the code returns 0 as required + // + if (ret == 0 && fh->bufferSize > 0) { + + assert(offset >= fh->buffersStartOffset); + assert(fh->buf); + + const size_t bufferReadIndex = offset - fh->buffersStartOffset; + assert(bufferReadIndex >= 0 && bufferReadIndex < fh->bufferSize); + + const size_t amount = min(fh->buffersStartOffset + fh->bufferSize - offset, size); + assert(amount >= 0 && amount <= fh->bufferSize); + + const char *offsetPtr = fh->buf + bufferReadIndex; + assert(offsetPtr >= fh->buf); + assert(offsetPtr + amount <= fh->buf + fh->bufferSize); + + memcpy(buf, offsetPtr, amount); + + ret = amount; + } + + // + // Critical section end + // + pthread_mutex_unlock(&fh->mutex); + + // fuse requires the below and the code should guarantee this assertion + // 3 cases on return: + // 1. entire read satisfied + // 2. partial read and isEOF - including 0 size read + // 3. error + assert(ret == size || isEOF || ret < 0); + + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_readdir.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_readdir.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_readdir.c new file mode 100644 index 0000000..326f573 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_readdir.c @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_stat_struct.h" +#include "fuse_connect.h" + +int dfs_readdir(const char *path, void *buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info *fi) +{ + int ret; + struct hdfsConn *conn = NULL; + hdfsFS fs; + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + + TRACE1("readdir", path) + + assert(dfs); + assert(path); + assert(buf); + + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; + } + fs = hdfsConnGetFs(conn); + + // Read dirents. Calling a variant that just returns the final path + // component (HDFS-975) would save us from parsing it out below. + int numEntries = 0; + hdfsFileInfo *info = hdfsListDirectory(fs, path, &numEntries); + + // NULL means either the directory doesn't exist or maybe IO error. + if (NULL == info) { + ret = (errno > 0) ? -errno : -ENOENT; + goto cleanup; + } + + int i ; + for (i = 0; i < numEntries; i++) { + if (NULL == info[i].mName) { + ERROR("Path %s info[%d].mName is NULL", path, i); + continue; + } + + struct stat st; + fill_stat_structure(&info[i], &st); + + // Find the final path component + const char *str = strrchr(info[i].mName, '/'); + if (NULL == str) { + ERROR("Invalid URI %s", info[i].mName); + continue; + } + str++; + + // pack this entry into the fuse buffer + int res = 0; + if ((res = filler(buf,str,&st,0)) != 0) { + ERROR("Readdir filler failed: %d\n",res); + } + } + + // insert '.' and '..' + const char *const dots [] = { ".",".."}; + for (i = 0 ; i < 2 ; i++) + { + struct stat st; + memset(&st, 0, sizeof(struct stat)); + + // set to 0 to indicate not supported for directory because we cannot (efficiently) get this info for every subdirectory + st.st_nlink = 0; + + // setup stat size and acl meta data + st.st_size = 512; + st.st_blksize = 512; + st.st_blocks = 1; + st.st_mode = (S_IFDIR | 0777); + st.st_uid = default_id; + st.st_gid = default_id; + // todo fix below times + st.st_atime = 0; + st.st_mtime = 0; + st.st_ctime = 0; + + const char *const str = dots[i]; + + // flatten the info using fuse's function into a buffer + int res = 0; + if ((res = filler(buf,str,&st,0)) != 0) { + ERROR("Readdir filler failed: %d\n",res); + } + } + // free the info pointers + hdfsFreeFileInfo(info,numEntries); + ret = 0; + +cleanup: + if (conn) { + hdfsConnRelease(conn); + } + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_release.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_release.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_release.c new file mode 100644 index 0000000..0316de6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_release.c @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_file_handle.h" +#include "fuse_connect.h" + +#include + +/** + * release a fuse_file_info structure. + * + * When this function is invoked, there are no more references to our + * fuse_file_info structure that exist anywhere. So there is no need for + * locking to protect this structure here. + * + * Another thread could open() the same file, and get a separate, different file + * descriptor with a different, separate fuse_file_info structure. In HDFS, + * this results in one writer winning and overwriting everything the other + * writer has done. + */ + +int dfs_release (const char *path, struct fuse_file_info *fi) { + TRACE1("release", path) + + // retrieve dfs specific data + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + + // check params and the context var + assert(path); + assert(dfs); + assert('/' == *path); + + int ret = 0; + dfs_fh *fh = (dfs_fh*)fi->fh; + assert(fh); + hdfsFile file_handle = (hdfsFile)fh->hdfsFH; + if (NULL != file_handle) { + if (hdfsCloseFile(hdfsConnGetFs(fh->conn), file_handle) != 0) { + ERROR("Could not close handle %ld for %s\n",(long)file_handle, path); + ret = -EIO; + } + } + free(fh->buf); + hdfsConnRelease(fh->conn); + pthread_mutex_destroy(&fh->mutex); + free(fh); + fi->fh = 0; + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_rename.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_rename.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_rename.c new file mode 100644 index 0000000..ad7c7e5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_rename.c @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_trash.h" +#include "fuse_connect.h" + +int dfs_rename(const char *from, const char *to) +{ + struct hdfsConn *conn = NULL; + hdfsFS fs; + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + int ret; + + TRACE1("rename", from) + + // check params and the context var + assert(from); + assert(to); + assert(dfs); + + assert('/' == *from); + assert('/' == *to); + + if (is_protected(from) || is_protected(to)) { + ERROR("Could not rename %s to %s", from, to); + return -EACCES; + } + + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; + } + fs = hdfsConnGetFs(conn); + if (hdfsRename(fs, from, to)) { + ERROR("Rename %s to %s failed", from, to); + ret = (errno > 0) ? -errno : -EIO; + goto cleanup; + } + ret = 0; + +cleanup: + if (conn) { + hdfsConnRelease(conn); + } + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_rmdir.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_rmdir.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_rmdir.c new file mode 100644 index 0000000..493807f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_rmdir.c @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_trash.h" +#include "fuse_connect.h" + +extern const char *const TrashPrefixDir; + +int dfs_rmdir(const char *path) +{ + struct hdfsConn *conn = NULL; + hdfsFS fs; + int ret; + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + int numEntries = 0; + hdfsFileInfo *info = NULL; + + TRACE1("rmdir", path) + + assert(path); + assert(dfs); + assert('/' == *path); + + if (is_protected(path)) { + ERROR("Trying to delete protected directory %s", path); + ret = -EACCES; + goto cleanup; + } + + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; + } + fs = hdfsConnGetFs(conn); + info = hdfsListDirectory(fs, path, &numEntries); + if (numEntries) { + ret = -ENOTEMPTY; + goto cleanup; + } + + if (hdfsDeleteWithTrash(fs, path, dfs->usetrash)) { + ERROR("Error trying to delete directory %s", path); + ret = -EIO; + goto cleanup; + } + ret = 0; + +cleanup: + if (info) { + hdfsFreeFileInfo(info, numEntries); + } + if (conn) { + hdfsConnRelease(conn); + } + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_statfs.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_statfs.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_statfs.c new file mode 100644 index 0000000..c9306a2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_statfs.c @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_connect.h" + + +int dfs_statfs(const char *path, struct statvfs *st) +{ + struct hdfsConn *conn = NULL; + hdfsFS fs; + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + int ret; + + TRACE1("statfs",path) + + assert(path); + assert(st); + assert(dfs); + + memset(st,0,sizeof(struct statvfs)); + + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; + } + fs = hdfsConnGetFs(conn); + + const tOffset cap = hdfsGetCapacity(fs); + const tOffset used = hdfsGetUsed(fs); + const tOffset bsize = hdfsGetDefaultBlockSize(fs); + + st->f_bsize = bsize; + st->f_frsize = bsize; + st->f_blocks = cap/bsize; + st->f_bfree = (cap-used)/bsize; + st->f_bavail = (cap-used)/bsize; + st->f_files = 1000; + st->f_ffree = 500; + st->f_favail = 500; + st->f_fsid = 1023; + st->f_flag = ST_RDONLY | ST_NOSUID; + st->f_namemax = 1023; + ret = 0; + +cleanup: + if (conn) { + hdfsConnRelease(conn); + } + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_symlink.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_symlink.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_symlink.c new file mode 100644 index 0000000..be6e7eb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_symlink.c @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" + + +int dfs_symlink(const char *from, const char *to) +{ + TRACE1("symlink", from) + (void)from; + (void)to; + // bugbug we need the FileSystem to support this posix API + return -ENOTSUP; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_truncate.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_truncate.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_truncate.c new file mode 100644 index 0000000..bf72ca6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_truncate.c @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_connect.h" + +/** + * For now implement truncate here and only for size == 0. + * Weak implementation in that we just delete the file and + * then re-create it, but don't set the user, group, and times to the old + * file's metadata. + */ +int dfs_truncate(const char *path, off_t size) +{ + struct hdfsConn *conn = NULL; + hdfsFS fs; + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + + TRACE1("truncate", path) + + assert(path); + assert('/' == *path); + assert(dfs); + + if (size != 0) { + return 0; + } + + int ret = dfs_unlink(path); + if (ret != 0) { + return ret; + } + + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; + } + fs = hdfsConnGetFs(conn); + + int flags = O_WRONLY | O_CREAT; + + hdfsFile file; + if ((file = (hdfsFile)hdfsOpenFile(fs, path, flags, 0, 0, 0)) == NULL) { + ERROR("Could not connect open file %s", path); + ret = -EIO; + goto cleanup; + } + + if (hdfsCloseFile(fs, file) != 0) { + ERROR("Could not close file %s", path); + ret = -EIO; + goto cleanup; + } + +cleanup: + if (conn) { + hdfsConnRelease(conn); + } + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_unlink.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_unlink.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_unlink.c new file mode 100644 index 0000000..45a4501 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_unlink.c @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_connect.h" +#include "fuse_trash.h" + +int dfs_unlink(const char *path) +{ + struct hdfsConn *conn = NULL; + hdfsFS fs; + int ret = 0; + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + + TRACE1("unlink", path) + + assert(path); + assert(dfs); + assert('/' == *path); + + if (is_protected(path)) { + ERROR("Trying to delete protected directory %s", path); + ret = -EACCES; + goto cleanup; + } + + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; + } + fs = hdfsConnGetFs(conn); + + if (hdfsDeleteWithTrash(fs, path, dfs->usetrash)) { + ERROR("Could not delete file %s", path); + ret = (errno > 0) ? -errno : -EIO; + goto cleanup; + } + ret = 0; + +cleanup: + if (conn) { + hdfsConnRelease(conn); + } + return ret; + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_utimens.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_utimens.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_utimens.c new file mode 100644 index 0000000..dccff92 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_utimens.c @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_connect.h" + +int dfs_utimens(const char *path, const struct timespec ts[2]) +{ + struct hdfsConn *conn = NULL; + hdfsFS fs; + int ret = 0; + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + + TRACE1("utimens", path) + + assert(path); + assert(dfs); + assert('/' == *path); + + time_t aTime = ts[0].tv_sec; + time_t mTime = ts[1].tv_sec; + + ret = fuseConnectAsThreadUid(&conn); + if (ret) { + fprintf(stderr, "fuseConnectAsThreadUid: failed to open a libhdfs " + "connection! error %d.\n", ret); + ret = -EIO; + goto cleanup; + } + fs = hdfsConnGetFs(conn); + + if (hdfsUtime(fs, path, mTime, aTime)) { + hdfsFileInfo *info = hdfsGetPathInfo(fs, path); + if (info == NULL) { + ret = (errno > 0) ? -errno : -ENOENT; + goto cleanup; + } + // Silently ignore utimens failure for directories, otherwise + // some programs like tar will fail. + if (info->mKind == kObjectKindDirectory) { + ret = 0; + } else { + ret = (errno > 0) ? -errno : -EACCES; + } + goto cleanup; + } + ret = 0; + +cleanup: + if (conn) { + hdfsConnRelease(conn); + } + return ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_write.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_write.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_write.c new file mode 100644 index 0000000..3090e9e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_impls_write.c @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_connect.h" +#include "fuse_dfs.h" +#include "fuse_impls.h" +#include "fuse_file_handle.h" + +int dfs_write(const char *path, const char *buf, size_t size, + off_t offset, struct fuse_file_info *fi) +{ + TRACE1("write", path) + + // retrieve dfs specific data + dfs_context *dfs = (dfs_context*)fuse_get_context()->private_data; + int ret = 0; + + // check params and the context var + assert(path); + assert(dfs); + assert('/' == *path); + assert(fi); + + dfs_fh *fh = (dfs_fh*)fi->fh; + assert(fh); + + hdfsFile file_handle = (hdfsFile)fh->hdfsFH; + assert(file_handle); + + // + // Critical section - make the sanity check (tell to see the writes are sequential) and the actual write + // (no returns until end) + // + pthread_mutex_lock(&fh->mutex); + + tSize length = 0; + hdfsFS fs = hdfsConnGetFs(fh->conn); + + tOffset cur_offset = hdfsTell(fs, file_handle); + if (cur_offset != offset) { + ERROR("User trying to random access write to a file %d != %d for %s", + (int)cur_offset, (int)offset, path); + ret = -ENOTSUP; + } else { + length = hdfsWrite(fs, file_handle, buf, size); + if (length <= 0) { + ERROR("Could not write all bytes for %s %d != %d (errno=%d)", + path, length, (int)size, errno); + if (errno == 0 || errno == EINTERNAL) { + ret = -EIO; + } else { + ret = -errno; + } + } + if (length != size) { + ERROR("Could not write all bytes for %s %d != %d (errno=%d)", + path, length, (int)size, errno); + } + } + + // + // Critical section end + // + + pthread_mutex_unlock(&fh->mutex); + + return ret == 0 ? length : ret; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c new file mode 100644 index 0000000..4da6da0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.c @@ -0,0 +1,192 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_dfs.h" +#include "fuse_init.h" +#include "fuse_options.h" +#include "fuse_context_handle.h" +#include "fuse_connect.h" + +#include +#include +#include + +static void print_env_vars(void) +{ + const char *cp = getenv("CLASSPATH"); + const char *ld = getenv("LD_LIBRARY_PATH"); + + ERROR("LD_LIBRARY_PATH=%s",ld == NULL ? "NULL" : ld); + ERROR("CLASSPATH=%s",cp == NULL ? "NULL" : cp); +} + +// Hacked up function to basically do: +// protectedpaths = split(options.protected,':'); + +static void init_protectedpaths(dfs_context *dfs) +{ + char *tmp = options.protected; + + // handle degenerate case up front. + if (tmp == NULL || 0 == *tmp) { + dfs->protectedpaths = (char**)malloc(sizeof(char*)); + dfs->protectedpaths[0] = NULL; + return; + } + + if (options.debug) { + print_options(); + } + + int i = 0; + while (tmp && (NULL != (tmp = index(tmp,':')))) { + tmp++; // pass the , + i++; + } + i++; // for the last entry + i++; // for the final NULL + dfs->protectedpaths = (char**)malloc(sizeof(char*)*i); + assert(dfs->protectedpaths); + tmp = options.protected; + int j = 0; + while (NULL != tmp && j < i) { + int length; + char *eos = index(tmp,':'); + if (NULL != eos) { + length = eos - tmp; // length of this value + } else { + length = strlen(tmp); + } + dfs->protectedpaths[j] = (char*)malloc(sizeof(char)*length+1); + assert(dfs->protectedpaths[j]); + strncpy(dfs->protectedpaths[j], tmp, length); + dfs->protectedpaths[j][length] = '\0'; + if (eos) { + tmp = eos + 1; + } else { + tmp = NULL; + } + j++; + } + dfs->protectedpaths[j] = NULL; +} + +static void dfsPrintOptions(FILE *fp, const struct options *o) +{ + INFO("Mounting with options: [ protected=%s, nn_uri=%s, nn_port=%d, " + "debug=%d, read_only=%d, initchecks=%d, " + "no_permissions=%d, usetrash=%d, entry_timeout=%d, " + "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d ]", + (o->protected ? o->protected : "(NULL)"), o->nn_uri, o->nn_port, + o->debug, o->read_only, o->initchecks, + o->no_permissions, o->usetrash, o->entry_timeout, + o->attribute_timeout, o->rdbuffer_size, o->direct_io); +} + +void *dfs_init(struct fuse_conn_info *conn) +{ + int ret; + + // + // Create a private struct of data we will pass to fuse here and which + // will then be accessible on every call. + // + dfs_context *dfs = calloc(1, sizeof(*dfs)); + if (!dfs) { + ERROR("FATAL: could not malloc dfs_context"); + exit(1); + } + + // initialize the context + dfs->debug = options.debug; + dfs->usetrash = options.usetrash; + dfs->protectedpaths = NULL; + dfs->rdbuffer_size = options.rdbuffer_size; + dfs->direct_io = options.direct_io; + + dfsPrintOptions(stderr, &options); + + init_protectedpaths(dfs); + assert(dfs->protectedpaths != NULL); + + if (dfs->rdbuffer_size <= 0) { + DEBUG("dfs->rdbuffersize <= 0 = %zd", dfs->rdbuffer_size); + dfs->rdbuffer_size = 32768; + } + + ret = fuseConnectInit(options.nn_uri, options.nn_port); + if (ret) { + ERROR("FATAL: dfs_init: fuseConnectInit failed with error %d!", ret); + print_env_vars(); + exit(EXIT_FAILURE); + } + if (options.initchecks == 1) { + ret = fuseConnectTest(); + if (ret) { + ERROR("FATAL: dfs_init: fuseConnectTest failed with error %d!", ret); + print_env_vars(); + exit(EXIT_FAILURE); + } + } + +#ifdef FUSE_CAP_ATOMIC_O_TRUNC + // If FUSE_CAP_ATOMIC_O_TRUNC is set, open("foo", O_CREAT | O_TRUNC) will + // result in dfs_open being called with O_TRUNC. + // + // If this capability is not present, fuse will try to use multiple + // operation to "simulate" open(O_TRUNC). This doesn't work very well with + // HDFS. + // Unfortunately, this capability is only implemented on Linux 2.6.29 or so. + // See HDFS-4140 for details. + if (conn->capable & FUSE_CAP_ATOMIC_O_TRUNC) { + conn->want |= FUSE_CAP_ATOMIC_O_TRUNC; + } +#endif + +#ifdef FUSE_CAP_ASYNC_READ + // We're OK with doing reads at the same time as writes. + if (conn->capable & FUSE_CAP_ASYNC_READ) { + conn->want |= FUSE_CAP_ASYNC_READ; + } +#endif + +#ifdef FUSE_CAP_BIG_WRITES + // Yes, we can read more than 4kb at a time. In fact, please do! + if (conn->capable & FUSE_CAP_BIG_WRITES) { + conn->want |= FUSE_CAP_BIG_WRITES; + } +#endif + +#ifdef FUSE_CAP_DONT_MASK + if ((options.no_permissions) && (conn->capable & FUSE_CAP_DONT_MASK)) { + // If we're handing permissions ourselves, we don't want the kernel + // applying its own umask. HDFS already implements its own per-user + // umasks! Sadly, this only actually does something on kernels 2.6.31 and + // later. + conn->want |= FUSE_CAP_DONT_MASK; + } +#endif + + return (void*)dfs; +} + + +void dfs_destroy(void *ptr) +{ + TRACE("destroy") +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.h new file mode 100644 index 0000000..681ab91 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_init.h @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __FUSE_INIT_H__ +#define __FUSE_INIT_H__ + +struct fuse_conn_info; + +/** + * These are responsible for initializing connections to dfs and internal + * data structures and then freeing them. + * i.e., what happens on mount and unmount. + * + */ +void *dfs_init(struct fuse_conn_info *conn); +void dfs_destroy (void *ptr); + +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c new file mode 100644 index 0000000..8461ce4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.c @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fuse_context_handle.h" +#include "fuse_dfs.h" +#include "fuse_options.h" + +#include +#include + +#define OLD_HDFS_URI_LOCATION "dfs://" +#define NEW_HDFS_URI_LOCATION "hdfs://" + +void print_options() { + printf("options:\n" + "\tprotected=%s\n" + "\tserver=%s\n" + "\tport=%d\n" + "\tdebug=%d\n" + "\tread_only=%d\n" + "\tusetrash=%d\n" + "\tentry_timeout=%d\n" + "\tattribute_timeout=%d\n" + "\tprivate=%d\n" + "\trdbuffer_size=%d (KBs)\n", + options.protected, options.nn_uri, options.nn_port, options.debug, + options.read_only, options.usetrash, options.entry_timeout, + options.attribute_timeout, options.private, + (int)options.rdbuffer_size / 1024); +} + +const char *program; + +/** macro to define options */ +#define DFSFS_OPT_KEY(t, p, v) { t, offsetof(struct options, p), v } + +void print_usage(const char *pname) +{ + printf("USAGE: %s [debug] [--help] [--version] " + "[-oprotected=] [-oport=] " + "[-oentry_timeout=] [-oattribute_timeout=] " + "[-odirect_io] [-onopoermissions] [-o] " + " [fuse options]\n", pname); + printf("NOTE: debugging option for fuse is -debug\n"); +} + + +/** keys for FUSE_OPT_ options */ +enum + { + KEY_VERSION, + KEY_HELP, + KEY_USETRASH, + KEY_NOTRASH, + KEY_RO, + KEY_RW, + KEY_PRIVATE, + KEY_BIGWRITES, + KEY_DEBUG, + KEY_INITCHECKS, + KEY_NOPERMISSIONS, + KEY_DIRECTIO, + }; + +struct fuse_opt dfs_opts[] = + { + DFSFS_OPT_KEY("server=%s", nn_uri, 0), + DFSFS_OPT_KEY("entry_timeout=%d", entry_timeout, 0), + DFSFS_OPT_KEY("attribute_timeout=%d", attribute_timeout, 0), + DFSFS_OPT_KEY("protected=%s", protected, 0), + DFSFS_OPT_KEY("port=%d", nn_port, 0), + DFSFS_OPT_KEY("rdbuffer=%d", rdbuffer_size,0), + + FUSE_OPT_KEY("private", KEY_PRIVATE), + FUSE_OPT_KEY("ro", KEY_RO), + FUSE_OPT_KEY("debug", KEY_DEBUG), + FUSE_OPT_KEY("initchecks", KEY_INITCHECKS), + FUSE_OPT_KEY("nopermissions", KEY_NOPERMISSIONS), + FUSE_OPT_KEY("big_writes", KEY_BIGWRITES), + FUSE_OPT_KEY("rw", KEY_RW), + FUSE_OPT_KEY("usetrash", KEY_USETRASH), + FUSE_OPT_KEY("notrash", KEY_NOTRASH), + FUSE_OPT_KEY("direct_io", KEY_DIRECTIO), + FUSE_OPT_KEY("-v", KEY_VERSION), + FUSE_OPT_KEY("--version", KEY_VERSION), + FUSE_OPT_KEY("-h", KEY_HELP), + FUSE_OPT_KEY("--help", KEY_HELP), + FUSE_OPT_END + }; + +int dfs_options(void *data, const char *arg, int key, struct fuse_args *outargs) +{ + (void) data; + int nn_uri_len; + + switch (key) { + case FUSE_OPT_KEY_OPT: + INFO("Ignoring option %s", arg); + return 1; + case KEY_VERSION: + INFO("%s %s\n", program, _FUSE_DFS_VERSION); + exit(0); + case KEY_HELP: + print_usage(program); + exit(0); + case KEY_USETRASH: + options.usetrash = 1; + break; + case KEY_NOTRASH: + options.usetrash = 0; + break; + case KEY_RO: + options.read_only = 1; + break; + case KEY_RW: + options.read_only = 0; + break; + case KEY_PRIVATE: + options.private = 1; + break; + case KEY_DEBUG: + fuse_opt_add_arg(outargs, "-d"); + options.debug = 1; + break; + case KEY_INITCHECKS: + options.initchecks = 1; + break; + case KEY_NOPERMISSIONS: + options.no_permissions = 1; + break; + case KEY_DIRECTIO: + options.direct_io = 1; + break; + case KEY_BIGWRITES: +#ifdef FUSE_CAP_BIG_WRITES + fuse_opt_add_arg(outargs, "-obig_writes"); +#endif + break; + default: { + // try and see if the arg is a URI + if (!strstr(arg, "://")) { + if (strcmp(arg,"ro") == 0) { + options.read_only = 1; + } else if (strcmp(arg,"rw") == 0) { + options.read_only = 0; + } else { + INFO("Adding FUSE arg %s", arg); + fuse_opt_add_arg(outargs, arg); + return 0; + } + } else { + if (options.nn_uri) { + INFO("Ignoring option %s because '-server' was already " + "specified!", arg); + return 1; + } + if (strstr(arg, OLD_HDFS_URI_LOCATION) == arg) { + // For historical reasons, we let people refer to hdfs:// as dfs:// + nn_uri_len = strlen(NEW_HDFS_URI_LOCATION) + + strlen(arg + strlen(OLD_HDFS_URI_LOCATION)) + 1; + options.nn_uri = malloc(nn_uri_len); + snprintf(options.nn_uri, nn_uri_len, "%s%s", NEW_HDFS_URI_LOCATION, + arg + strlen(OLD_HDFS_URI_LOCATION)); + } else { + options.nn_uri = strdup(arg); + } + } + } + } + return 0; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h new file mode 100644 index 0000000..4bfc235 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_options.h @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __FUSE_OPTIONS_H__ +#define __FUSE_OPTIONS_H__ + +/** options for fuse_opt.h */ +struct options { + char* protected; + char* nn_uri; + int nn_port; + int debug; + int read_only; + int initchecks; + int no_permissions; + int usetrash; + int entry_timeout; + int attribute_timeout; + int private; + size_t rdbuffer_size; + int direct_io; +} options; + +extern struct fuse_opt dfs_opts[]; +void print_options(); +void print_usage(const char *pname); +int dfs_options(void *data, const char *arg, int key, struct fuse_args *outargs); + +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.c new file mode 100644 index 0000000..e3a0725 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.c @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "fuse_dfs.h" +#include "fuse_stat_struct.h" +#include "fuse_context_handle.h" + +/* + * getpwuid and getgrgid return static structs so we safeguard the contents + * while retrieving fields using the 2 structs below. + * NOTE: if using both, always get the passwd struct firt! + */ +extern pthread_mutex_t passwdstruct_mutex; +extern pthread_mutex_t groupstruct_mutex; + +const int default_id = 99; // nobody - not configurable since soon uids in dfs, yeah! +const int blksize = 512; + +/** + * Converts from a hdfs hdfsFileInfo to a POSIX stat struct + * + */ +int fill_stat_structure(hdfsFileInfo *info, struct stat *st) +{ + assert(st); + assert(info); + + // initialize the stat structure + memset(st, 0, sizeof(struct stat)); + + // by default: set to 0 to indicate not supported for directory because we cannot (efficiently) get this info for every subdirectory + st->st_nlink = (info->mKind == kObjectKindDirectory) ? 0 : 1; + + uid_t owner_id = default_id; + if (info->mOwner != NULL) { + // + // Critical section - protect from concurrent calls in different threads since + // the struct below is static. + // (no returns until end) + // + pthread_mutex_lock(&passwdstruct_mutex); + + struct passwd *passwd_info = getpwnam(info->mOwner); + owner_id = passwd_info == NULL ? default_id : passwd_info->pw_uid; + + // + // End critical section + // + pthread_mutex_unlock(&passwdstruct_mutex); + + } + + gid_t group_id = default_id; + + if (info->mGroup != NULL) { + // + // Critical section - protect from concurrent calls in different threads since + // the struct below is static. + // (no returns until end) + // + pthread_mutex_lock(&groupstruct_mutex); + + struct group *grp = getgrnam(info->mGroup); + group_id = grp == NULL ? default_id : grp->gr_gid; + + // + // End critical section + // + pthread_mutex_unlock(&groupstruct_mutex); + + } + + short perm = (info->mKind == kObjectKindDirectory) ? (S_IFDIR | 0777) : (S_IFREG | 0666); + if (info->mPermissions > 0) { + perm = (info->mKind == kObjectKindDirectory) ? S_IFDIR: S_IFREG ; + perm |= info->mPermissions; + } + + // set stat metadata + st->st_size = (info->mKind == kObjectKindDirectory) ? 4096 : info->mSize; + st->st_blksize = blksize; + st->st_blocks = ceil(st->st_size/st->st_blksize); + st->st_mode = perm; + st->st_uid = owner_id; + st->st_gid = group_id; + st->st_atime = info->mLastAccess; + st->st_mtime = info->mLastMod; + st->st_ctime = info->mLastMod; + + return 0; +} + http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.h new file mode 100644 index 0000000..d42a371 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_stat_struct.h @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __FUSE_STAT_STRUCT_H__ +#define __FUSE_STAT_STRUCT_H__ + +#include +#include +#include + +#include "hdfs.h" + +/** + * Converts from a hdfs hdfsFileInfo to a POSIX stat struct + * Should be thread safe. + */ +int fill_stat_structure(hdfsFileInfo *info, struct stat *st) ; + +extern const int default_id; +extern const int blksize; +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c new file mode 100644 index 0000000..5e58087 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.c @@ -0,0 +1,244 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "fuse_context_handle.h" +#include "fuse_dfs.h" +#include "fuse_trash.h" +#include "fuse_users.h" + +#define TRASH_RENAME_TRIES 100 +#define ALREADY_IN_TRASH_ERR 9000 + +/** + * Split a path into a parent directory and a base path component. + * + * @param abs_path The absolute path. + * @param pcomp (out param) Will be set to the last path component. + * Malloced. + * @param parent_dir (out param) Will be set to the parent directory. + * Malloced. + * + * @return 0 on success. + * On success, both *pcomp and *parent_dir will contain + * malloc'ed strings. + * EINVAL if the path wasn't absolute. + * EINVAL if there is no parent directory (i.e. abs_path=/) + * ENOMEM if we ran out of memory. + */ +static int get_parent_dir(const char *abs_path, char **pcomp, + char **parent_dir) +{ + int ret; + char *pdir = NULL, *pc = NULL, *last_slash; + + pdir = strdup(abs_path); + if (!pdir) { + ret = ENOMEM; + goto done; + } + last_slash = rindex(pdir, '/'); + if (!last_slash) { + ERROR("get_parent_dir(%s): expected absolute path.\n", abs_path); + ret = EINVAL; + goto done; + } + if (last_slash[1] == '\0') { + *last_slash = '\0'; + last_slash = rindex(pdir, '/'); + if (!last_slash) { + ERROR("get_parent_dir(%s): there is no parent dir.\n", abs_path); + ret = EINVAL; + goto done; + } + } + pc = strdup(last_slash + 1); + if (!pc) { + ret = ENOMEM; + goto done; + } + *last_slash = '\0'; + ret = 0; +done: + if (ret) { + free(pdir); + free(pc); + return ret; + } + *pcomp = pc; + *parent_dir = pdir; + return 0; +} + +/** + * Get the base path to the trash. This will depend on the user ID. + * For example, a user whose ID maps to 'foo' will get back the path + * "/user/foo/.Trash/Current". + * + * @param trash_base (out param) the base path to the trash. + * Malloced. + * + * @return 0 on success; error code otherwise. + */ +static int get_trash_base(char **trash_base) +{ + const char * const PREFIX = "/user/"; + const char * const SUFFIX = "/.Trash/Current"; + char *user_name = NULL, *base = NULL; + uid_t uid = fuse_get_context()->uid; + int ret; + + user_name = getUsername(uid); + if (!user_name) { + ERROR("get_trash_base(): failed to get username for uid %"PRId64"\n", + (uint64_t)uid); + ret = EIO; + goto done; + } + if (asprintf(&base, "%s%s%s", PREFIX, user_name, SUFFIX) < 0) { + base = NULL; + ret = ENOMEM; + goto done; + } + ret = 0; +done: + free(user_name); + if (ret) { + free(base); + return ret; + } + *trash_base = base; + return 0; +} + +// +// NOTE: this function is a c implementation of org.apache.hadoop.fs.Trash.moveToTrash(Path path). +// +int move_to_trash(const char *abs_path, hdfsFS userFS) +{ + int ret; + char *pcomp = NULL, *parent_dir = NULL, *trash_base = NULL; + char *target_dir = NULL, *target = NULL; + + ret = get_parent_dir(abs_path, &pcomp, &parent_dir); + if (ret) { + goto done; + } + ret = get_trash_base(&trash_base); + if (ret) { + goto done; + } + int trash_base_len = strlen(trash_base); + if (!strncmp(trash_base, abs_path, trash_base_len) + && (strlen(abs_path) == trash_base_len || abs_path[trash_base_len] == '/')) { + INFO("move_to_trash(%s): file is already in the trash; deleting.", + abs_path); + ret = ALREADY_IN_TRASH_ERR; + goto done; + } + if (asprintf(&target_dir, "%s%s", trash_base, parent_dir) < 0) { + ret = ENOMEM; + target_dir = NULL; + goto done; + } + if (asprintf(&target, "%s/%s", target_dir, pcomp) < 0) { + ret = ENOMEM; + target = NULL; + goto done; + } + // create the target trash directory in trash (if needed) + if (hdfsExists(userFS, target_dir) != 0) { + // make the directory to put it in in the Trash - NOTE + // hdfsCreateDirectory also creates parents, so Current will be created if it does not exist. + if (hdfsCreateDirectory(userFS, target_dir)) { + ret = errno; + ERROR("move_to_trash(%s) error: hdfsCreateDirectory(%s) failed with error %d", + abs_path, target_dir, ret); + goto done; + } + } else if (hdfsExists(userFS, target) == 0) { + // If there is already a file in the trash with this path, append a number. + int idx; + for (idx = 1; idx < TRASH_RENAME_TRIES; idx++) { + free(target); + if (asprintf(&target, "%s/%s.%d", target_dir, pcomp, idx) < 0) { + target = NULL; + ret = ENOMEM; + goto done; + } + if (hdfsExists(userFS, target) != 0) { + break; + } + } + if (idx == TRASH_RENAME_TRIES) { + ERROR("move_to_trash(%s) error: there are already %d files in the trash " + "with this name.\n", abs_path, TRASH_RENAME_TRIES); + ret = EINVAL; + goto done; + } + } + if (hdfsRename(userFS, abs_path, target)) { + ret = errno; + ERROR("move_to_trash(%s): failed to rename the file to %s: error %d", + abs_path, target, ret); + goto done; + } + + ret = 0; +done: + if ((ret != 0) && (ret != ALREADY_IN_TRASH_ERR)) { + ERROR("move_to_trash(%s) failed with error %d", abs_path, ret); + } + free(pcomp); + free(parent_dir); + free(trash_base); + free(target_dir); + free(target); + return ret; +} + +int hdfsDeleteWithTrash(hdfsFS userFS, const char *path, int useTrash) +{ + int tried_to_move_to_trash = 0; + if (useTrash) { + tried_to_move_to_trash = 1; + if (move_to_trash(path, userFS) == 0) { + return 0; + } + } + if (hdfsDelete(userFS, path, 1)) { + int err = errno; + if (err < 0) { + err = -err; + } + ERROR("hdfsDeleteWithTrash(%s): hdfsDelete failed: error %d.", + path, err); + return -err; + } + if (tried_to_move_to_trash) { + ERROR("hdfsDeleteWithTrash(%s): deleted the file instead.\n", path); + } + return 0; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3112f263/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.h new file mode 100644 index 0000000..220ce3d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/fuse_trash.h @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __FUSE_TRASH_H__ +#define __FUSE_TRASH_H__ + +#include + +int hdfsDeleteWithTrash(hdfsFS userFS, const char *path, int useTrash); + +#endif