apr-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Joe Orton <jor...@redhat.com>
Subject [RFC] "tailfile" bucket type
Date Thu, 15 Jun 2006 13:31:43 GMT
For PR 39807; this implements a new bucket type which has slightly 
different semantics to the FILE bucket: it represents the tail of a file 
from a fixed offset to EOF, rather than a segment of fixed length. This 
allows the user to represent files of arbitrary size without having to 
chop them up into buckets.  (and generally do other interesting things 
using files for which the length is not constant)

It's not really possible to do this by modifying the FILE bucket type 
(as I'd hoped) - it's really an interface guarantee that FILE buckets 
have a known length.

diff below shows the diff relative to apr_bucket_file.c so is rather 
confusing (seems that is the only way "svn diff" will work for copied 
files)

Index: buckets/apr_buckets_tailfile.c
===================================================================
--- buckets/apr_buckets_tailfile.c	(revision 411793)
+++ buckets/apr_buckets_tailfile.c	(working copy)
@@ -19,17 +19,7 @@
 #include "apr_file_io.h"
 #include "apr_buckets.h"
 
-#if APR_HAS_MMAP
-#include "apr_mmap.h"
-
-/* mmap support for static files based on ideas from John Heidemann's
- * patch against 1.0.5.  See
- * <http://www.isi.edu/~johnh/SOFTWARE/APACHE/index.html>.
- */
-
-#endif /* APR_HAS_MMAP */
-
-static void file_bucket_destroy(void *data)
+static void tailfile_bucket_destroy(void *data)
 {
     apr_bucket_file *f = data;
 
@@ -40,39 +30,7 @@
     }
 }
 
-#if APR_HAS_MMAP
-static int file_make_mmap(apr_bucket *e, apr_size_t filelength,
-                           apr_off_t fileoffset, apr_pool_t *p)
-{
-    apr_bucket_file *a = e->data;
-    apr_mmap_t *mm;
-
-    if (!a->can_mmap) {
-        return 0;
-    }
-
-    if (filelength > APR_MMAP_LIMIT) {
-        if (apr_mmap_create(&mm, a->fd, fileoffset, APR_MMAP_LIMIT,
-                            APR_MMAP_READ, p) != APR_SUCCESS)
-        {
-            return 0;
-        }
-        apr_bucket_split(e, APR_MMAP_LIMIT);
-        filelength = APR_MMAP_LIMIT;
-    }
-    else if ((filelength < APR_MMAP_THRESHOLD) ||
-             (apr_mmap_create(&mm, a->fd, fileoffset, filelength,
-                              APR_MMAP_READ, p) != APR_SUCCESS))
-    {
-        return 0;
-    }
-    apr_bucket_mmap_make(e, mm, 0, filelength);
-    file_bucket_destroy(a);
-    return 1;
-}
-#endif
-
-static apr_status_t file_bucket_read(apr_bucket *e, const char **str,
+static apr_status_t tailfile_bucket_read(apr_bucket *e, const char **str,
                                      apr_size_t *len, apr_read_type_e block)
 {
     apr_bucket_file *a = e->data;
@@ -80,18 +38,11 @@
     apr_bucket *b = NULL;
     char *buf;
     apr_status_t rv;
-    apr_size_t filelength = e->length;  /* bytes remaining in file past offset */
     apr_off_t fileoffset = e->start;
 #if APR_HAS_THREADS && !APR_HAS_XTHREAD_FILES
     apr_int32_t flags;
 #endif
 
-#if APR_HAS_MMAP
-    if (file_make_mmap(e, filelength, fileoffset, a->readpool)) {
-        return apr_bucket_read(e, str, len, block);
-    }
-#endif
-
 #if APR_HAS_THREADS && !APR_HAS_XTHREAD_FILES
     if ((flags = apr_file_flags_get(f)) & APR_XTHREAD) {
         /* this file descriptor is shared across multiple threads and
@@ -108,9 +59,7 @@
     }
 #endif
 
-    *len = (filelength > APR_BUCKET_BUFF_SIZE)
-               ? APR_BUCKET_BUFF_SIZE
-               : filelength;
+    *len = APR_BUCKET_BUFF_SIZE;
     *str = NULL;  /* in case we die prematurely */
     buf = apr_bucket_alloc(*len, e->list);
 
@@ -125,7 +74,6 @@
         apr_bucket_free(buf);
         return rv;
     }
-    filelength -= *len;
     /*
      * Change the current bucket to refer to what we read,
      * even if we read nothing because we hit EOF.
@@ -133,73 +81,58 @@
     apr_bucket_heap_make(e, buf, *len, apr_bucket_free);
 
     /* If we have more to read from the file, then create another bucket */
-    if (filelength > 0 && rv != APR_EOF) {
+    if (rv != APR_EOF) {
         /* for efficiency, we can just build a new apr_bucket struct
          * to wrap around the existing file bucket */
         b = apr_bucket_alloc(sizeof(*b), e->list);
         b->start  = fileoffset + (*len);
-        b->length = filelength;
+        b->length = (apr_size_t)-1;
         b->data   = a;
-        b->type   = &apr_bucket_type_file;
+        b->type   = &apr_bucket_type_tailfile;
         b->free   = apr_bucket_free;
         b->list   = e->list;
         APR_BUCKET_INSERT_AFTER(e, b);
     }
     else {
-        file_bucket_destroy(a);
+        tailfile_bucket_destroy(a);
     }
 
     *str = buf;
-    return rv;
+
+    /* Reaching EOF is expected eventually; return success then too. */
+    return APR_SUCCESS;
 }
 
-APU_DECLARE(apr_bucket *) apr_bucket_file_make(apr_bucket *b, apr_file_t *fd,
-                                               apr_off_t offset,
-                                               apr_size_t len, apr_pool_t *p)
+APU_DECLARE(apr_bucket *) apr_bucket_tailfile_make(apr_bucket *b, apr_file_t *fd,
+                                                   apr_off_t offset, apr_pool_t *p)
 {
     apr_bucket_file *f;
 
     f = apr_bucket_alloc(sizeof(*f), b->list);
     f->fd = fd;
     f->readpool = p;
-#if APR_HAS_MMAP
-    f->can_mmap = 1;
-#endif
 
-    b = apr_bucket_shared_make(b, f, offset, len);
-    b->type = &apr_bucket_type_file;
+    b = apr_bucket_shared_make(b, f, offset, (apr_size_t)-1);
+    b->type = &apr_bucket_type_tailfile;
 
     return b;
 }
 
-APU_DECLARE(apr_bucket *) apr_bucket_file_create(apr_file_t *fd,
-                                                 apr_off_t offset,
-                                                 apr_size_t len, apr_pool_t *p,
-                                                 apr_bucket_alloc_t *list)
+APU_DECLARE(apr_bucket *) apr_bucket_tailfile_create(apr_file_t *fd,
+                                                     apr_off_t offset,
+                                                     apr_pool_t *p,
+                                                     apr_bucket_alloc_t *list)
 {
     apr_bucket *b = apr_bucket_alloc(sizeof(*b), list);
 
     APR_BUCKET_INIT(b);
     b->free = apr_bucket_free;
     b->list = list;
-    return apr_bucket_file_make(b, fd, offset, len, p);
+    return apr_bucket_tailfile_make(b, fd, offset, p);
 }
 
-APU_DECLARE(apr_status_t) apr_bucket_file_enable_mmap(apr_bucket *e,
-                                                      int enabled)
+static apr_status_t tailfile_bucket_setaside(apr_bucket *data, apr_pool_t *reqpool)
 {
-#if APR_HAS_MMAP
-    apr_bucket_file *a = e->data;
-    a->can_mmap = enabled;
-    return APR_SUCCESS;
-#else
-    return APR_ENOTIMPL;
-#endif /* APR_HAS_MMAP */
-}
-
-
-static apr_status_t file_bucket_setaside(apr_bucket *data, apr_pool_t *reqpool)
-{
     apr_bucket_file *a = data->data;
     apr_file_t *fd = NULL;
     apr_file_t *f = a->fd;
@@ -218,11 +151,11 @@
     return APR_SUCCESS;
 }
 
-APU_DECLARE_DATA const apr_bucket_type_t apr_bucket_type_file = {
-    "FILE", 5, APR_BUCKET_DATA,
-    file_bucket_destroy,
-    file_bucket_read,
-    file_bucket_setaside,
+APU_DECLARE_DATA const apr_bucket_type_t apr_bucket_type_tailfile = {
+    "TAILFILE", 5, APR_BUCKET_DATA,
+    tailfile_bucket_destroy,
+    tailfile_bucket_read,
+    tailfile_bucket_setaside,
     apr_bucket_shared_split,
     apr_bucket_shared_copy
 };
Index: include/apr_buckets.h
===================================================================
--- include/apr_buckets.h	(revision 411793)
+++ include/apr_buckets.h	(working copy)
@@ -466,6 +466,12 @@
  */
 #define APR_BUCKET_IS_FILE(e)        ((e)->type == &apr_bucket_type_file)
 /**
+ * Determine if a bucket is a TAILFILE bucket
+ * @param e The bucket to inspect
+ * @return true or false
+ */
+#define APR_BUCKET_IS_TAILFILE(e)        ((e)->type == &apr_bucket_type_tailfile)
+/**
  * Determine if a bucket is a PIPE bucket
  * @param e The bucket to inspect
  * @return true or false
@@ -1076,10 +1082,16 @@
  */
 APU_DECLARE_DATA extern const apr_bucket_type_t apr_bucket_type_eos;
 /**
- * The FILE bucket type.  This bucket represents a file on disk
+ * The FILE bucket type.  This bucket represents a fixed segment of a
+ * file on disk
  */
 APU_DECLARE_DATA extern const apr_bucket_type_t apr_bucket_type_file;
 /**
+ * The TAILFILE bucket type.  This bucket represents the tail of a
+ * file on disk
+ */
+APU_DECLARE_DATA extern const apr_bucket_type_t apr_bucket_type_tailfile;
+/**
  * The HEAP bucket type.  This bucket represents a data allocated from the
  * heap.
  */
@@ -1418,7 +1430,7 @@
                                                apr_file_t *thispipe);
 
 /**
- * Create a bucket referring to a file.
+ * Create a bucket respresenting a fixed segment of a file.
  * @param fd The file to put in the bucket
  * @param offset The offset where the data of interest begins in the file
  * @param len The amount of data in the file we are interested in
@@ -1426,6 +1438,12 @@
  *          while reading from this file bucket
  * @param list The freelist from which this bucket should be allocated
  * @return The new bucket, or NULL if allocation failed
+ * @remark If the file is truncated such that the segment of the file
+ * referenced by the bucket no longer exists, an attempt to read
+ * from the bucket will fail with APR_EOF. 
+ * @remark apr_brigade_insert_file() should generally be used to
+ * insert files into brigades, since that function will correctly
+ * handle large file issues.
  */
 APU_DECLARE(apr_bucket *) apr_bucket_file_create(apr_file_t *fd,
                                                  apr_off_t offset,
@@ -1434,7 +1452,7 @@
                                                  apr_bucket_alloc_t *list);
 
 /**
- * Make the bucket passed in a bucket refer to a file
+ * Make the bucket passed in a bucket refer to a fixed segment of a file
  * @param b The bucket to make into a FILE bucket
  * @param fd The file to put in the bucket
  * @param offset The offset where the data of interest begins in the file
@@ -1456,6 +1474,37 @@
 APU_DECLARE(apr_status_t) apr_bucket_file_enable_mmap(apr_bucket *b,
                                                       int enabled);
 
+/**
+ * Create a bucket representing a tail segment of the file.  This
+ * function will create a bucket of unspecified length which represents
+ * the file from the given offset until EOF.
+ * @param fd The file to put in the bucket
+ * @param offset The offset where the data of interest begins in the file
+ * @param p The pool into which any needed structures should be created
+ *          while reading from this file bucket
+ * @param list The freelist from which this bucket should be allocated
+ * @return The new bucket, or NULL if allocation failed
+ * @remark apr_brigade_insert_file() should generally be used to insert
+ * files into brigades, since that function will correctly handle 
+ * large file issues.
+ */
+APU_DECLARE(apr_bucket *) apr_bucket_tailfile_create(apr_file_t *fd,
+                                                     apr_off_t offset,
+                                                     apr_pool_t *p,
+                                                     apr_bucket_alloc_t *list);
+
+/**
+ * Make the bucket passed in a bucket refer to a tail segment of a file.
+ * @param b The bucket to make into a FILE bucket
+ * @param fd The file to put in the bucket
+ * @param offset The offset where the data of interest begins in the file
+ * @param p The pool into which any needed structures should be created
+ *          while reading from this file bucket
+ * @return The new bucket, or NULL if allocation failed
+ */
+APU_DECLARE(apr_bucket *) apr_bucket_tailfile_make(apr_bucket *b, apr_file_t *fd,
+                                                   apr_off_t offset, apr_pool_t *p);
+
 /** @} */
 #ifdef __cplusplus
 }
Index: test/testbuckets.c
===================================================================
--- test/testbuckets.c	(revision 411793)
+++ test/testbuckets.c	(working copy)
@@ -468,6 +468,67 @@
     apr_bucket_alloc_destroy(ba);
 }
 
+static apr_file_t *make_test_binfile(abts_case *tc, const char *fname,
+                                     const char *contents, apr_size_t len)
+{
+    apr_file_t *f;
+
+    ABTS_ASSERT(tc, "create test file",
+                apr_file_open(&f, fname,
+                              APR_READ|APR_WRITE|APR_TRUNCATE|APR_CREATE,
+                              APR_OS_DEFAULT, p) == APR_SUCCESS);
+    
+    ABTS_ASSERT(tc, "write test file contents",
+                apr_file_write_full(f, contents, len, NULL) == APR_SUCCESS);
+
+    return f;
+}
+
+#define TESTLEN (APR_BUCKET_BUFF_SIZE + 5)
+
+static void test_tailfile(abts_case *tc, void *arg)
+{
+    apr_bucket_alloc_t *ba = apr_bucket_alloc_create(p);
+    apr_bucket_brigade *bb = apr_brigade_create(p, ba);
+    char data[TESTLEN], *pnt;
+    apr_file_t *f;
+    apr_bucket *e;
+    apr_size_t len, dlen;
+    apr_status_t rv;
+
+    memset(data, 'A', sizeof data);
+
+    f = make_test_binfile(tc, "testfile.txt", data, sizeof data);
+
+    e = apr_bucket_tailfile_create(f, 0, p, ba);
+    
+    ABTS_ASSERT(tc, "created tailfile bucket", APR_BUCKET_IS_TAILFILE(e));
+    
+    APR_BRIGADE_INSERT_HEAD(bb, e);
+
+    dlen = sizeof data;
+    pnt = data;
+
+    do {
+        const char *buf;
+
+        rv = apr_bucket_read(e, &buf, &len, APR_BLOCK_READ);
+        ABTS_ASSERT(tc, "read from bucket OK", rv == APR_SUCCESS);
+
+        ABTS_ASSERT(tc, "returned data match",
+                    len <= dlen && memcmp(pnt, buf, len) == 0);
+        
+        dlen -= len;
+        pnt += len;
+        
+        e = APR_BUCKET_NEXT(e);
+    } while (len > 0);
+
+    apr_file_close(f);
+    apr_brigade_destroy(bb);
+    apr_bucket_alloc_destroy(ba);
+}
+
 abts_suite *testbuckets(abts_suite *suite)
 {
     suite = ADD_SUITE(suite);
@@ -483,6 +544,7 @@
     abts_run_test(suite, test_manyfile, NULL);
     abts_run_test(suite, test_truncfile, NULL);
     abts_run_test(suite, test_partition, NULL);
+    abts_run_test(suite, test_tailfile, NULL);
 
     return suite;
 }

Mime
View raw message