subversion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From stef...@apache.org
Subject svn commit: r1595582 - in /subversion/trunk/subversion/libsvn_fs_x: ./ cached_data.c fs_x.c index.c index.h pack.c verify.c
Date Sun, 18 May 2014 09:57:38 GMT
Author: stefan2
Date: Sun May 18 09:57:37 2014
New Revision: 1595582

URL: http://svn.apache.org/r1595582
Log:
Sync'ing FSX with FSFS:
Merge r1584235,1590460,1590464 from /subversion/libsvn_fs_fs into
subversion/libsvn_fs_x.  Conflicts were due structural and naming
differences between FSX and FSFS.

This ports the "independent p2l index page size" patches.

Modified:
    subversion/trunk/subversion/libsvn_fs_x/   (props changed)
    subversion/trunk/subversion/libsvn_fs_x/cached_data.c
    subversion/trunk/subversion/libsvn_fs_x/fs_x.c
    subversion/trunk/subversion/libsvn_fs_x/index.c
    subversion/trunk/subversion/libsvn_fs_x/index.h
    subversion/trunk/subversion/libsvn_fs_x/pack.c
    subversion/trunk/subversion/libsvn_fs_x/verify.c

Propchange: subversion/trunk/subversion/libsvn_fs_x/
------------------------------------------------------------------------------
  Merged /subversion/trunk/subversion/libsvn_fs_fs:r1584235,1590460,1590464

Modified: subversion/trunk/subversion/libsvn_fs_x/cached_data.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/cached_data.c?rev=1595582&r1=1595581&r2=1595582&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/cached_data.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/cached_data.c Sun May 18 09:57:37 2014
@@ -2701,9 +2701,10 @@ block_read(void **result,
   offset = wanted_offset;
   do
     {
-      SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, revision, offset,
-                                         scratch_pool));
+      /* fetch list of items in the block surrounding OFFSET */
       SVN_ERR(aligned_seek(fs, revision_file, &block_start, offset, iterpool));
+      SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, revision, block_start,
+                                         ffd->block_size, scratch_pool));
 
       /* read all items from the block */
       for (i = 0; i < entries->nelts; ++i)

Modified: subversion/trunk/subversion/libsvn_fs_x/fs_x.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/fs_x.c?rev=1595582&r1=1595581&r2=1595582&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/fs_x.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/fs_x.c Sun May 18 09:57:37 2014
@@ -252,7 +252,7 @@ read_config(fs_x_data_t *ffd,
   SVN_ERR(svn_config_get_int64(config, &ffd->p2l_page_size,
                                CONFIG_SECTION_IO,
                                CONFIG_OPTION_P2L_PAGE_SIZE,
-                               64));
+                               0x400));
 
   ffd->block_size *= 0x400;
   ffd->p2l_page_size *= 0x400;
@@ -428,11 +428,10 @@ write_config(svn_fs_t *fs,
 "### of tens of MB each,  increasing the page size will shorten the index"   NL
 "### file at the expense of a slightly increased latency in sections with"   NL
 "### smaller changes."                                                       NL
-"### For practical reasons,  this should match block-size.  Differing"       NL
-"### values are perfectly legal but may result in some processing overhead." NL
+"### For source code repositories, this should be about 16x the block-size." NL
 "### Must be a power of 2."                                                  NL
-"### p2l-page-size is 64 kBytes by default."                                 NL
-"# " CONFIG_OPTION_P2L_PAGE_SIZE " = 64"                                     NL
+"### p2l-page-size is 1024 kBytes by default."                               NL
+"# " CONFIG_OPTION_P2L_PAGE_SIZE " = 1024"                                   NL
 ;
 #undef NL
   return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG, pool),

Modified: subversion/trunk/subversion/libsvn_fs_x/index.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/index.c?rev=1595582&r1=1595581&r2=1595582&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/index.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/index.c Sun May 18 09:57:37 2014
@@ -1436,6 +1436,8 @@ prefetch_l2p_pages(svn_boolean_t *end,
 
   for (i = 0; i < pages->nelts && !*end; ++i)
     {
+      svn_boolean_t is_cached;
+
       l2p_page_table_entry_t *entry
         = &APR_ARRAY_IDX(pages, i, l2p_page_table_entry_t);
       if (i == exlcuded_page_no)
@@ -1451,9 +1453,9 @@ prefetch_l2p_pages(svn_boolean_t *end,
 
       /* page already in cache? */
       key.page = i;
-      SVN_ERR(svn_cache__has_key(end, ffd->l2p_page_cache,
+      SVN_ERR(svn_cache__has_key(&is_cached, ffd->l2p_page_cache,
                                  &key, iterpool));
-      if (!*end)
+      if (!is_cached)
         {
           /* no in cache -> read from stream (data already buffered in APR)
            * and cache the result */
@@ -1527,8 +1529,8 @@ l2p_index_lookup(apr_off_t *offset,
       svn_boolean_t end;
       apr_off_t max_offset
         = APR_ALIGN(info_baton.entry.offset + info_baton.entry.size,
-                    0x10000);
-      apr_off_t min_offset = max_offset - 0x10000;
+                    ffd->block_size);
+      apr_off_t min_offset = max_offset - ffd->block_size;
 
       /* read the relevant page */
       SVN_ERR(get_l2p_page(&page, &stream, fs, info_baton.first_revision,
@@ -2420,27 +2422,140 @@ get_p2l_keys(p2l_page_info_baton_t *page
   return SVN_NO_ERROR;
 }
 
-/* Body of svn_fs_x__p2l_index_lookup.  Use / autoconstruct *STREAM as
- * your input based on REVISION.
+/* qsort-compatible compare function that compares the OFFSET of the
+ * svn_fs_x__p2l_entry_t in *LHS with the apr_off_t in *RHS. */
+static int
+compare_start_p2l_entry(const void *lhs,
+                        const void *rhs)
+{
+  const svn_fs_x__p2l_entry_t *entry = lhs;
+  apr_off_t start = *(const apr_off_t*)rhs;
+  apr_off_t diff = entry->offset - start;
+
+  /* restrict result to int */
+  return diff < 0 ? -1 : (diff == 0 ? 0 : 1);
+}
+
+/* From the PAGE_ENTRIES array of svn_fs_x__p2l_entry_t, ordered
+ * by their OFFSET member, copy all elements overlapping the range
+ * [BLOCK_START, BLOCK_END) to ENTRIES.  If RESOLVE_PTR is set, the ITEMS
+ * sub-array in each entry needs to be de-serialized. */
+static void
+append_p2l_entries(apr_array_header_t *entries,
+                   apr_array_header_t *page_entries,
+                   apr_off_t block_start,
+                   apr_off_t block_end,
+                   svn_boolean_t resolve_ptr)
+{
+  const svn_fs_x__p2l_entry_t *entry;
+  int idx = svn_sort__bsearch_lower_bound(page_entries, &block_start,
+                                          compare_start_p2l_entry);
+
+  /* start at the first entry that overlaps with BLOCK_START */
+  if (idx > 0)
+    {
+      entry = &APR_ARRAY_IDX(page_entries, idx - 1, svn_fs_x__p2l_entry_t);
+      if (entry->offset + entry->size > block_start)
+        --idx;
+    }
+
+  /* copy all entries covering the requested range */
+  for ( ; idx < page_entries->nelts; ++idx)
+    {
+      svn_fs_x__p2l_entry_t *copy;
+      entry = &APR_ARRAY_IDX(page_entries, idx, svn_fs_x__p2l_entry_t);
+      if (entry->offset >= block_end)
+        break;
+
+      /* Copy the entry record. */
+      copy = apr_array_push(entries);
+      *copy = *entry;
+
+      /* Copy the items of that entries. */
+      if (entry->item_count)
+        {
+          const svn_fs_x__id_part_t *items
+            = resolve_ptr
+            ? svn_temp_deserializer__ptr(page_entries->elts,
+                                         (const void * const *)&entry->items)
+            : entry->items;
+
+          copy->items = apr_pmemdup(entries->pool, items,
+                                    entry->item_count * sizeof(*items));
+        }
+    }
+}
+
+/* Auxilliary struct passed to p2l_entries_func selecting the relevant
+ * data range. */
+typedef struct p2l_entries_baton_t
+{
+  apr_off_t start;
+  apr_off_t end;
+} p2l_entries_baton_t;
+
+/* Implement svn_cache__partial_getter_func_t: extract p2l entries from
+ * the page in DATA which overlap the p2l_entries_baton_t in BATON.
+ * The target array is already provided in *OUT.
  */
 static svn_error_t *
-p2l_index_lookup(apr_array_header_t **entries,
+p2l_entries_func(void **out,
+                 const void *data,
+                 apr_size_t data_len,
+                 void *baton,
+                 apr_pool_t *result_pool)
+{
+  apr_array_header_t *entries = *(apr_array_header_t **)out;
+  const apr_array_header_t *raw_page = data;
+  p2l_entries_baton_t *block = baton;
+  int i;
+
+  /* Make PAGE a readable APR array. */
+  apr_array_header_t page = *raw_page;
+  page.elts = (void *)svn_temp_deserializer__ptr(raw_page,
+                                    (const void * const *)&raw_page->elts);
+
+  /* append relevant information to result */
+  append_p2l_entries(entries, &page, block->start, block->end, TRUE);
+
+  return SVN_NO_ERROR;
+}
+
+
+/* Body of svn_fs_x__p2l_index_lookup.  However, do a single index page
+ * lookup and append the result to the ENTRIES array provided by the caller.
+ * Use successive calls to cover larger ranges.
+ */
+static svn_error_t *
+p2l_index_lookup(apr_array_header_t *entries,
                  packed_number_stream_t **stream,
                  svn_fs_t *fs,
                  svn_revnum_t revision,
-                 apr_off_t offset,
+                 apr_off_t block_start,
+                 apr_off_t block_end,
                  apr_pool_t *pool)
 {
   fs_x_data_t *ffd = fs->fsap_data;
   svn_fs_x__page_cache_key_t key;
   svn_boolean_t is_cached = FALSE;
   p2l_page_info_baton_t page_info;
+  apr_array_header_t *local_result = entries;
+
+  /* baton selecting the relevant entries from the one page we access */
+  p2l_entries_baton_t block;
+  block.start = block_start;
+  block.end = block_end;
 
-  /* look for this page in our cache */
-  SVN_ERR(get_p2l_keys(&page_info, &key, stream, fs, revision, offset,
+  /* if we requested an empty range, the result would be empty */
+  SVN_ERR_ASSERT(block_start < block_end);
+
+  /* look for the fist page of the range in our cache */
+  SVN_ERR(get_p2l_keys(&page_info, &key, stream, fs, revision, block_start,
                        pool));
-  SVN_ERR(svn_cache__get((void**)entries, &is_cached, ffd->p2l_page_cache,
-                         &key, pool));
+  SVN_ERR(svn_cache__get_partial((void**)&local_result, &is_cached,
+                                 ffd->p2l_page_cache, &key, p2l_entries_func,
+                                 &block, pool));
+
   if (!is_cached)
     {
       svn_boolean_t end;
@@ -2448,6 +2563,7 @@ p2l_index_lookup(apr_array_header_t **en
       apr_off_t original_page_start = page_info.page_start;
       int leaking_bucket = 4;
       p2l_page_info_baton_t prefetch_info = page_info;
+      apr_array_header_t *page_entries;
 
       apr_off_t max_offset
         = APR_ALIGN(page_info.next_offset, ffd->block_size);
@@ -2472,14 +2588,18 @@ p2l_index_lookup(apr_array_header_t **en
         }
 
       /* fetch page from disk and put it into the cache */
-      SVN_ERR(get_p2l_page(entries, stream, fs,
+      SVN_ERR(get_p2l_page(&page_entries, stream, fs,
                            page_info.first_revision,
                            page_info.start_offset,
                            page_info.next_offset,
                            page_info.page_start,
-                           page_info.page_size, pool, pool));
+                           page_info.page_size, pool, iterpool));
 
-      SVN_ERR(svn_cache__set(ffd->p2l_page_cache, &key, *entries, pool));
+      SVN_ERR(svn_cache__set(ffd->p2l_page_cache, &key, page_entries,
+                             iterpool));
+
+      /* append relevant information to result */
+      append_p2l_entries(entries, page_entries, block_start, block_end, FALSE);
 
       /* pre-fetch following pages */
       end = FALSE;
@@ -2500,6 +2620,40 @@ p2l_index_lookup(apr_array_header_t **en
       svn_pool_destroy(iterpool);
     }
 
+  /* We access a valid page (otherwise, we had seen an error in the
+   * get_p2l_keys request).  Hence, at least one entry must be found. */
+  SVN_ERR_ASSERT(entries->nelts > 0);
+
+  /* Add an "unused" entry if it extends beyond the end of the data file.
+   * Since the index page size might be smaller than the current data
+   * read block size, the trailing "unused" entry in this index may not
+   * fully cover the end of the last block. */
+  if (page_info.page_no + 1 >= page_info.page_count)
+    {
+      svn_fs_x__p2l_entry_t *entry
+        = &APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_x__p2l_entry_t);
+
+      apr_off_t entry_end = entry->offset + entry->size;
+      if (entry_end < block_end)
+        {
+          if (entry->type == SVN_FS_X__ITEM_TYPE_UNUSED)
+            {
+              /* extend the terminal filler */
+              entry->size = block_end - entry->offset;
+            }
+          else
+            {
+              /* No terminal filler. Add one. */
+              entry = apr_array_push(entries);
+              entry->offset = entry_end;
+              entry->size = block_end - entry_end;
+              entry->type = SVN_FS_X__ITEM_TYPE_UNUSED;
+              entry->item_count = 0;
+              entry->items = NULL;
+            }
+        }
+    }
+
   return SVN_NO_ERROR;
 }
 
@@ -2507,17 +2661,51 @@ svn_error_t *
 svn_fs_x__p2l_index_lookup(apr_array_header_t **entries,
                            svn_fs_t *fs,
                            svn_revnum_t revision,
-                           apr_off_t offset,
+                           apr_off_t block_start,
+                           apr_off_t block_size,
                            apr_pool_t *pool)
 {
   packed_number_stream_t *stream = NULL;
 
-  /* look for this page in our cache */
-  SVN_ERR(p2l_index_lookup(entries, &stream, fs, revision, offset, pool));
+  apr_off_t block_end = block_start + block_size;
+
+  /* the receiving container */
+  int last_count = 0;
+  apr_array_header_t *result = apr_array_make(pool, 16,
+                                              sizeof(svn_fs_x__p2l_entry_t));
+
+  /* Fetch entries page-by-page.  Since the p2l index is supposed to cover
+   * every single byte in the rev / pack file - even unused sections -
+   * every iteration must result in some progress. */
+  while (block_start < block_end)
+    {
+      svn_fs_x__p2l_entry_t *entry;
+      SVN_ERR(p2l_index_lookup(result, &stream, fs, revision, block_start,
+                               block_end, pool));
+      SVN_ERR_ASSERT(result->nelts > 0);
+
+      /* continue directly behind last item */
+      entry = &APR_ARRAY_IDX(result, result->nelts-1, svn_fs_x__p2l_entry_t);
+      block_start = entry->offset + entry->size;
+
+      /* Some paranoia check.  Successive iterations should never return
+       * duplicates but if it did, we might get into trouble later on. */
+      if (last_count > 0 && last_count < result->nelts)
+        {
+           entry = &APR_ARRAY_IDX(result, last_count - 1,
+                                  svn_fs_x__p2l_entry_t);
+           SVN_ERR_ASSERT(APR_ARRAY_IDX(result, last_count,
+                                        svn_fs_x__p2l_entry_t).offset
+                          >= entry->offset + entry->size);
+        }
+
+      last_count = result->nelts;
+    }
 
   /* make sure we close files after usage */
   SVN_ERR(packed_stream_close(stream));
 
+  *entries = result;
   return SVN_NO_ERROR;
 }
 
@@ -2611,8 +2799,9 @@ p2l_entry_lookup(svn_fs_x__p2l_entry_t *
     {
       /* do a standard index lookup.  This is will automatically prefetch
        * data to speed up future lookups. */
-      apr_array_header_t *entries;
-      SVN_ERR(p2l_index_lookup(&entries, stream, fs, revision, offset, pool));
+      apr_array_header_t *entries = apr_array_make(pool, 1, sizeof(**entry_p));
+      SVN_ERR(p2l_index_lookup(entries, stream, fs, revision, offset,
+                               offset + 1, pool));
 
       /* Find the entry that we want. */
       *entry_p = svn_sort__array_lookup(entries, &offset, NULL,

Modified: subversion/trunk/subversion/libsvn_fs_x/index.h
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/index.h?rev=1595582&r1=1595581&r2=1595582&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/index.h (original)
+++ subversion/trunk/subversion/libsvn_fs_x/index.h Sun May 18 09:57:37 2014
@@ -168,9 +168,9 @@ svn_fs_x__p2l_index_create(svn_fs_t *fs,
                            apr_pool_t *pool);
 
 /* Use the phys-to-log mapping files in FS to build a list of entries
- * that (partly) share in the same cluster as the item at global OFFSET
- * in the rep file containing REVISION.  Return the array in *ENTRIES,
- * elements being of type svn_fs_x__p2l_entry_t.
+ * that (at least partly) overlap with the range given by BLOCK_START
+ * offset and BLOCK_SIZE in the rep / pack file containing REVISION.
+ * Return the array in *ENTRIES with svn_fs_fs__p2l_entry_t as elements.
  * Use POOL for allocations.
  *
  * Note that (only) the first and the last mapping may cross a cluster
@@ -180,7 +180,8 @@ svn_error_t *
 svn_fs_x__p2l_index_lookup(apr_array_header_t **entries,
                            svn_fs_t *fs,
                            svn_revnum_t revision,
-                           apr_off_t offset,
+                           apr_off_t block_start,
+                           apr_off_t block_size,
                            apr_pool_t *pool);
 
 /* Use the phys-to-log mapping files in FS to return the entry for the

Modified: subversion/trunk/subversion/libsvn_fs_x/pack.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/pack.c?rev=1595582&r1=1595581&r2=1595582&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/pack.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/pack.c Sun May 18 09:57:37 2014
@@ -1756,6 +1756,7 @@ static svn_error_t *
 pack_range(pack_context_t *context,
            apr_pool_t *pool)
 {
+  fs_x_data_t *ffd = context->fs->fsap_data;
   apr_pool_t *revpool = svn_pool_create(pool);
   apr_pool_t *iterpool = svn_pool_create(pool);
 
@@ -1788,9 +1789,11 @@ pack_range(pack_context_t *context,
           /* read one cluster */
           int i;
           apr_array_header_t *entries;
+          svn_pool_clear(iterpool);
+
           SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, context->fs,
                                              revision, offset,
-                                             iterpool));
+                                             ffd->p2l_page_size, iterpool));
 
           for (i = 0; i < entries->nelts; ++i)
             {
@@ -1840,8 +1843,6 @@ pack_range(pack_context_t *context,
 
           if (context->cancel_func)
             SVN_ERR(context->cancel_func(context->cancel_baton));
-
-          svn_pool_clear(iterpool);
         }
 
       svn_pool_clear(revpool);
@@ -1887,6 +1888,7 @@ static svn_error_t *
 append_revision(pack_context_t *context,
                 apr_pool_t *pool)
 {
+  fs_x_data_t *ffd = context->fs->fsap_data;
   apr_off_t offset = 0;
   apr_pool_t *iterpool = svn_pool_create(pool);
   apr_file_t *rev_file;
@@ -1919,7 +1921,7 @@ append_revision(pack_context_t *context,
       apr_array_header_t *entries;
       SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, context->fs,
                                           context->start_rev, offset,
-                                          iterpool));
+                                          ffd->p2l_page_size, iterpool));
 
       for (i = 0; i < entries->nelts; ++i)
         {

Modified: subversion/trunk/subversion/libsvn_fs_x/verify.c
URL: http://svn.apache.org/viewvc/subversion/trunk/subversion/libsvn_fs_x/verify.c?rev=1595582&r1=1595581&r2=1595582&view=diff
==============================================================================
--- subversion/trunk/subversion/libsvn_fs_x/verify.c (original)
+++ subversion/trunk/subversion/libsvn_fs_x/verify.c Sun May 18 09:57:37 2014
@@ -250,6 +250,7 @@ compare_p2l_to_l2p_index(svn_fs_t *fs,
                          void *cancel_baton,
                          apr_pool_t *pool)
 {
+  fs_x_data_t *ffd = fs->fsap_data;
   apr_pool_t *iterpool = svn_pool_create(pool);
   apr_off_t max_offset;
   apr_off_t offset = 0;
@@ -267,7 +268,7 @@ compare_p2l_to_l2p_index(svn_fs_t *fs,
 
       /* get all entries for the current block */
       SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, start, offset,
-                                         iterpool));
+                                         ffd->p2l_page_size, iterpool));
       if (entries->nelts == 0)
         return svn_error_createf(SVN_ERR_FS_ITEM_INDEX_CORRUPTION,
                                  NULL,



Mime
View raw message