Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/noderevs.h URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/noderevs.h?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_x/noderevs.h (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_x/noderevs.h Sat Jan 3 14:00:41 2015 @@ -33,7 +33,7 @@ * * In its serialized form, the svn_fs_x__noderevs_t container extracts * most of that redundancy and the run-time representation is also much - * smaller than sum of the respective node_revision_t objects. + * smaller than sum of the respective svn_fs_x__noderev_t objects. * * As with other containers, this one has two modes: 'construction', in * which you may add data to it, and 'getter' in which there is only r/o @@ -47,7 +47,7 @@ typedef struct svn_fs_x__noderevs_t svn_ /* Create and populate noderev containers. */ /* Create and return a new noderevs container with an initial capacity of - * INITIAL_COUNT node_revision_t objects. Allocate the result in POOL. + * INITIAL_COUNT svn_fs_x__noderev_t objects. Allocate the result in POOL. */ svn_fs_x__noderevs_t * svn_fs_x__noderevs_create(int initial_count, @@ -58,7 +58,7 @@ svn_fs_x__noderevs_create(int initial_co */ apr_size_t svn_fs_x__noderevs_add(svn_fs_x__noderevs_t *container, - node_revision_t *noderev); + svn_fs_x__noderev_t *noderev); /* Return a rough estimate in bytes for the serialized representation * of CONTAINER. @@ -72,7 +72,7 @@ svn_fs_x__noderevs_estimate_size(const s * the result in POOL and return it in *NODEREV_P. */ svn_error_t * -svn_fs_x__noderevs_get(node_revision_t **noderev_p, +svn_fs_x__noderevs_get(svn_fs_x__noderev_t **noderev_p, const svn_fs_x__noderevs_t *container, apr_size_t idx, apr_pool_t *pool); @@ -116,7 +116,7 @@ svn_fs_x__deserialize_noderevs_container apr_pool_t *pool); /* Implements svn_cache__partial_getter_func_t for svn_fs_x__noderevs_t, - * setting *OUT to the node_revision_t selected by the apr_uint32_t index + * setting *OUT to the svn_fs_x__noderev_t selected by the apr_uint32_t index * passed in as *BATON. This function is similar to svn_fs_x__noderevs_get * but operates on the cache serialized representation of the container. */
Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_x/pack.c Sat Jan 3 14:00:41 2015 @@ -101,7 +101,7 @@ typedef struct path_order_t svn_prefix_string__t *path; /* node ID for this PATH in REVISION */ - svn_fs_x__id_part_t node_id; + svn_fs_x__id_t node_id; /* when this change happened */ svn_revnum_t revision; @@ -113,10 +113,10 @@ typedef struct path_order_t apr_int64_t expanded_size; /* item ID of the noderev linked to the change. May be (0, 0). */ - svn_fs_x__id_part_t noderev_id; + svn_fs_x__id_t noderev_id; /* item ID of the representation containing the new data. May be (0, 0). */ - svn_fs_x__id_part_t rep_id; + svn_fs_x__id_t rep_id; } path_order_t; /* Represents a reference from item FROM to item TO. FROM may be a noderev @@ -125,8 +125,8 @@ typedef struct path_order_t */ typedef struct reference_t { - svn_fs_x__id_part_t to; - svn_fs_x__id_part_t from; + svn_fs_x__id_t to; + svn_fs_x__id_t from; } reference_t; /* This structure keeps track of all the temporary data and status that @@ -251,7 +251,7 @@ initialize_pack_context(pack_context_t * void *cancel_baton, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; const char *temp_dir; int max_revs = MIN(ffd->max_files_per_dir, max_items); @@ -357,12 +357,6 @@ static svn_error_t * close_pack_context(pack_context_t *context, apr_pool_t *pool) { - const char *l2p_index_path - = apr_pstrcat(pool, context->pack_file_path, PATH_EXT_L2P_INDEX, - SVN_VA_NULL); - const char *p2l_index_path - = apr_pstrcat(pool, context->pack_file_path, PATH_EXT_P2L_INDEX, - SVN_VA_NULL); const char *proto_l2p_index_path; const char *proto_p2l_index_path; @@ -371,18 +365,17 @@ close_pack_context(pack_context_t *conte context->proto_l2p_index, pool)); SVN_ERR(svn_io_file_name_get(&proto_p2l_index_path, context->proto_p2l_index, pool)); - + /* finalize proto index files */ SVN_ERR(svn_io_file_close(context->proto_l2p_index, pool)); SVN_ERR(svn_io_file_close(context->proto_p2l_index, pool)); - /* Create the actual index files*/ - SVN_ERR(svn_fs_x__l2p_index_create(context->fs, l2p_index_path, - proto_l2p_index_path, - context->shard_rev, pool)); - SVN_ERR(svn_fs_x__p2l_index_create(context->fs, p2l_index_path, - proto_p2l_index_path, - context->shard_rev, pool)); + /* Append the actual index data to the pack file. */ + SVN_ERR(svn_fs_x__add_index_data(context->fs, context->pack_file, + proto_l2p_index_path, + proto_p2l_index_path, + context->shard_rev, + pool)); /* remove proto index files */ SVN_ERR(svn_io_remove_file2(proto_l2p_index_path, FALSE, pool)); @@ -421,7 +414,7 @@ copy_file_data(pack_context_t *context, /* use streaming copies for larger data blocks. That may require * the allocation of larger buffers and we should make sure that * this extra memory is released asap. */ - fs_x_data_t *ffd = context->fs->fsap_data; + svn_fs_x__data_t *ffd = context->fs->fsap_data; apr_pool_t *copypool = svn_pool_create(pool); char *buffer = apr_palloc(copypool, ffd->block_size); @@ -476,17 +469,18 @@ static svn_error_t * copy_item_to_temp(pack_context_t *context, apr_array_header_t *entries, apr_file_t *temp_file, - apr_file_t *rev_file, + svn_fs_x__revision_file_t *rev_file, svn_fs_x__p2l_entry_t *entry, apr_pool_t *pool) { svn_fs_x__p2l_entry_t *new_entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool); - new_entry->offset = 0; - SVN_ERR(svn_io_file_seek(temp_file, SEEK_CUR, &new_entry->offset, pool)); + + SVN_ERR(svn_fs_x__get_file_offset(&new_entry->offset, temp_file, pool)); APR_ARRAY_PUSH(entries, svn_fs_x__p2l_entry_t *) = new_entry; - SVN_ERR(copy_file_data(context, temp_file, rev_file, entry->size, pool)); + SVN_ERR(copy_file_data(context, temp_file, rev_file->file, entry->size, + pool)); return SVN_NO_ERROR; } @@ -536,7 +530,7 @@ add_item_rep_mapping(pack_context_t *con */ static svn_fs_x__p2l_entry_t * get_item(pack_context_t *context, - const svn_fs_x__id_part_t *id, + const svn_fs_x__id_t *id, svn_boolean_t reset) { svn_fs_x__p2l_entry_t *result = NULL; @@ -561,26 +555,22 @@ get_item(pack_context_t *context, */ static svn_error_t * copy_rep_to_temp(pack_context_t *context, - apr_file_t *rev_file, + svn_fs_x__revision_file_t *rev_file, svn_fs_x__p2l_entry_t *entry, apr_pool_t *pool) { svn_fs_x__rep_header_t *rep_header; - svn_stream_t *stream; apr_off_t source_offset = entry->offset; /* create a copy of ENTRY, make it point to the copy destination and * store it in CONTEXT */ entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool); - entry->offset = 0; - SVN_ERR(svn_io_file_seek(context->reps_file, SEEK_CUR, &entry->offset, - pool)); + SVN_ERR(svn_fs_x__get_file_offset(&entry->offset, context->reps_file, pool)); add_item_rep_mapping(context, entry); /* read & parse the representation header */ - stream = svn_stream_from_aprfile2(rev_file, TRUE, pool); - SVN_ERR(svn_fs_x__read_rep_header(&rep_header, stream, pool)); - svn_stream_close(stream); + SVN_ERR(svn_fs_x__read_rep_header(&rep_header, rev_file->stream, pool, + pool)); /* if the representation is a delta against some other rep, link the two */ if ( rep_header->type == svn_fs_x__rep_delta @@ -596,9 +586,9 @@ copy_rep_to_temp(pack_context_t *context } /* copy the whole rep (including header!) to our temp file */ - SVN_ERR(svn_io_file_seek(rev_file, SEEK_SET, &source_offset, pool)); - SVN_ERR(copy_file_data(context, context->reps_file, rev_file, entry->size, - pool)); + SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &source_offset, pool)); + SVN_ERR(copy_file_data(context, context->reps_file, rev_file->file, + entry->size, pool)); return SVN_NO_ERROR; } @@ -679,34 +669,30 @@ tweak_path_for_ordering(const char *orig */ static svn_error_t * copy_node_to_temp(pack_context_t *context, - apr_file_t *rev_file, + svn_fs_x__revision_file_t *rev_file, svn_fs_x__p2l_entry_t *entry, apr_pool_t *pool) { path_order_t *path_order = apr_pcalloc(context->info_pool, sizeof(*path_order)); - node_revision_t *noderev; + svn_fs_x__noderev_t *noderev; const char *sort_path; - svn_stream_t *stream; apr_off_t source_offset = entry->offset; /* read & parse noderev */ - stream = svn_stream_from_aprfile2(rev_file, TRUE, pool); - SVN_ERR(svn_fs_x__read_noderev(&noderev, stream, pool)); - svn_stream_close(stream); + SVN_ERR(svn_fs_x__read_noderev(&noderev, rev_file->stream, pool, pool)); /* create a copy of ENTRY, make it point to the copy destination and * store it in CONTEXT */ entry = svn_fs_x__p2l_entry_dup(entry, context->info_pool); - entry->offset = 0; - SVN_ERR(svn_io_file_seek(context->reps_file, SEEK_CUR, - &entry->offset, pool)); + SVN_ERR(svn_fs_x__get_file_offset(&entry->offset, context->reps_file, + pool)); add_item_rep_mapping(context, entry); /* copy the noderev to our temp file */ - SVN_ERR(svn_io_file_seek(rev_file, SEEK_SET, &source_offset, pool)); - SVN_ERR(copy_file_data(context, context->reps_file, rev_file, entry->size, - pool)); + SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &source_offset, pool)); + SVN_ERR(copy_file_data(context, context->reps_file, rev_file->file, + entry->size, pool)); /* if the node has a data representation, make that the node's "base". * This will (often) cause the noderev to be placed right in front of @@ -731,10 +717,10 @@ copy_node_to_temp(pack_context_t *contex * It will not be stored in the final pack file. */ sort_path = tweak_path_for_ordering(noderev->created_path, pool); path_order->path = svn_prefix_string__create(context->paths, sort_path); - path_order->node_id = *svn_fs_x__id_node_id(noderev->id); - path_order->revision = svn_fs_x__id_rev(noderev->id); + path_order->node_id = noderev->node_id; + path_order->revision = svn_fs_x__get_revnum(noderev->noderev_id.change_set); path_order->is_dir = noderev->kind == svn_node_dir; - path_order->noderev_id = *svn_fs_x__id_noderev_id(noderev->id); + path_order->noderev_id = noderev->noderev_id; APR_ARRAY_PUSH(context->path_order, path_order_t *) = path_order; return SVN_NO_ERROR; @@ -768,58 +754,6 @@ sort_items(apr_array_header_t *entries) (int (*)(const void *, const void *))compare_p2l_info); } -/* Decorator for svn_fs_x__p2l_entry_t that associates it with a sorted - * variant of its ITEMS array. - */ -typedef struct sub_item_ordered_t -{ - /* ENTRY that got wrapped */ - svn_fs_x__p2l_entry_t *entry; - - /* Array of pointers into ENTRY->ITEMS, sorted by their revision member - * _descending_ order. May be NULL if ENTRY->ITEM_COUNT < 2. */ - svn_fs_x__id_part_t **order; -} sub_item_ordered_t; - -/* implements compare_fn_t. Place LHS before RHS, if the latter is younger. - * Used to sort sub_item_ordered_t::order - */ -static int -compare_sub_items(const svn_fs_x__id_part_t * const * lhs, - const svn_fs_x__id_part_t * const * rhs) -{ - return (*lhs)->change_set < (*rhs)->change_set - ? 1 - : ((*lhs)->change_set > (*rhs)->change_set ? -1 : 0); -} - -/* implements compare_fn_t. Place LHS before RHS, if the latter belongs to - * a newer revision. - */ -static int -compare_p2l_info_rev(const sub_item_ordered_t * lhs, - const sub_item_ordered_t * rhs) -{ - svn_fs_x__id_part_t *lhs_part; - svn_fs_x__id_part_t *rhs_part; - - assert(lhs != rhs); - if (lhs->entry->item_count == 0) - return rhs->entry->item_count == 0 ? 0 : -1; - if (rhs->entry->item_count == 0) - return 1; - - lhs_part = lhs->order ? lhs->order[lhs->entry->item_count - 1] - : &lhs->entry->items[0]; - rhs_part = rhs->order ? rhs->order[rhs->entry->item_count - 1] - : &rhs->entry->items[0]; - - if (lhs_part->change_set == rhs_part->change_set) - return 0; - - return lhs_part->change_set < rhs_part->change_set ? -1 : 1; -} - /* implements compare_fn_t. Sort descending by PATH, NODE_ID and REVISION. */ static int @@ -840,7 +774,7 @@ compare_path_order(const path_order_t * return diff; /* reverse order on node (i.e. latest first) */ - diff = svn_fs_x__id_part_compare(&rhs->node_id, &lhs->node_id); + diff = svn_fs_x__id_compare(&rhs->node_id, &lhs->node_id); if (diff) return diff; @@ -860,8 +794,8 @@ compare_references(const reference_t * c const reference_t * lhs = *lhs_p; const reference_t * rhs = *rhs_p; - int diff = svn_fs_x__id_part_compare(&lhs->to, &rhs->to); - return diff ? diff : svn_fs_x__id_part_compare(&lhs->from, &rhs->from); + int diff = svn_fs_x__id_compare(&lhs->to, &rhs->to); + return diff ? diff : svn_fs_x__id_compare(&lhs->from, &rhs->from); } /* Order the data collected in CONTEXT such that we can place them in the @@ -882,7 +816,7 @@ sort_reps(pack_context_t *context) static apr_ssize_t get_block_left(pack_context_t *context) { - fs_x_data_t *ffd = context->fs->fsap_data; + svn_fs_x__data_t *ffd = context->fs->fsap_data; return ffd->block_size - (context->pack_offset % ffd->block_size); } @@ -895,7 +829,7 @@ static svn_error_t * auto_pad_block(pack_context_t *context, apr_pool_t *pool) { - fs_x_data_t *ffd = context->fs->fsap_data; + svn_fs_x__data_t *ffd = context->fs->fsap_data; /* This is the maximum number of bytes "wasted" that way per block. * Larger items will cross the block boundaries. */ @@ -944,7 +878,7 @@ find_first_reference(pack_context_t *con reference_t *reference = APR_ARRAY_IDX(context->references, current, reference_t *); - if (svn_fs_x__id_part_compare(&reference->to, item->items) < 0) + if (svn_fs_x__id_compare(&reference->to, item->items) < 0) lower = current + 1; else upper = current - 1; @@ -965,7 +899,7 @@ is_reference_match(pack_context_t *conte return FALSE; reference = APR_ARRAY_IDX(context->references, idx, reference_t *); - return svn_fs_x__id_part_eq(&reference->to, item->items); + return svn_fs_x__id_eq(&reference->to, item->items); } /* Starting at IDX in CONTEXT->PATH_ORDER, select all representations and @@ -1002,8 +936,7 @@ select_reps(pack_context_t *context, path_order_t *current_path = APR_ARRAY_IDX(path_order, idx, path_order_t *); - if (!svn_fs_x__id_part_eq(&start_path->node_id, - ¤t_path->node_id)) + if (!svn_fs_x__id_eq(&start_path->node_id, ¤t_path->node_id)) break; APR_ARRAY_IDX(path_order, idx, path_order_t *) = NULL; @@ -1102,7 +1035,7 @@ write_nodes_container(pack_context_t *co container_entry->type = SVN_FS_X__ITEM_TYPE_NODEREVS_CONT; container_entry->item_count = items->nelts; container_entry->items = apr_palloc(context->info_pool, - sizeof(svn_fs_x__id_part_t) * container_entry->item_count); + sizeof(svn_fs_x__id_t) * container_entry->item_count); for (i = 0; i < items->nelts; ++i) container_entry->items[i] @@ -1157,7 +1090,7 @@ store_nodes(pack_context_t *context, apr_size_t pack_savings = 0; for (i = 0; i < node_parts->nelts; ++i) { - node_revision_t *noderev; + svn_fs_x__noderev_t *noderev; svn_fs_x__p2l_entry_t *entry = APR_ARRAY_IDX(node_parts, i, svn_fs_x__p2l_entry_t *); @@ -1207,7 +1140,7 @@ store_nodes(pack_context_t *context, /* item will fit into the block. */ SVN_ERR(svn_io_file_seek(temp_file, APR_SET, &entry->offset, iterpool)); - SVN_ERR(svn_fs_x__read_noderev(&noderev, stream, iterpool)); + SVN_ERR(svn_fs_x__read_noderev(&noderev, stream, iterpool, iterpool)); svn_fs_x__noderevs_add(*container, noderev); container_size += entry->size; @@ -1245,13 +1178,13 @@ write_reps_container(pack_context_t *con SVN_ERR(svn_fs_x__write_reps_container(pack_stream, container, pool)); SVN_ERR(svn_stream_close(pack_stream)); - SVN_ERR(svn_io_file_seek(context->pack_file, SEEK_CUR, &offset, pool)); + SVN_ERR(svn_io_file_seek(context->pack_file, APR_CUR, &offset, pool)); container_entry.offset = context->pack_offset; container_entry.size = offset - container_entry.offset; container_entry.type = SVN_FS_X__ITEM_TYPE_REPS_CONT; container_entry.item_count = sub_items->nelts; - container_entry.items = (svn_fs_x__id_part_t *)sub_items->elts; + container_entry.items = (svn_fs_x__id_t *)sub_items->elts; context->pack_offset = offset; APR_ARRAY_PUSH(new_entries, svn_fs_x__p2l_entry_t *) @@ -1283,14 +1216,15 @@ write_reps_containers(pack_context_t *co svn_fs_x__reps_builder_t *container = svn_fs_x__reps_builder_create(context->fs, container_pool); apr_array_header_t *sub_items - = apr_array_make(pool, 64, sizeof(svn_fs_x__id_part_t)); - svn_stream_t *temp_stream - = svn_stream_from_aprfile2(temp_file, TRUE, pool); + = apr_array_make(pool, 64, sizeof(svn_fs_x__id_t)); + svn_fs_x__revision_file_t *file; + + SVN_ERR(svn_fs_x__wrap_temp_rev_file(&file, context->fs, temp_file, pool)); /* copy all items in strict order */ for (i = entries->nelts-1; i >= 0; --i) { - representation_t representation = { 0 }; + svn_fs_x__representation_t representation = { 0 }; svn_stringbuf_t *contents; svn_stream_t *stream; apr_size_t list_index; @@ -1327,12 +1261,12 @@ write_reps_containers(pack_context_t *co /* select the change list in the source file, parse it and add it to * the container */ - SVN_ERR(svn_io_file_seek(temp_file, SEEK_SET, &entry->offset, + SVN_ERR(svn_io_file_seek(temp_file, APR_SET, &entry->offset, iterpool)); SVN_ERR(svn_fs_x__get_representation_length(&representation.size, &representation.expanded_size, - context->fs, temp_file, - temp_stream, entry, iterpool)); + context->fs, file, + entry, iterpool)); SVN_ERR(svn_fs_x__get_contents(&stream, context->fs, &representation, FALSE, iterpool)); contents = svn_stringbuf_create_ensure(representation.expanded_size, @@ -1348,7 +1282,7 @@ write_reps_containers(pack_context_t *co SVN_ERR_ASSERT(list_index == sub_items->nelts); block_left -= entry->size; - APR_ARRAY_PUSH(sub_items, svn_fs_x__id_part_t) = entry->items[0]; + APR_ARRAY_PUSH(sub_items, svn_fs_x__id_t) = entry->items[0]; svn_pool_clear(iterpool); } @@ -1414,7 +1348,7 @@ store_items(pack_context_t *context, /* select the item in the source file and copy it into the target * pack file */ - SVN_ERR(svn_io_file_seek(temp_file, SEEK_SET, &entry->offset, + SVN_ERR(svn_io_file_seek(temp_file, APR_SET, &entry->offset, iterpool)); SVN_ERR(copy_file_data(context, context->pack_file, temp_file, entry->size, iterpool)); @@ -1444,7 +1378,7 @@ copy_reps_from_temp(pack_context_t *cont apr_file_t *temp_file, apr_pool_t *pool) { - fs_x_data_t *ffd = context->fs->fsap_data; + svn_fs_x__data_t *ffd = context->fs->fsap_data; apr_pool_t *iterpool = svn_pool_create(pool); apr_pool_t *container_pool = svn_pool_create(pool); @@ -1558,13 +1492,13 @@ write_changes_container(pack_context_t * container, pool)); SVN_ERR(svn_stream_close(pack_stream)); - SVN_ERR(svn_io_file_seek(context->pack_file, SEEK_CUR, &offset, pool)); + SVN_ERR(svn_io_file_seek(context->pack_file, APR_CUR, &offset, pool)); container_entry.offset = context->pack_offset; container_entry.size = offset - container_entry.offset; container_entry.type = SVN_FS_X__ITEM_TYPE_CHANGES_CONT; container_entry.item_count = sub_items->nelts; - container_entry.items = (svn_fs_x__id_part_t *)sub_items->elts; + container_entry.items = (svn_fs_x__id_t *)sub_items->elts; context->pack_offset = offset; APR_ARRAY_PUSH(new_entries, svn_fs_x__p2l_entry_t *) @@ -1596,7 +1530,7 @@ write_changes_containers(pack_context_t svn_fs_x__changes_t *container = svn_fs_x__changes_create(1000, container_pool); apr_array_header_t *sub_items - = apr_array_make(pool, 64, sizeof(svn_fs_x__id_part_t)); + = apr_array_make(pool, 64, sizeof(svn_fs_x__id_t)); apr_array_header_t *new_entries = apr_array_make(context->info_pool, 16, entries->elt_size); svn_stream_t *temp_stream @@ -1654,15 +1588,15 @@ write_changes_containers(pack_context_t /* select the change list in the source file, parse it and add it to * the container */ - SVN_ERR(svn_io_file_seek(temp_file, SEEK_SET, &entry->offset, + SVN_ERR(svn_io_file_seek(temp_file, APR_SET, &entry->offset, iterpool)); - SVN_ERR(svn_fs_x__read_changes(&changes, temp_stream, iterpool)); + SVN_ERR(svn_fs_x__read_changes(&changes, temp_stream, pool, iterpool)); SVN_ERR(svn_fs_x__changes_append_list(&list_index, container, changes)); SVN_ERR_ASSERT(list_index == sub_items->nelts); block_left -= estimated_size; estimated_addition += estimated_size; - APR_ARRAY_PUSH(sub_items, svn_fs_x__id_part_t) = entry->items[0]; + APR_ARRAY_PUSH(sub_items, svn_fs_x__id_t) = entry->items[0]; svn_pool_clear(iterpool); } @@ -1720,13 +1654,10 @@ static svn_error_t * write_l2p_index(pack_context_t *context, apr_pool_t *pool) { - apr_pool_t *iterpool = svn_pool_create(pool); - svn_revnum_t prev_rev = SVN_INVALID_REVNUM; - int i; - apr_uint32_t k; - svn_priority_queue__t *queue; - apr_size_t count = 0; - apr_array_header_t *sub_item_orders; + apr_pool_t *scratch_pool = svn_pool_create(pool); + const char *temp_name; + const char *proto_index; + apr_off_t offset = 0; /* lump all items into one bucket. As target, use the bucket that * probably has the most entries already. */ @@ -1734,88 +1665,24 @@ write_l2p_index(pack_context_t *context, append_entries(context->reps, context->file_props); append_entries(context->reps, context->dir_props); - /* wrap P2L entries such that we have access to the sub-items in revision - order. The ENTRY_COUNT member will point to the next item to read+1. */ - sub_item_orders - = apr_array_make(pool, context->reps->nelts, sizeof(sub_item_ordered_t)); - sub_item_orders->nelts = context->reps->nelts; - - for (i = 0; i < context->reps->nelts; ++i) - { - svn_fs_x__p2l_entry_t *entry - = APR_ARRAY_IDX(context->reps, i, svn_fs_x__p2l_entry_t *); - sub_item_ordered_t *ordered - = &APR_ARRAY_IDX(sub_item_orders, i, sub_item_ordered_t); - - /* skip unused regions (e.g. padding) */ - if (entry->item_count == 0) - continue; - - assert(entry); - ordered->entry = entry; - count += entry->item_count; - - if (entry->item_count > 1) - { - ordered->order - = apr_palloc(pool, sizeof(*ordered->order) * entry->item_count); - for (k = 0; k < entry->item_count; ++k) - ordered->order[k] = &entry->items[k]; - - qsort(ordered->order, entry->item_count, sizeof(*ordered->order), - (int (*)(const void *, const void *))compare_sub_items); - } - } - - /* we need to write the index in ascending revision order */ - queue = svn_priority_queue__create - (sub_item_orders, - (int (*)(const void *, const void *))compare_p2l_info_rev); - - /* write index entries */ - for (i = 0; i < count; ++i) - { - svn_fs_x__id_part_t *sub_item; - sub_item_ordered_t *ordered = svn_priority_queue__peek(queue); - - if (ordered->entry->item_count > 0) - { - /* if there is only one item, we skip the overhead of having an - extra array for the item order */ - sub_item = ordered->order - ? ordered->order[ordered->entry->item_count - 1] - : &ordered->entry->items[0]; - - /* next revision? */ - if (prev_rev != svn_fs_x__get_revnum(sub_item->change_set)) - { - prev_rev = svn_fs_x__get_revnum(sub_item->change_set); - SVN_ERR(svn_fs_x__l2p_proto_index_add_revision - (context->proto_l2p_index, iterpool)); - } - - /* add entry */ - SVN_ERR(svn_fs_x__l2p_proto_index_add_entry - (context->proto_l2p_index, ordered->entry->offset, - (apr_uint32_t)(sub_item - ordered->entry->items), - sub_item->number, iterpool)); - - /* make ITEM_COUNT point the next sub-item to use+1 */ - --ordered->entry->item_count; - } - - /* process remaining sub-items (if any) of that container later */ - if (ordered->entry->item_count) - svn_priority_queue__update(queue); - else - svn_priority_queue__pop(queue); - - /* keep memory usage in check */ - if (i % 256 == 0) - svn_pool_clear(iterpool); - } + /* Let the index code do the expensive L2P -> P2L transformation. */ + SVN_ERR(svn_fs_x__l2p_index_from_p2l_entries(&temp_name, + context->fs, + context->reps, + pool, scratch_pool)); + + /* Append newly written segment to exisiting proto index file. */ + SVN_ERR(svn_io_file_name_get(&proto_index, context->proto_l2p_index, + scratch_pool)); + + SVN_ERR(svn_io_file_flush(context->proto_l2p_index, scratch_pool)); + SVN_ERR(svn_io_append_file(temp_name, proto_index, scratch_pool)); + SVN_ERR(svn_io_remove_file2(temp_name, FALSE, scratch_pool)); + SVN_ERR(svn_io_file_seek(context->proto_l2p_index, APR_END, &offset, + scratch_pool)); - svn_pool_destroy(iterpool); + /* Done. */ + svn_pool_destroy(scratch_pool); return SVN_NO_ERROR; } @@ -1827,7 +1694,7 @@ static svn_error_t * pack_range(pack_context_t *context, apr_pool_t *pool) { - fs_x_data_t *ffd = context->fs->fsap_data; + svn_fs_x__data_t *ffd = context->fs->fsap_data; apr_pool_t *revpool = svn_pool_create(pool); apr_pool_t *iterpool = svn_pool_create(pool); @@ -1836,26 +1703,19 @@ pack_range(pack_context_t *context, for (revision = context->start_rev; revision < context->end_rev; ++revision) { apr_off_t offset = 0; - apr_finfo_t finfo; - apr_file_t *rev_file; + svn_fs_x__revision_file_t *rev_file; - /* Get the size of the file. */ - const char *path = svn_dirent_join(context->shard_dir, - apr_psprintf(revpool, "%ld", - revision), - revpool); - SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, revpool)); - - SVN_ERR(svn_io_file_open(&rev_file, path, - APR_READ | APR_BUFFERED | APR_BINARY, - APR_OS_DEFAULT, revpool)); + /* Get the rev file dimensions (mainly index locations). */ + SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, context->fs, + revision, revpool, iterpool)); + SVN_ERR(svn_fs_x__auto_read_footer(rev_file)); /* store the indirect array index */ APR_ARRAY_PUSH(context->rev_offsets, int) = context->reps->nelts; /* read the phys-to-log index file until we covered the whole rev file. * That index contains enough info to build both target indexes from it. */ - while (offset < finfo.size) + while (offset < rev_file->l2p_offset) { /* read one cluster */ int i; @@ -1863,8 +1723,9 @@ pack_range(pack_context_t *context, svn_pool_clear(iterpool); SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, context->fs, - revision, offset, - ffd->p2l_page_size, iterpool)); + rev_file, revision, offset, + ffd->p2l_page_size, iterpool, + iterpool)); for (i = 0; i < entries->nelts; ++i) { @@ -1878,9 +1739,9 @@ pack_range(pack_context_t *context, /* process entry while inside the rev file */ offset = entry->offset; - if (offset < finfo.size) + if (offset < rev_file->l2p_offset) { - SVN_ERR(svn_io_file_seek(rev_file, SEEK_SET, &offset, + SVN_ERR(svn_io_file_seek(rev_file->file, APR_SET, &offset, iterpool)); if (entry->type == SVN_FS_X__ITEM_TYPE_CHANGES) @@ -1959,10 +1820,10 @@ static svn_error_t * append_revision(pack_context_t *context, apr_pool_t *pool) { - fs_x_data_t *ffd = context->fs->fsap_data; + svn_fs_x__data_t *ffd = context->fs->fsap_data; apr_off_t offset = 0; apr_pool_t *iterpool = svn_pool_create(pool); - apr_file_t *rev_file; + svn_fs_x__revision_file_t *rev_file; apr_finfo_t finfo; /* Get the size of the file. */ @@ -1973,11 +1834,11 @@ append_revision(pack_context_t *context, SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, pool)); /* Copy all the bits from the rev file to the end of the pack file. */ - SVN_ERR(svn_io_file_open(&rev_file, path, - APR_READ | APR_BUFFERED | APR_BINARY, - APR_OS_DEFAULT, pool)); - SVN_ERR(copy_file_data(context, context->pack_file, rev_file, finfo.size, - iterpool)); + SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, context->fs, + context->start_rev, pool, + iterpool)); + SVN_ERR(copy_file_data(context, context->pack_file, rev_file->file, + finfo.size, iterpool)); /* mark the start of a new revision */ SVN_ERR(svn_fs_x__l2p_proto_index_add_revision(context->proto_l2p_index, @@ -1990,9 +1851,10 @@ append_revision(pack_context_t *context, /* read one cluster */ int i; apr_array_header_t *entries; - SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, context->fs, - context->start_rev, offset, - ffd->p2l_page_size, iterpool)); + SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, context->fs, rev_file, + context->start_rev, offset, + ffd->p2l_page_size, iterpool, + iterpool)); for (i = 0; i < entries->nelts; ++i) { @@ -2075,7 +1937,7 @@ pack_log_addressed(svn_fs_t *fs, /* phase 1: determine the size of the revisions to pack */ SVN_ERR(svn_fs_x__l2p_get_max_ids(&max_ids, fs, shard_rev, context.shard_end_rev - shard_rev, - pool)); + pool, pool)); /* pack revisions in ranges that don't exceed MAX_MEM */ for (i = 0; i < max_ids->nelts; ++i) @@ -2130,7 +1992,7 @@ svn_fs_x__get_packed_offset(apr_off_t *r svn_revnum_t rev, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; svn_stream_t *manifest_stream; svn_boolean_t is_cached; svn_revnum_t shard; @@ -2256,7 +2118,7 @@ pack_shard(const char *revs_dir, void *cancel_baton, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; const char *rev_shard_path, *rev_pack_file_dir; const char *revprops_shard_path, *revprops_pack_file_dir; @@ -2372,7 +2234,7 @@ pack_body(void *baton, apr_pool_t *pool) { struct pack_baton *pb = baton; - fs_x_data_t *ffd = pb->fs->fsap_data; + svn_fs_x__data_t *ffd = pb->fs->fsap_data; apr_int64_t completed_shards; apr_int64_t i; svn_revnum_t youngest; Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_x/recovery.c Sat Jan 3 14:00:41 2015 @@ -55,10 +55,11 @@ recover_get_largest_revision(svn_fs_t *f while (1) { svn_error_t *err; - apr_file_t *file; + svn_fs_x__revision_file_t *file; svn_pool_clear(iterpool); - err = svn_fs_x__open_pack_or_rev_file(&file, fs, right, iterpool); + err = svn_fs_x__open_pack_or_rev_file(&file, fs, right, iterpool, + iterpool); if (err && err->apr_err == SVN_ERR_FS_NO_SUCH_REVISION) { svn_error_clear(err); @@ -78,10 +79,11 @@ recover_get_largest_revision(svn_fs_t *f { svn_revnum_t probe = left + ((right - left) / 2); svn_error_t *err; - apr_file_t *file; + svn_fs_x__revision_file_t *file; svn_pool_clear(iterpool); - err = svn_fs_x__open_pack_or_rev_file(&file, fs, probe, iterpool); + err = svn_fs_x__open_pack_or_rev_file(&file, fs, probe, iterpool, + iterpool); if (err && err->apr_err == SVN_ERR_FS_NO_SUCH_REVISION) { svn_error_clear(err); @@ -116,13 +118,18 @@ recover_body(void *baton, apr_pool_t *po { struct recover_baton *b = baton; svn_fs_t *fs = b->fs; - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; svn_revnum_t max_rev; svn_revnum_t youngest_rev; svn_node_kind_t youngest_revprops_kind; /* Lose potentially corrupted data in temp files */ - SVN_ERR(svn_fs_x__cleanup_revprop_namespace(fs)); + SVN_ERR(svn_fs_x__reset_revprop_generation_file(fs, pool)); + + /* The admin may have created a plain copy of this repo before attempting + to recover it (hotcopy may or may not work with corrupted repos). + Bump the instance ID. */ + SVN_ERR(svn_fs_x__set_uuid(fs, fs->uuid, NULL, pool)); /* We need to know the largest revision in the filesystem. */ SVN_ERR(recover_get_largest_revision(fs, &max_rev, pool)); Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.c Sat Jan 3 14:00:41 2015 @@ -61,7 +61,7 @@ open_rep_cache(void *baton, apr_pool_t *pool) { svn_fs_t *fs = baton; - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; svn_sqlite__db_t *sdb; const char *db_path; int version; @@ -118,7 +118,7 @@ svn_error_t * svn_fs_x__open_rep_cache(svn_fs_t *fs, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; svn_error_t *err = svn_atomic__init_once(&ffd->rep_cache_db_opened, open_rep_cache, fs, pool); return svn_error_quick_wrap(err, _("Couldn't open rep-cache database")); @@ -141,7 +141,7 @@ svn_error_t * svn_fs_x__walk_rep_reference(svn_fs_t *fs, svn_revnum_t start, svn_revnum_t end, - svn_error_t *(*walker)(representation_t *, + svn_error_t *(*walker)(svn_fs_x__representation_t *, void *, svn_fs_t *, apr_pool_t *), @@ -150,7 +150,7 @@ svn_fs_x__walk_rep_reference(svn_fs_t *f void *cancel_baton, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; svn_sqlite__stmt_t *stmt; svn_boolean_t have_row; int iterations = 0; @@ -183,7 +183,7 @@ svn_fs_x__walk_rep_reference(svn_fs_t *f SVN_ERR(svn_sqlite__step(&have_row, stmt)); while (have_row) { - representation_t *rep; + svn_fs_x__representation_t *rep; const char *sha1_digest; svn_error_t *err; svn_checksum_t *checksum; @@ -200,7 +200,7 @@ svn_fs_x__walk_rep_reference(svn_fs_t *f return svn_error_compose_create(err, svn_sqlite__reset(stmt)); } - /* Construct a representation_t. */ + /* Construct a svn_fs_x__representation_t. */ rep = apr_pcalloc(iterpool, sizeof(*rep)); sha1_digest = svn_sqlite__column_text(stmt, 0, iterpool); err = svn_checksum_parse_hex(&checksum, svn_checksum_sha1, @@ -234,12 +234,12 @@ svn_fs_x__walk_rep_reference(svn_fs_t *f If you extend this function, check the callsite to see if you have to make it not-ignore additional error codes. */ svn_error_t * -svn_fs_x__get_rep_reference(representation_t **rep, +svn_fs_x__get_rep_reference(svn_fs_x__representation_t **rep, svn_fs_t *fs, svn_checksum_t *checksum, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; svn_sqlite__stmt_t *stmt; svn_boolean_t have_row; @@ -291,10 +291,10 @@ svn_fs_x__get_rep_reference(representati svn_error_t * svn_fs_x__set_rep_reference(svn_fs_t *fs, - representation_t *rep, + svn_fs_x__representation_t *rep, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; svn_sqlite__stmt_t *stmt; svn_error_t *err; svn_checksum_t checksum; @@ -322,7 +322,7 @@ svn_fs_x__set_rep_reference(svn_fs_t *fs err = svn_sqlite__insert(NULL, stmt); if (err) { - representation_t *old_rep; + svn_fs_x__representation_t *old_rep; if (err->apr_err != SVN_ERR_SQLITE_CONSTRAINT) return svn_error_trace(err); @@ -351,7 +351,7 @@ svn_fs_x__del_rep_reference(svn_fs_t *fs svn_revnum_t youngest, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; svn_sqlite__stmt_t *stmt; if (! ffd->rep_cache_db) @@ -369,7 +369,7 @@ svn_error_t * svn_fs_x__lock_rep_cache(svn_fs_t *fs, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; if (! ffd->rep_cache_db) SVN_ERR(svn_fs_x__open_rep_cache(fs, pool)); Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.h URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.h?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.h (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_x/rep-cache.h Sat Jan 3 14:00:41 2015 @@ -51,7 +51,7 @@ svn_error_t * svn_fs_x__walk_rep_reference(svn_fs_t *fs, svn_revnum_t start, svn_revnum_t end, - svn_error_t *(*walker)(representation_t *rep, + svn_error_t *(*walker)(svn_fs_x__representation_t *rep, void *walker_baton, svn_fs_t *fs, apr_pool_t *scratch_pool), @@ -65,7 +65,7 @@ svn_fs_x__walk_rep_reference(svn_fs_t *f opened, just set *REP to NULL. Returns SVN_ERR_FS_CORRUPT if a reference beyond HEAD is detected. */ svn_error_t * -svn_fs_x__get_rep_reference(representation_t **rep, +svn_fs_x__get_rep_reference(svn_fs_x__representation_t **rep, svn_fs_t *fs, svn_checksum_t *checksum, apr_pool_t *pool); @@ -77,7 +77,7 @@ svn_fs_x__get_rep_reference(representati If the rep cache database has not been opened, this may be a no op. */ svn_error_t * svn_fs_x__set_rep_reference(svn_fs_t *fs, - representation_t *rep, + svn_fs_x__representation_t *rep, apr_pool_t *pool); /* Delete from the cache all reps corresponding to revisions younger Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_x/reps.c Sat Jan 3 14:00:41 2015 @@ -401,7 +401,7 @@ svn_fs_x__reps_builder_create(svn_fs_t * svn_error_t * svn_fs_x__reps_add_base(svn_fs_x__reps_builder_t *builder, - representation_t *rep, + svn_fs_x__representation_t *rep, int priority, apr_pool_t *scratch_pool) { Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/reps.h URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/reps.h?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_x/reps.h (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_x/reps.h Sat Jan 3 14:00:41 2015 @@ -91,7 +91,7 @@ svn_fs_x__reps_builder_create(svn_fs_t * */ svn_error_t * svn_fs_x__reps_add_base(svn_fs_x__reps_builder_t *builder, - representation_t *rep, + svn_fs_x__representation_t *rep, int priority, apr_pool_t *scratch_pool); Modified: subversion/branches/authzperf/subversion/libsvn_fs_x/revprops.c URL: http://svn.apache.org/viewvc/subversion/branches/authzperf/subversion/libsvn_fs_x/revprops.c?rev=1649205&r1=1649204&r2=1649205&view=diff ============================================================================== --- subversion/branches/authzperf/subversion/libsvn_fs_x/revprops.c (original) +++ subversion/branches/authzperf/subversion/libsvn_fs_x/revprops.c Sat Jan 3 14:00:41 2015 @@ -21,6 +21,7 @@ */ #include <assert.h> +#include <apr_md5.h> #include "svn_pools.h" #include "svn_hash.h" @@ -42,11 +43,15 @@ process got aborted and that we have re-read revprops. */ #define REVPROP_CHANGE_TIMEOUT (10 * 1000000) -/* The following are names of atomics that will be used to communicate - * revprop updates across all processes on this machine. */ -#define ATOMIC_REVPROP_GENERATION "rev-prop-generation" -#define ATOMIC_REVPROP_TIMEOUT "rev-prop-timeout" -#define ATOMIC_REVPROP_NAMESPACE "rev-prop-atomics" +/* In case of an inconsistent read, close the generation file, yield, + re-open and re-read. This is the number of times we try this before + giving up. */ +#define GENERATION_READ_RETRY_COUNT 100 + +/* Maximum size of the generation number file contents (including NUL). */ +#define CHECKSUMMED_NUMBER_BUFFER_LEN \ + (SVN_INT64_BUFFER_SIZE + 3 + APR_MD5_DIGESTSIZE * 2) + svn_error_t * svn_fs_x__upgrade_pack_revprops(svn_fs_t *fs, @@ -56,7 +61,7 @@ svn_fs_x__upgrade_pack_revprops(svn_fs_t void *cancel_baton, apr_pool_t *scratch_pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; const char *revprops_shard_path; const char *revprops_pack_file_dir; apr_int64_t shard; @@ -108,7 +113,7 @@ svn_fs_x__upgrade_cleanup_pack_revprops( void *cancel_baton, apr_pool_t *scratch_pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; const char *revprops_shard_path; apr_int64_t shard; apr_int64_t first_unpacked_shard @@ -147,179 +152,251 @@ svn_fs_x__upgrade_cleanup_pack_revprops( * * Revprop caching needs to be activated and will be deactivated for the * respective FS instance if the necessary infrastructure could not be - * initialized. In deactivated mode, there is almost no runtime overhead - * associated with revprop caching. As long as no revprops are being read - * or changed, revprop caching imposes no overhead. + * initialized. As long as no revprops are being read or changed, revprop + * caching imposes no overhead. * * When activated, we cache revprops using (revision, generation) pairs * as keys with the generation being incremented upon every revprop change. * Since the cache is process-local, the generation needs to be tracked * for at least as long as the process lives but may be reset afterwards. * - * To track the revprop generation, we use two-layer approach. On the lower - * level, we use named atomics to have a system-wide consistent value for - * the current revprop generation. However, those named atomics will only - * remain valid for as long as at least one process / thread in the system - * accesses revprops in the respective repository. The underlying shared - * memory gets cleaned up afterwards. - * - * On the second level, we will use a persistent file to track the latest - * revprop generation. It will be written upon each revprop change but - * only be read if we are the first process to initialize the named atomics - * with that value. - * - * The overhead for the second and following accesses to revprops is - * almost zero on most systems. - * - * - * Tech aspects: - * ------------- - * - * A problem is that we need to provide a globally available file name to - * back the SHM implementation on OSes that need it. We can only assume - * write access to some file within the respective repositories. Because - * a given server process may access thousands of repositories during its - * lifetime, keeping the SHM data alive for all of them is also not an - * option. - * - * So, we store the new revprop generation on disk as part of each - * setrevprop call, i.e. this write will be serialized and the write order - * be guaranteed by the repository write lock. - * - * The only racy situation occurs when the data is being read again by two - * processes concurrently but in that situation, the first process to - * finish that procedure is guaranteed to be the only one that initializes - * the SHM data. Since even writers will first go through that - * initialization phase, they will never operate on stale data. + * We track the revprop generation in a persistent, unbuffered file that + * we may keep open for the lifetime of the svn_fs_t. It is the OS' + * responsibility to provide us with the latest contents upon read. To + * detect incomplete updates due to non-atomic reads, we put a MD5 checksum + * next to the actual generation number and verify that it matches. + * + * Since we cannot guarantee that the OS will provide us with up-to-date + * data buffers for open files, we re-open and re-read the file before + * modifying it. This will prevent lost updates. + * + * A race condition exists between switching to the modified revprop data + * and bumping the generation number. In particular, the process may crash + * just after switching to the new revprop data and before bumping the + * generation. To be able to detect this scenario, we bump the generation + * twice per revprop change: once immediately before (creating an odd number) + * and once after the atomic switch (even generation). + * + * A writer holding the write lock can immediately assume a crashed writer + * in case of an odd generation or they would not have been able to acquire + * the lock. A reader detecting an odd generation will use that number and + * be forced to re-read any revprop data - usually getting the new revprops + * already. If the generation file modification timestamp is too old, the + * reader will assume a crashed writer, acquire the write lock and bump + * the generation if it is still odd. So, for about REVPROP_CHANGE_TIMEOUT + * after the crash, reader caches may be stale. */ -/* Read revprop generation as stored on disk for repository FS. The result - * is returned in *CURRENT. Default to 2 if no such file is available. +/* If the revprop generation file in FS is open, close it. This is a no-op + * if the file is not open. */ static svn_error_t * -read_revprop_generation_file(apr_int64_t *current, - svn_fs_t *fs, - apr_pool_t *pool) +close_revprop_generation_file(svn_fs_t *fs, + apr_pool_t *scratch_pool) { - svn_error_t *err; - apr_file_t *file; - char buf[80]; - apr_size_t len; - const char *path = svn_fs_x__path_revprop_generation(fs, pool); - - err = svn_io_file_open(&file, path, - APR_READ | APR_BUFFERED, - APR_OS_DEFAULT, pool); - if (err && APR_STATUS_IS_ENOENT(err->apr_err)) + svn_fs_x__data_t *ffd = fs->fsap_data; + if (ffd->revprop_generation_file) { - svn_error_clear(err); - *current = 2; - - return SVN_NO_ERROR; + SVN_ERR(svn_io_file_close(ffd->revprop_generation_file, scratch_pool)); + ffd->revprop_generation_file = NULL; } - SVN_ERR(err); - len = sizeof(buf); - SVN_ERR(svn_io_read_length_line(file, buf, &len, pool)); + return SVN_NO_ERROR; +} - /* Check that the first line contains only digits. */ - SVN_ERR(svn_fs_x__check_file_buffer_numeric(buf, 0, path, - "Revprop Generation", pool)); - SVN_ERR(svn_cstring_atoi64(current, buf)); +/* Make sure the revprop_generation member in FS is set. If READ_ONLY is + * set, open the file w/o write permission if the file is not open yet. + * The file is kept open if it has sufficient rights (or more) but will be + * closed and re-opened if it provided insufficient access rights. + * + * Call only for repos that support revprop caching. + */ +static svn_error_t * +open_revprop_generation_file(svn_fs_t *fs, + svn_boolean_t read_only, + apr_pool_t *scratch_pool) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + apr_int32_t flags = read_only ? APR_READ : (APR_READ | APR_WRITE); + + /* Close the current file handle if it has insufficient rights. */ + if ( ffd->revprop_generation_file + && (apr_file_flags_get(ffd->revprop_generation_file) & flags) != flags) + SVN_ERR(close_revprop_generation_file(fs, scratch_pool)); + + /* If not open already, open with sufficient rights. */ + if (ffd->revprop_generation_file == NULL) + { + const char *path = svn_fs_x__path_revprop_generation(fs, scratch_pool); + SVN_ERR(svn_io_file_open(&ffd->revprop_generation_file, path, + flags, APR_OS_DEFAULT, fs->pool)); + } - return svn_io_file_close(file, pool); + return SVN_NO_ERROR; } -/* Write the CURRENT revprop generation to disk for repository FS. +/* Return the textual representation of NUMBER and its checksum in *BUFFER. */ -svn_error_t * -svn_fs_x__write_revprop_generation_file(svn_fs_t *fs, - apr_int64_t current, - apr_pool_t *pool) +static svn_error_t * +checkedsummed_number(svn_stringbuf_t **buffer, + apr_int64_t number, + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { - char buf[SVN_INT64_BUFFER_SIZE]; - apr_size_t len = svn__i64toa(buf, current); - buf[len] = '\n'; + svn_checksum_t *checksum; + const char *digest; + + char str[SVN_INT64_BUFFER_SIZE]; + apr_size_t len = svn__i64toa(str, number); + str[len] = 0; - SVN_ERR(svn_io_write_atomic(svn_fs_x__path_revprop_generation(fs, pool), - buf, len + 1, - NULL /* copy_perms */, pool)); + SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, str, len, scratch_pool)); + digest = svn_checksum_to_cstring_display(checksum, scratch_pool); + + *buffer = svn_stringbuf_createf(result_pool, "%s %s\n", digest, str); return SVN_NO_ERROR; } -/* Make sure the revprop_namespace member in FS is set. */ +/* Extract the generation number from the text BUFFER of LEN bytes and + * verify it against the checksum in the same BUFFER. If they match, return + * the generation in *NUMBER. Otherwise, return an error. + * BUFFER does not need to be NUL-terminated. + */ static svn_error_t * -ensure_revprop_namespace(svn_fs_t *fs) +verify_extract_number(apr_int64_t *number, + const char *buffer, + apr_size_t len, + apr_pool_t *scratch_pool) { - fs_x_data_t *ffd = fs->fsap_data; + const char *digest_end = strchr(buffer, ' '); - return ffd->revprop_namespace == NULL - ? svn_atomic_namespace__create(&ffd->revprop_namespace, - svn_dirent_join(fs->path, - ATOMIC_REVPROP_NAMESPACE, - fs->pool), - fs->pool) - : SVN_NO_ERROR; -} + /* Does the buffer even contain checksum _and_ number? */ + if (digest_end != NULL) + { + svn_checksum_t *expected; + svn_checksum_t *actual; -svn_error_t * -svn_fs_x__cleanup_revprop_namespace(svn_fs_t *fs) -{ - const char *name = svn_dirent_join(fs->path, - ATOMIC_REVPROP_NAMESPACE, - fs->pool); - return svn_error_trace(svn_atomic_namespace__cleanup(name, fs->pool)); + SVN_ERR(svn_checksum_parse_hex(&expected, svn_checksum_md5, buffer, + scratch_pool)); + SVN_ERR(svn_checksum(&actual, svn_checksum_md5, digest_end + 1, + (buffer + len) - (digest_end + 1), scratch_pool)); + + if (svn_checksum_match(expected, actual)) + return svn_error_trace(svn_cstring_atoi64(number, digest_end + 1)); + } + + /* Incomplete buffer or not a match. */ + return svn_error_create(SVN_ERR_FS_INVALID_GENERATION, NULL, + _("Invalid generation number data.")); } -/* Make sure the revprop_generation member in FS is set and, if necessary, - * initialized with the latest value stored on disk. +/* Read revprop generation as stored on disk for repository FS. The result is + * returned in *CURRENT. Call only for repos that support revprop caching. */ static svn_error_t * -ensure_revprop_generation(svn_fs_t *fs, apr_pool_t *pool) +read_revprop_generation_file(apr_int64_t *current, + svn_fs_t *fs, + apr_pool_t *scratch_pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; + apr_pool_t *iterpool = svn_pool_create(scratch_pool); + char buf[CHECKSUMMED_NUMBER_BUFFER_LEN]; + apr_size_t len; + apr_off_t offset = 0; + int i; + svn_error_t *err = SVN_NO_ERROR; - SVN_ERR(ensure_revprop_namespace(fs)); - if (ffd->revprop_generation == NULL) + /* Retry in case of incomplete file buffer updates. */ + for (i = 0; i < GENERATION_READ_RETRY_COUNT; ++i) { - apr_int64_t current; - - SVN_ERR(svn_named_atomic__get(&ffd->revprop_generation, - ffd->revprop_namespace, - ATOMIC_REVPROP_GENERATION, - TRUE)); + svn_error_clear(err); + svn_pool_clear(iterpool); - /* If the generation is at 0, we just created a new namespace - * (it would be at least 2 otherwise). Read the latest generation - * from disk and if we are the first one to initialize the atomic - * (i.e. is still 0), set it to the value just gotten. + /* If we can't even access the data, things are very wrong. + * Don't retry in that case. */ - SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation)); - if (current == 0) - { - SVN_ERR(read_revprop_generation_file(¤t, fs, pool)); - SVN_ERR(svn_named_atomic__cmpxchg(NULL, current, 0, - ffd->revprop_generation)); - } + SVN_ERR(open_revprop_generation_file(fs, TRUE, iterpool)); + SVN_ERR(svn_io_file_seek(ffd->revprop_generation_file, APR_SET, &offset, + iterpool)); + + len = sizeof(buf); + SVN_ERR(svn_io_read_length_line(ffd->revprop_generation_file, buf, &len, + iterpool)); + + /* Some data has been read. It will most likely be complete and + * consistent. Extract and verify anyway. */ + err = verify_extract_number(current, buf, len, iterpool); + if (!err) + break; + + /* Got unlucky and data was invalid. Retry. */ + SVN_ERR(close_revprop_generation_file(fs, iterpool)); + +#if APR_HAS_THREADS + apr_thread_yield(); +#else + apr_sleep(0); +#endif } - return SVN_NO_ERROR; + svn_pool_destroy(iterpool); + + /* If we had to give up, propagate the error. */ + return svn_error_trace(err); } -/* Make sure the revprop_timeout member in FS is set. */ +/* Write the CURRENT revprop generation to disk for repository FS. + * Call only for repos that support revprop caching. + */ static svn_error_t * -ensure_revprop_timeout(svn_fs_t *fs) +write_revprop_generation_file(svn_fs_t *fs, + apr_int64_t current, + apr_pool_t *scratch_pool) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + svn_stringbuf_t *buffer; + apr_off_t offset = 0; + + SVN_ERR(checkedsummed_number(&buffer, current, scratch_pool, scratch_pool)); + + SVN_ERR(open_revprop_generation_file(fs, FALSE, scratch_pool)); + SVN_ERR(svn_io_file_seek(ffd->revprop_generation_file, APR_SET, &offset, + scratch_pool)); + SVN_ERR(svn_io_file_write_full(ffd->revprop_generation_file, buffer->data, + buffer->len, NULL, scratch_pool)); + SVN_ERR(svn_io_file_flush_to_disk(ffd->revprop_generation_file, + scratch_pool)); + + return SVN_NO_ERROR; +} + +svn_error_t * +svn_fs_x__reset_revprop_generation_file(svn_fs_t *fs, + apr_pool_t *scratch_pool) { - fs_x_data_t *ffd = fs->fsap_data; + const char *path = svn_fs_x__path_revprop_generation(fs, scratch_pool); + svn_stringbuf_t *buffer; + + /* Unconditionally close the revprop generation file. + * Don't care about FS formats. This ensures consistent internal state. */ + SVN_ERR(close_revprop_generation_file(fs, scratch_pool)); + + /* Unconditionally remove any old revprop generation file. + * Don't care about FS formats. This ensures consistent on-disk state + * for old format repositories. */ + SVN_ERR(svn_io_remove_file2(path, TRUE, scratch_pool)); + + /* Write the initial revprop generation file contents, if supported by + * the current format. This ensures consistent on-disk state for new + * format repositories. */ + SVN_ERR(checkedsummed_number(&buffer, 0, scratch_pool, scratch_pool)); + SVN_ERR(svn_io_write_atomic(path, buffer->data, buffer->len, NULL, + scratch_pool)); - SVN_ERR(ensure_revprop_namespace(fs)); - return ffd->revprop_timeout == NULL - ? svn_named_atomic__get(&ffd->revprop_timeout, - ffd->revprop_namespace, - ATOMIC_REVPROP_TIMEOUT, - TRUE) - : SVN_NO_ERROR; + /* ffd->revprop_generation_file will be re-opened on demand. */ + + return SVN_NO_ERROR; } /* Create an error object with the given MESSAGE and pass it to the @@ -344,32 +421,18 @@ log_revprop_cache_init_warning(svn_fs_t /* Test whether revprop cache and necessary infrastructure are available in FS. */ static svn_boolean_t -has_revprop_cache(svn_fs_t *fs, apr_pool_t *pool) +has_revprop_cache(svn_fs_t *fs, + apr_pool_t *scratch_pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; svn_error_t *error; /* is the cache (still) enabled? */ if (ffd->revprop_cache == NULL) return FALSE; - /* is it efficient? */ - if (!svn_named_atomic__is_efficient()) - { - /* access to it would be quite slow - * -> disable the revprop cache for good - */ - ffd->revprop_cache = NULL; - log_revprop_cache_init_warning(fs, NULL, - "Revprop caching for '%s' disabled" - " because it would be inefficient.", - pool); - - return FALSE; - } - - /* try to access our SHM-backed infrastructure */ - error = ensure_revprop_generation(fs, pool); + /* try initialize our file-backed infrastructure */ + error = open_revprop_generation_file(fs, TRUE, scratch_pool); if (error) { /* failure -> disable revprop cache for good */ @@ -377,9 +440,9 @@ has_revprop_cache(svn_fs_t *fs, apr_pool ffd->revprop_cache = NULL; log_revprop_cache_init_warning(fs, error, "Revprop caching for '%s' disabled " - "because SHM infrastructure for revprop " + "because infrastructure for revprop " "caching failed to initialize.", - pool); + scratch_pool); return FALSE; } @@ -393,8 +456,8 @@ typedef struct revprop_generation_fixup_ /* revprop generation to read */ apr_int64_t *generation; - /* containing the revprop_generation member to query */ - fs_x_data_t *ffd; + /* file system context */ + svn_fs_t *fs; } revprop_generation_upgrade_t; /* If the revprop generation has an odd value, it means the original writer @@ -406,23 +469,31 @@ typedef struct revprop_generation_fixup_ */ static svn_error_t * revprop_generation_fixup(void *void_baton, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { revprop_generation_upgrade_t *baton = void_baton; - assert(baton->ffd->has_write_lock); + svn_fs_x__data_t *ffd = baton->fs->fsap_data; + assert(ffd->has_write_lock); + + /* Make sure we don't operate on stale OS buffers. */ + SVN_ERR(close_revprop_generation_file(baton->fs, scratch_pool)); /* Maybe, either the original revprop writer or some other reader has already corrected / bumped the revprop generation. Thus, we need - to read it again. */ - SVN_ERR(svn_named_atomic__read(baton->generation, - baton->ffd->revprop_generation)); + to read it again. However, we will now be the only ones changing + the file contents due to us holding the write lock. */ + SVN_ERR(read_revprop_generation_file(baton->generation, baton->fs, + scratch_pool)); /* Cause everyone to re-read revprops upon their next access, if the last revprop write did not complete properly. */ - while (*baton->generation % 2) - SVN_ERR(svn_named_atomic__add(baton->generation, - 1, - baton->ffd->revprop_generation)); + if (*baton->generation % 2) + { + ++*baton->generation; + SVN_ERR(write_revprop_generation_file(baton->fs, + *baton->generation, + scratch_pool)); + } return SVN_NO_ERROR; } @@ -433,42 +504,46 @@ revprop_generation_fixup(void *void_bato static svn_error_t * read_revprop_generation(apr_int64_t *generation, svn_fs_t *fs, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { apr_int64_t current = 0; - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; /* read the current revprop generation number */ - SVN_ERR(ensure_revprop_generation(fs, pool)); - SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation)); + SVN_ERR(read_revprop_generation_file(¤t, fs, scratch_pool)); /* is an unfinished revprop write under the way? */ if (current % 2) { - apr_int64_t timeout = 0; + svn_boolean_t timeout = FALSE; - /* read timeout for the write operation */ - SVN_ERR(ensure_revprop_timeout(fs)); - SVN_ERR(svn_named_atomic__read(&timeout, ffd->revprop_timeout)); - - /* has the writer process been aborted, - * i.e. has the timeout been reached? + /* Has the writer process been aborted? + * Either by timeout or by us being the writer now. */ - if (apr_time_now() > timeout) + if (!ffd->has_write_lock) + { + apr_time_t mtime; + SVN_ERR(svn_io_file_affected_time(&mtime, + svn_fs_x__path_revprop_generation(fs, scratch_pool), + scratch_pool)); + timeout = apr_time_now() > mtime + REVPROP_CHANGE_TIMEOUT; + } + + if (ffd->has_write_lock || timeout) { revprop_generation_upgrade_t baton; baton.generation = ¤t; - baton.ffd = ffd; + baton.fs = fs; /* Ensure that the original writer process no longer exists by * acquiring the write lock to this repository. Then, fix up * the revprop generation. */ if (ffd->has_write_lock) - SVN_ERR(revprop_generation_fixup(&baton, pool)); + SVN_ERR(revprop_generation_fixup(&baton, scratch_pool)); else SVN_ERR(svn_fs_x__with_write_lock(fs, revprop_generation_fixup, - &baton, pool)); + &baton, scratch_pool)); } } @@ -477,64 +552,54 @@ read_revprop_generation(apr_int64_t *gen return SVN_NO_ERROR; } -/* Set the revprop generation to the next odd number to indicate that - there is a revprop write process under way. If that times out, - readers shall recover from that state & re-read revprops. - Use the access object in FS to set the shared mem value. */ -static svn_error_t * -begin_revprop_change(svn_fs_t *fs, apr_pool_t *pool) -{ - apr_int64_t current; - fs_x_data_t *ffd = fs->fsap_data; - - /* set the timeout for the write operation */ - SVN_ERR(ensure_revprop_timeout(fs)); - SVN_ERR(svn_named_atomic__write(NULL, - apr_time_now() + REVPROP_CHANGE_TIMEOUT, - ffd->revprop_timeout)); +/* Set the revprop generation in FS to the next odd number to indicate + that there is a revprop write process under way. Return that value + in *GENERATION. If the change times out, readers shall recover from + that state & re-read revprops. + This is a no-op for repo formats that don't support revprop caching. */ +static svn_error_t * +begin_revprop_change(apr_int64_t *generation, + svn_fs_t *fs, + apr_pool_t *scratch_pool) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + SVN_ERR_ASSERT(ffd->has_write_lock); + + /* Close and re-open to make sure we read the latest data. */ + SVN_ERR(close_revprop_generation_file(fs, scratch_pool)); + SVN_ERR(open_revprop_generation_file(fs, FALSE, scratch_pool)); - /* set the revprop generation to an odd value to indicate - * that a write is in progress + /* Set the revprop generation to an odd value to indicate + * that a write is in progress. */ - SVN_ERR(ensure_revprop_generation(fs, pool)); - do - { - SVN_ERR(svn_named_atomic__add(¤t, - 1, - ffd->revprop_generation)); - } - while (current % 2 == 0); + SVN_ERR(read_revprop_generation(generation, fs, scratch_pool)); + ++*generation; + SVN_ERR(write_revprop_generation_file(fs, *generation, scratch_pool)); return SVN_NO_ERROR; } -/* Set the revprop generation to the next even number to indicate that +/* Set the revprop generation in FS to the next even generation after + the odd value in GENERATION to indicate that a) readers shall re-read revprops, and - b) the write process has been completed (no recovery required) - Use the access object in FS to set the shared mem value. */ + b) the write process has been completed (no recovery required). + This is a no-op for repo formats that don't support revprop caching. */ static svn_error_t * -end_revprop_change(svn_fs_t *fs, apr_pool_t *pool) -{ - apr_int64_t current = 1; - fs_x_data_t *ffd = fs->fsap_data; - - /* set the revprop generation to an even value to indicate - * that a write has been completed - */ - SVN_ERR(ensure_revprop_generation(fs, pool)); - do - { - SVN_ERR(svn_named_atomic__add(¤t, - 1, - ffd->revprop_generation)); - } - while (current % 2); - - /* Save the latest generation to disk. FS is currently in a "locked" - * state such that we can be sure the be the only ones to write that - * file. +end_revprop_change(svn_fs_t *fs, + apr_int64_t generation, + apr_pool_t *scratch_pool) +{ + svn_fs_x__data_t *ffd = fs->fsap_data; + SVN_ERR_ASSERT(ffd->has_write_lock); + SVN_ERR_ASSERT(generation % 2); + + /* Set the revprop generation to an even value to indicate + * that a write has been completed. Since we held the write + * lock, nobody else could have updated the file contents. */ - return svn_fs_x__write_revprop_generation_file(fs, current, pool); + SVN_ERR(write_revprop_generation_file(fs, generation + 1, scratch_pool)); + + return SVN_NO_ERROR; } /* Container for all data required to access the packed revprop file @@ -613,8 +678,8 @@ parse_revprop(apr_hash_t **properties, SVN_ERR(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool)); if (has_revprop_cache(fs, pool)) { - fs_x_data_t *ffd = fs->fsap_data; - pair_cache_key_t key = { 0 }; + svn_fs_x__data_t *ffd = fs->fsap_data; + svn_fs_x__pair_cache_key_t key = { 0 }; key.revision = revision; key.second = generation; @@ -668,6 +733,19 @@ read_non_packed_revprop(apr_hash_t **pro return SVN_NO_ERROR; } +/* Return the minimum length of any packed revprop file name in REVPROPS. */ +static apr_size_t +get_min_filename_len(packed_revprops_t *revprops) +{ + char number_buffer[SVN_INT64_BUFFER_SIZE]; + + /* The revprop filenames have the format <REV>.<COUNT> - with <REV> being + * at least the first rev in the shard and <COUNT> having at least one + * digit. Thus, the minimum is 2 + #decimal places in the start rev. + */ + return svn__i64toa(number_buffer, revprops->manifest_start) + 2; +} + /* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST * members. Use POOL for allocating results and SCRATCH_POOL for temporaries. */ @@ -677,49 +755,98 @@ get_revprop_packname(svn_fs_t *fs, apr_pool_t *pool, apr_pool_t *scratch_pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; svn_stringbuf_t *content = NULL; const char *manifest_file_path; - int idx; + int idx, rev_count; + char *buffer, *buffer_end; + const char **filenames, **filenames_end; + apr_size_t min_filename_len; + + /* Determine the dimensions. Rev 0 is excluded from the first shard. */ + rev_count = ffd->max_files_per_dir; + revprops->manifest_start + = revprops->revision - (revprops->revision % rev_count); + if (revprops->manifest_start == 0) + { + ++revprops->manifest_start; + --rev_count; + } + + revprops->manifest = apr_array_make(pool, rev_count, sizeof(const char*)); - /* read content of the manifest file */ + /* No line in the file can be less than this number of chars long. */ + min_filename_len = get_min_filename_len(revprops); + + /* Read the content of the manifest file */ revprops->folder = svn_fs_x__path_revprops_pack_shard(fs, revprops->revision, pool); manifest_file_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); SVN_ERR(svn_fs_x__read_content(&content, manifest_file_path, pool)); - /* parse the manifest. Every line is a file name */ - revprops->manifest = apr_array_make(pool, ffd->max_files_per_dir, - sizeof(const char*)); - - /* Read all lines. Since the last line ends with a newline, we will - end up with a valid but empty string after the last entry. */ - while (content->data && *content->data) - { - APR_ARRAY_PUSH(revprops->manifest, const char*) = content->data; - content->data = strchr(content->data, '\n'); - if (content->data) - { - *content->data = 0; - content->data++; - } + /* There CONTENT must have a certain minimal size and there no + * unterminated lines at the end of the file. Both guarantees also + * simplify the parser loop below. + */ + if ( content->len < rev_count * (min_filename_len + 1) + || content->data[content->len - 1] != '\n') + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Packed revprop manifest for r%ld not " + "properly terminated"), revprops->revision); + + /* Chop (parse) the manifest CONTENT into filenames, one per line. + * We only have to replace all newlines with NUL and add all line + * starts to REVPROPS->MANIFEST. + * + * There must be exactly REV_COUNT lines and that is the number of + * lines we parse from BUFFER to FILENAMES. Set the end pointer for + * the source BUFFER such that BUFFER+MIN_FILENAME_LEN is still valid + * BUFFER_END is always valid due to CONTENT->LEN > MIN_FILENAME_LEN. + * + * Please note that this loop is performance critical for e.g. 'svn log'. + * It is run 1000x per revprop access, i.e. per revision and about + * 50 million times per sec (and CPU core). + */ + for (filenames = (const char **)revprops->manifest->elts, + filenames_end = filenames + rev_count, + buffer = content->data, + buffer_end = buffer + content->len - min_filename_len; + (filenames < filenames_end) && (buffer < buffer_end); + ++filenames) + { + /* BUFFER always points to the start of the next line / filename. */ + *filenames = buffer; + + /* Find the next EOL. This is guaranteed to stay within the CONTENT + * buffer because we left enough room after BUFFER_END and we know + * we will always see a newline as the last non-NUL char. */ + buffer += min_filename_len; + while (*buffer != '\n') + ++buffer; + + /* Found EOL. Turn it into the filename terminator and move BUFFER + * to the start of the next line or CONTENT buffer end. */ + *buffer = '\0'; + ++buffer; } - content = NULL; /* No longer a valid stringbuf. */ - /* Index for our revision. Rev 0 is excluded from the first shard. */ - revprops->manifest_start = revprops->revision - - (revprops->revision % ffd->max_files_per_dir); - if (revprops->manifest_start == 0) - ++revprops->manifest_start; - idx = (int)(revprops->revision - revprops->manifest_start); + /* We must have reached the end of both buffers. */ + if (buffer < content->data + content->len) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + _("Packed revprop manifest for r%ld " + "has too many entries"), revprops->revision); - if (revprops->manifest->nelts <= idx) + if (filenames < filenames_end) return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Packed revprop manifest for r%ld too " - "small"), revprops->revision); + _("Packed revprop manifest for r%ld " + "has too few entries"), revprops->revision); + + /* The target array has now exactly one entry per revision. */ + revprops->manifest->nelts = rev_count; /* Now get the file name */ + idx = (int)(revprops->revision - revprops->manifest_start); revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*); return SVN_NO_ERROR; @@ -732,13 +859,14 @@ same_shard(svn_fs_t *fs, svn_revnum_t r1, svn_revnum_t r2) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir); } /* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS, - * fill the START_REVISION, SIZES, OFFSETS members. Also, make - * PACKED_REVPROPS point to the first serialized revprop. + * fill the START_REVISION member, and make PACKED_REVPROPS point to the + * first serialized revprop. If READ_ALL is set, initialize the SIZES + * and OFFSETS members as well. * * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as * well as the SERIALIZED_SIZE member. If revprop caching has been @@ -747,6 +875,7 @@ same_shard(svn_fs_t *fs, static svn_error_t * parse_packed_revprops(svn_fs_t *fs, packed_revprops_t *revprops, + svn_boolean_t read_all, apr_pool_t *pool, apr_pool_t *scratch_pool) { @@ -802,11 +931,14 @@ parse_packed_revprops(svn_fs_t *fs, revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset); revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset); - /* STREAM still points to the first entry in the sizes list. - * Init / construct REVPROPS members. */ + /* STREAM still points to the first entry in the sizes list. */ revprops->start_revision = (svn_revnum_t)first_rev; - revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset)); - revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset)); + if (read_all) + { + /* Init / construct REVPROPS members. */ + revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset)); + revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset)); + } /* Now parse, revision by revision, the size and content of each * revisions' revprops. */ @@ -814,7 +946,6 @@ parse_packed_revprops(svn_fs_t *fs, { apr_int64_t size; svn_string_t serialized; - apr_hash_t *properties; svn_revnum_t revision = (svn_revnum_t)(first_rev + i); svn_pool_clear(iterpool); @@ -835,20 +966,18 @@ parse_packed_revprops(svn_fs_t *fs, revprops->generation, &serialized, pool, iterpool)); revprops->serialized_size = serialized.len; + + /* If we only wanted the revprops for REVISION then we are done. */ + if (!read_all) + break; } - else + + if (read_all) { - /* If revprop caching is enabled, parse any revprops. - * They will get cached as a side-effect of this. */ - if (has_revprop_cache(fs, pool)) - SVN_ERR(parse_revprop(&properties, fs, revision, - revprops->generation, &serialized, - iterpool, iterpool)); + /* fill REVPROPS data structures */ + APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len; + APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset; } - - /* fill REVPROPS data structures */ - APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len; - APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset; revprops->total_size += serialized.len; offset += serialized.len; @@ -859,6 +988,8 @@ parse_packed_revprops(svn_fs_t *fs, /* In filesystem FS, read the packed revprops for revision REV into * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled. + * If you want to modify revprop contents / update REVPROPS, READ_ALL + * must be set. Otherwise, only the properties of REV are being provided. * Allocate data in POOL. */ static svn_error_t * @@ -866,6 +997,7 @@ read_pack_revprop(packed_revprops_t **re svn_fs_t *fs, svn_revnum_t rev, apr_int64_t generation, + svn_boolean_t read_all, apr_pool_t *pool) { apr_pool_t *iterpool = svn_pool_create(pool); @@ -924,7 +1056,7 @@ read_pack_revprop(packed_revprops_t **re _("Failed to read revprop pack file for r%ld"), rev); /* parse it. RESULT will be complete afterwards. */ - err = parse_packed_revprops(fs, result, pool, iterpool); + err = parse_packed_revprops(fs, result, read_all, pool, iterpool); svn_pool_destroy(iterpool); if (err) return svn_error_createf(SVN_ERR_FS_CORRUPT, err, @@ -943,9 +1075,10 @@ svn_error_t * svn_fs_x__get_revision_proplist(apr_hash_t **proplist_p, svn_fs_t *fs, svn_revnum_t rev, + svn_boolean_t bypass_cache, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; apr_int64_t generation = 0; /* not found, yet */ @@ -955,10 +1088,10 @@ svn_fs_x__get_revision_proplist(apr_hash SVN_ERR(svn_fs_x__ensure_revision_exists(rev, fs, pool)); /* Try cache lookup first. */ - if (has_revprop_cache(fs, pool)) + if (!bypass_cache && has_revprop_cache(fs, pool)) { svn_boolean_t is_cached; - pair_cache_key_t key = { 0 }; + svn_fs_x__pair_cache_key_t key = { 0 }; SVN_ERR(read_revprop_generation(&generation, fs, pool)); @@ -993,7 +1126,7 @@ svn_fs_x__get_revision_proplist(apr_hash if (!*proplist_p) { packed_revprops_t *revprops; - SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool)); + SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, FALSE, pool)); *proplist_p = revprops->properties; } @@ -1054,17 +1187,19 @@ switch_to_new_revprop(svn_fs_t *fs, svn_boolean_t bump_generation, apr_pool_t *pool) { + apr_int64_t generation; + /* Now, we may actually be replacing revprops. Make sure that all other threads and processes will know about this. */ if (bump_generation) - SVN_ERR(begin_revprop_change(fs, pool)); + SVN_ERR(begin_revprop_change(&generation, fs, pool)); SVN_ERR(svn_fs_x__move_into_place(tmp_path, final_path, perms_reference, pool)); /* Indicate that the update (if relevant) has been completed. */ if (bump_generation) - SVN_ERR(end_revprop_change(fs, pool)); + SVN_ERR(end_revprop_change(fs, generation, pool)); /* Clean up temporary files, if necessary. */ if (files_to_delete) @@ -1146,7 +1281,7 @@ repack_revprops(svn_fs_t *fs, svn_stream_t *file_stream, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; svn_stream_t *stream; int i; @@ -1274,7 +1409,7 @@ write_packed_revprop(const char **final_ apr_hash_t *proplist, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; packed_revprops_t *revprops; apr_int64_t generation = 0; svn_stream_t *stream; @@ -1288,7 +1423,7 @@ write_packed_revprop(const char **final_ SVN_ERR(read_revprop_generation(&generation, fs, pool)); /* read contents of the current pack file */ - SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool)); + SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, TRUE, pool)); /* serialize the new revprops */ serialized = svn_stringbuf_create_empty(pool); @@ -1433,20 +1568,18 @@ svn_fs_x__set_revision_proplist(svn_fs_t is_packed = svn_fs_x__is_packed_revprop(fs, rev); /* Test whether revprops already exist for this revision. - * Only then will we need to bump the revprop generation. */ - if (has_revprop_cache(fs, pool)) + * Only then will we need to bump the revprop generation. + * The fact that they did not yet exist is never cached. */ + if (is_packed) { - if (is_packed) - { - bump_generation = TRUE; - } - else - { - svn_node_kind_t kind; - SVN_ERR(svn_io_check_path(svn_fs_x__path_revprops(fs, rev, pool), - &kind, pool)); - bump_generation = kind != svn_node_none; - } + bump_generation = TRUE; + } + else + { + svn_node_kind_t kind; + SVN_ERR(svn_io_check_path(svn_fs_x__path_revprops(fs, rev, pool), + &kind, pool)); + bump_generation = kind != svn_node_none; } /* Serialize the new revprop data */ @@ -1481,7 +1614,7 @@ svn_fs_x__packed_revprop_available(svn_b svn_revnum_t revision, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; + svn_fs_x__data_t *ffd = fs->fsap_data; svn_stringbuf_t *content = NULL; /* try to read the manifest file */
