Modified: subversion/branches/move-tracking-2/subversion/libsvn_fs_x/transaction.c URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-2/subversion/libsvn_fs_x/transaction.c?rev=1641647&r1=1641646&r2=1641647&view=diff ============================================================================== --- subversion/branches/move-tracking-2/subversion/libsvn_fs_x/transaction.c (original) +++ subversion/branches/move-tracking-2/subversion/libsvn_fs_x/transaction.c Tue Nov 25 16:26:48 2014 @@ -751,7 +751,6 @@ svn_fs_x__put_node_revision(svn_fs_t *fs svn_boolean_t fresh_txn_root, apr_pool_t *pool) { - fs_x_data_t *ffd = fs->fsap_data; apr_file_t *noderev_file; noderev->is_fresh_txn_root = fresh_txn_root; @@ -768,7 +767,7 @@ svn_fs_x__put_node_revision(svn_fs_t *fs SVN_ERR(svn_fs_x__write_noderev(svn_stream_from_aprfile2(noderev_file, TRUE, pool), - noderev, ffd->format, pool)); + noderev, pool)); SVN_ERR(svn_io_file_close(noderev_file, pool)); @@ -777,12 +776,12 @@ svn_fs_x__put_node_revision(svn_fs_t *fs /* For the in-transaction NODEREV within FS, write the sha1->rep mapping * file in the respective transaction, if rep sharing has been enabled etc. - * Use POOL for temporary allocations. + * Use SCATCH_POOL for temporary allocations. */ static svn_error_t * store_sha1_rep_mapping(svn_fs_t *fs, node_revision_t *noderev, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { fs_x_data_t *ffd = fs->fsap_data; @@ -797,20 +796,21 @@ store_sha1_rep_mapping(svn_fs_t *fs, = svn_fs_x__get_txn_id(noderev->data_rep->id.change_set); const char *file_name = svn_fs_x__path_txn_sha1(fs, txn_id, - noderev->data_rep->sha1_digest, pool); + noderev->data_rep->sha1_digest, + scratch_pool); svn_stringbuf_t *rep_string = svn_fs_x__unparse_representation(noderev->data_rep, - ffd->format, (noderev->kind == svn_node_dir), - pool); + scratch_pool, scratch_pool); + SVN_ERR(svn_io_file_open(&rep_file, file_name, APR_WRITE | APR_CREATE | APR_TRUNCATE - | APR_BUFFERED, APR_OS_DEFAULT, pool)); + | APR_BUFFERED, APR_OS_DEFAULT, scratch_pool)); SVN_ERR(svn_io_file_write_full(rep_file, rep_string->data, - rep_string->len, NULL, pool)); + rep_string->len, NULL, scratch_pool)); - SVN_ERR(svn_io_file_close(rep_file, pool)); + SVN_ERR(svn_io_file_close(rep_file, scratch_pool)); } return SVN_NO_ERROR; @@ -1158,7 +1158,7 @@ create_new_txn_noderev_from_rev(svn_fs_t { node_revision_t *noderev; - SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, src, pool)); + SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, src, pool, pool)); /* This must be a root node. */ SVN_ERR_ASSERT( svn_fs_x__id_node_id(noderev->id)->number == 0 @@ -1242,12 +1242,7 @@ create_txn_dir(const char **id_p, *txn_id = cb.txn_number; *id_p = svn_fs_x__txn_name(*txn_id, pool); - txn_dir = svn_dirent_join_many(pool, - fs->path, - PATH_TXNS_DIR, - apr_pstrcat(pool, *id_p, PATH_EXT_TXN, - SVN_VA_NULL), - SVN_VA_NULL); + txn_dir = svn_fs_x__path_txn_dir(fs, *txn_id, pool); return svn_io_dir_make(txn_dir, APR_OS_DEFAULT, pool); } @@ -1276,7 +1271,7 @@ svn_fs_x__create_txn(svn_fs_txn_t **txn_ *txn_p = txn; /* Create a new root node for this transaction. */ - SVN_ERR(svn_fs_x__rev_get_root(&root_id, fs, rev, pool)); + SVN_ERR(svn_fs_x__rev_get_root(&root_id, fs, rev, pool, pool)); SVN_ERR(create_new_txn_noderev_from_rev(fs, ftd->txn_id, root_id, pool)); /* Create an empty rev file. */ @@ -1427,7 +1422,7 @@ svn_fs_x__get_txn(transaction_t **txn_p, SVN_ERR(get_txn_proplist(txn->proplist, fs, txn_id, pool)); root_id = svn_fs_x__id_txn_create_root(txn_id, pool); - SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, root_id, pool)); + SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, root_id, pool, pool)); txn->root_id = svn_fs_x__id_copy(noderev->id, pool); txn->base_id = svn_fs_x__id_copy(noderev->predecessor_id, pool); @@ -1823,8 +1818,10 @@ svn_fs_x__add_change(svn_fs_t *fs, ? svn_tristate_true : svn_tristate_false; change->node_kind = node_kind; + change->copyfrom_known = TRUE; change->copyfrom_rev = copyfrom_rev; - change->copyfrom_path = apr_pstrdup(pool, copyfrom_path); + if (copyfrom_path) + change->copyfrom_path = apr_pstrdup(pool, copyfrom_path); svn_hash_sets(changes, path, change); SVN_ERR(svn_fs_x__write_changes(svn_stream_from_aprfile2(file, TRUE, pool), @@ -1872,9 +1869,11 @@ struct rep_write_baton /* Receives the low-level checksum when closing REP_STREAM. */ apr_uint32_t fnv1a_checksum; - apr_pool_t *pool; + /* Local / scratch pool, available for temporary allocations. */ + apr_pool_t *scratch_pool; - apr_pool_t *parent_pool; + /* Outer / result pool. */ + apr_pool_t *result_pool; }; /* Handler for the write method of the representation writable stream. @@ -1905,14 +1904,18 @@ shards_spanned(int *spanned, apr_pool_t *pool) { fs_x_data_t *ffd = fs->fsap_data; - int shard_size = ffd->max_files_per_dir ? ffd->max_files_per_dir : 1; + int shard_size = ffd->max_files_per_dir; + apr_pool_t *iterpool; int count = walk ? 1 : 0; /* The start of a walk already touches a shard. */ svn_revnum_t shard, last_shard = ffd->youngest_rev_cache / shard_size; + iterpool = svn_pool_create(pool); while (walk-- && noderev->predecessor_count) { + svn_pool_clear(iterpool); SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, - noderev->predecessor_id, pool)); + noderev->predecessor_id, pool, + iterpool)); shard = svn_fs_x__id_rev(noderev->id) / shard_size; if (shard != last_shard) { @@ -1920,6 +1923,7 @@ shards_spanned(int *spanned, last_shard = shard; } } + svn_pool_destroy(iterpool); *spanned = count; return SVN_NO_ERROR; @@ -1946,6 +1950,7 @@ choose_delta_base(representation_t **rep int walk; node_revision_t *base; fs_x_data_t *ffd = fs->fsap_data; + apr_pool_t *iterpool; /* If we have no predecessors, or that one is empty, then use the empty * stream as a base. */ @@ -1992,9 +1997,15 @@ choose_delta_base(representation_t **rep if noderev has ten predecessors and we want the eighth file rev, walk back two predecessors.) */ base = noderev; + iterpool = svn_pool_create(pool); while ((count++) < noderev->predecessor_count) - SVN_ERR(svn_fs_x__get_node_revision(&base, fs, - base->predecessor_id, pool)); + { + svn_pool_clear(iterpool); + SVN_ERR(svn_fs_x__get_node_revision(&base, fs, + base->predecessor_id, pool, + iterpool)); + } + svn_pool_destroy(iterpool); /* return a suitable base representation */ *rep = props ? base->prop_rep : base->data_rep; @@ -2052,8 +2063,9 @@ rep_write_cleanup(void *data) struct rep_write_baton *b = data; /* Truncate and close the protorevfile. */ - err = svn_io_file_trunc(b->file, b->rep_offset, b->pool); - err = svn_error_compose_create(err, svn_io_file_close(b->file, b->pool)); + err = svn_io_file_trunc(b->file, b->rep_offset, b->scratch_pool); + err = svn_error_compose_create(err, svn_io_file_close(b->file, + b->scratch_pool)); /* Remove our lock regardless of any preceding errors so that the being_written flag is always removed and stays consistent with the @@ -2061,8 +2073,8 @@ rep_write_cleanup(void *data) going away. */ err = svn_error_compose_create(err, unlock_proto_rev(b->fs, - svn_fs_x__id_txn_id(b->noderev->id), - b->lockcookie, b->pool)); + svn_fs_x__id_txn_id(b->noderev->id), + b->lockcookie, b->scratch_pool)); if (err) { apr_status_t rc = err->apr_err; @@ -2099,27 +2111,29 @@ rep_write_get_baton(struct rep_write_bat b->md5_checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); b->fs = fs; - b->parent_pool = pool; - b->pool = svn_pool_create(pool); + b->result_pool = pool; + b->scratch_pool = svn_pool_create(pool); b->rep_size = 0; b->noderev = noderev; /* Open the prototype rev file and seek to its end. */ SVN_ERR(get_writable_proto_rev(&file, &b->lockcookie, fs, svn_fs_x__id_txn_id(noderev->id), - b->pool)); + b->scratch_pool)); b->file = file; b->rep_stream = svn_checksum__wrap_write_stream_fnv1a_32x4( &b->fnv1a_checksum, - svn_stream_from_aprfile2(file, TRUE, b->pool), - b->pool); + svn_stream_from_aprfile2(file, TRUE, + b->scratch_pool), + b->scratch_pool); - SVN_ERR(svn_fs_x__get_file_offset(&b->rep_offset, file, b->pool)); + SVN_ERR(svn_fs_x__get_file_offset(&b->rep_offset, file, b->scratch_pool)); /* Get the base for this delta. */ - SVN_ERR(choose_delta_base(&base_rep, fs, noderev, FALSE, b->pool)); - SVN_ERR(svn_fs_x__get_contents(&source, fs, base_rep, TRUE, b->pool)); + SVN_ERR(choose_delta_base(&base_rep, fs, noderev, FALSE, b->scratch_pool)); + SVN_ERR(svn_fs_x__get_contents(&source, fs, base_rep, TRUE, + b->scratch_pool)); /* Write out the rep header. */ if (base_rep) @@ -2133,24 +2147,27 @@ rep_write_get_baton(struct rep_write_bat { header.type = svn_fs_x__rep_self_delta; } - SVN_ERR(svn_fs_x__write_rep_header(&header, b->rep_stream, b->pool)); + SVN_ERR(svn_fs_x__write_rep_header(&header, b->rep_stream, + b->scratch_pool)); /* Now determine the offset of the actual svndiff data. */ - SVN_ERR(svn_fs_x__get_file_offset(&b->delta_start, file, b->pool)); + SVN_ERR(svn_fs_x__get_file_offset(&b->delta_start, file, + b->scratch_pool)); /* Cleanup in case something goes wrong. */ - apr_pool_cleanup_register(b->pool, b, rep_write_cleanup, + apr_pool_cleanup_register(b->scratch_pool, b, rep_write_cleanup, apr_pool_cleanup_null); /* Prepare to write the svndiff data. */ svn_txdelta_to_svndiff3(&wh, &whb, - svn_stream_disown(b->rep_stream, b->pool), + svn_stream_disown(b->rep_stream, b->result_pool), diff_version, ffd->delta_compression_level, pool); - b->delta_stream = svn_txdelta_target_push(wh, whb, source, b->pool); + b->delta_stream = svn_txdelta_target_push(wh, whb, source, + b->result_pool); *wb_p = b; @@ -2163,15 +2180,16 @@ rep_write_get_baton(struct rep_write_bat there may be new duplicate representations within the same uncommitted revision, those can be passed in REPS_HASH (maps a sha1 digest onto representation_t*), otherwise pass in NULL for REPS_HASH. - POOL will be used for allocations. The lifetime of the returned rep is - limited by both, POOL and REP lifetime. + Use RESULT_POOL for *OLD_REP allocations and SCRATCH_POOL for temporaries. + The lifetime of *OLD_REP is limited by both, RESULT_POOL and REP lifetime. */ static svn_error_t * get_shared_rep(representation_t **old_rep, svn_fs_t *fs, representation_t *rep, apr_hash_t *reps_hash, - apr_pool_t *pool) + apr_pool_t *result_pool, + apr_pool_t *scratch_pool) { svn_error_t *err; fs_x_data_t *ffd = fs->fsap_data; @@ -2195,12 +2213,12 @@ get_shared_rep(representation_t **old_re svn_checksum_t checksum; checksum.digest = rep->sha1_digest; checksum.kind = svn_checksum_sha1; - err = svn_fs_x__get_rep_reference(old_rep, fs, &checksum, pool); + err = svn_fs_x__get_rep_reference(old_rep, fs, &checksum, result_pool); /* ### Other error codes that we shouldn't mask out? */ if (err == SVN_NO_ERROR) { if (*old_rep) - SVN_ERR(svn_fs_x__check_rep(*old_rep, fs, pool)); + SVN_ERR(svn_fs_x__check_rep(*old_rep, fs, scratch_pool)); } else if (err->apr_err == SVN_ERR_FS_CORRUPT || SVN_ERROR_IN_CATEGORY(err->apr_err, @@ -2235,17 +2253,19 @@ get_shared_rep(representation_t **old_re const char *file_name = svn_fs_x__path_txn_sha1(fs, svn_fs_x__get_txn_id(rep->id.change_set), - rep->sha1_digest, pool); + rep->sha1_digest, scratch_pool); /* in our txn, is there a rep file named with the wanted SHA1? If so, read it and use that rep. */ - SVN_ERR(svn_io_check_path(file_name, &kind, pool)); + SVN_ERR(svn_io_check_path(file_name, &kind, scratch_pool)); if (kind == svn_node_file) { svn_stringbuf_t *rep_string; - SVN_ERR(svn_stringbuf_from_file2(&rep_string, file_name, pool)); - SVN_ERR(svn_fs_x__parse_representation(old_rep, rep_string, pool)); + SVN_ERR(svn_stringbuf_from_file2(&rep_string, file_name, + scratch_pool)); + SVN_ERR(svn_fs_x__parse_representation(old_rep, rep_string, + result_pool, scratch_pool)); } } @@ -2292,14 +2312,14 @@ rep_write_contents_close(void *baton) apr_off_t offset; apr_int64_t txn_id; - rep = apr_pcalloc(b->parent_pool, sizeof(*rep)); + rep = apr_pcalloc(b->result_pool, sizeof(*rep)); /* Close our delta stream so the last bits of svndiff are written out. */ SVN_ERR(svn_stream_close(b->delta_stream)); /* Determine the length of the svndiff data. */ - SVN_ERR(svn_fs_x__get_file_offset(&offset, b->file, b->pool)); + SVN_ERR(svn_fs_x__get_file_offset(&offset, b->file, b->scratch_pool)); rep->size = offset - b->delta_start; /* Fill in the rest of the representation field. */ @@ -2309,16 +2329,17 @@ rep_write_contents_close(void *baton) /* Finalize the checksum. */ SVN_ERR(digests_final(rep, b->md5_checksum_ctx, b->sha1_checksum_ctx, - b->parent_pool)); + b->result_pool)); /* Check and see if we already have a representation somewhere that's identical to the one we just wrote out. */ - SVN_ERR(get_shared_rep(&old_rep, b->fs, rep, NULL, b->parent_pool)); + SVN_ERR(get_shared_rep(&old_rep, b->fs, rep, NULL, b->result_pool, + b->scratch_pool)); if (old_rep) { /* We need to erase from the protorev the data we just wrote. */ - SVN_ERR(svn_io_file_trunc(b->file, b->rep_offset, b->pool)); + SVN_ERR(svn_io_file_trunc(b->file, b->rep_offset, b->scratch_pool)); /* Use the old rep for this content. */ b->noderev->data_rep = old_rep; @@ -2327,9 +2348,10 @@ rep_write_contents_close(void *baton) { /* Write out our cosmetic end marker. */ SVN_ERR(svn_stream_puts(b->rep_stream, "ENDREP\n")); - SVN_ERR(allocate_item_index(&rep->id.number, b->fs, txn_id, b->pool)); + SVN_ERR(allocate_item_index(&rep->id.number, b->fs, txn_id, + b->scratch_pool)); SVN_ERR(store_l2p_index_entry(b->fs, txn_id, b->rep_offset, - rep->id.number, b->pool)); + rep->id.number, b->scratch_pool)); b->noderev->data_rep = rep; } @@ -2337,11 +2359,11 @@ rep_write_contents_close(void *baton) SVN_ERR(svn_stream_close(b->rep_stream)); /* Remove cleanup callback. */ - apr_pool_cleanup_kill(b->pool, b, rep_write_cleanup); + apr_pool_cleanup_kill(b->scratch_pool, b, rep_write_cleanup); /* Write out the new node-rev information. */ SVN_ERR(svn_fs_x__put_node_revision(b->fs, b->noderev->id, b->noderev, - FALSE, b->pool)); + FALSE, b->scratch_pool)); if (!old_rep) { svn_fs_x__p2l_entry_t entry; @@ -2350,20 +2372,20 @@ rep_write_contents_close(void *baton) noderev_id.number = rep->id.number; entry.offset = b->rep_offset; - SVN_ERR(svn_fs_x__get_file_offset(&offset, b->file, b->pool)); + SVN_ERR(svn_fs_x__get_file_offset(&offset, b->file, b->scratch_pool)); entry.size = offset - b->rep_offset; entry.type = SVN_FS_X__ITEM_TYPE_FILE_REP; entry.item_count = 1; entry.items = &noderev_id; entry.fnv1_checksum = b->fnv1a_checksum; - SVN_ERR(store_sha1_rep_mapping(b->fs, b->noderev, b->pool)); - SVN_ERR(store_p2l_index_entry(b->fs, txn_id, &entry, b->pool)); + SVN_ERR(store_sha1_rep_mapping(b->fs, b->noderev, b->scratch_pool)); + SVN_ERR(store_p2l_index_entry(b->fs, txn_id, &entry, b->scratch_pool)); } - SVN_ERR(svn_io_file_close(b->file, b->pool)); - SVN_ERR(unlock_proto_rev(b->fs, txn_id, b->lockcookie, b->pool)); - svn_pool_destroy(b->pool); + SVN_ERR(svn_io_file_close(b->file, b->scratch_pool)); + SVN_ERR(unlock_proto_rev(b->fs, txn_id, b->lockcookie, b->scratch_pool)); + svn_pool_destroy(b->scratch_pool); return SVN_NO_ERROR; } @@ -2552,7 +2574,8 @@ write_directory_to_stream(svn_stream_t * If ITEM_TYPE is IS_PROPS equals SVN_FS_FS__ITEM_TYPE_*_PROPS, assume that we want to a props representation as the base for our delta. If FINAL_REVISION is not SVN_INVALID_REVNUM, use it to determine whether - to write to the proto-index files. Perform temporary allocations in POOL. + to write to the proto-index files. + Perform temporary allocations in SCRATCH_POOL. */ static svn_error_t * write_container_delta_rep(representation_t *rep, @@ -2563,9 +2586,9 @@ write_container_delta_rep(representation svn_fs_x__txn_id_t txn_id, node_revision_t *noderev, apr_hash_t *reps_hash, - int item_type, + apr_uint32_t item_type, svn_revnum_t final_revision, - apr_pool_t *pool) + apr_pool_t *scratch_pool) { fs_x_data_t *ffd = fs->fsap_data; svn_txdelta_window_handler_t diff_wh; @@ -2589,10 +2612,10 @@ write_container_delta_rep(representation || (item_type == SVN_FS_X__ITEM_TYPE_DIR_PROPS); /* Get the base for this delta. */ - SVN_ERR(choose_delta_base(&base_rep, fs, noderev, is_props, pool)); - SVN_ERR(svn_fs_x__get_contents(&source, fs, base_rep, FALSE, pool)); + SVN_ERR(choose_delta_base(&base_rep, fs, noderev, is_props, scratch_pool)); + SVN_ERR(svn_fs_x__get_contents(&source, fs, base_rep, FALSE, scratch_pool)); - SVN_ERR(svn_fs_x__get_file_offset(&offset, file, pool)); + SVN_ERR(svn_fs_x__get_file_offset(&offset, file, scratch_pool)); /* Write out the rep header. */ if (base_rep) @@ -2609,45 +2632,48 @@ write_container_delta_rep(representation file_stream = svn_checksum__wrap_write_stream_fnv1a_32x4( &entry.fnv1_checksum, - svn_stream_from_aprfile2(file, TRUE, pool), - pool); - SVN_ERR(svn_fs_x__write_rep_header(&header, file_stream, pool)); - SVN_ERR(svn_fs_x__get_file_offset(&delta_start, file, pool)); + svn_stream_from_aprfile2(file, TRUE, + scratch_pool), + scratch_pool); + SVN_ERR(svn_fs_x__write_rep_header(&header, file_stream, scratch_pool)); + SVN_ERR(svn_fs_x__get_file_offset(&delta_start, file, scratch_pool)); /* Prepare to write the svndiff data. */ svn_txdelta_to_svndiff3(&diff_wh, &diff_whb, - svn_stream_disown(file_stream, pool), + svn_stream_disown(file_stream, scratch_pool), diff_version, ffd->delta_compression_level, - pool); + scratch_pool); - whb = apr_pcalloc(pool, sizeof(*whb)); - whb->stream = svn_txdelta_target_push(diff_wh, diff_whb, source, pool); + whb = apr_pcalloc(scratch_pool, sizeof(*whb)); + whb->stream = svn_txdelta_target_push(diff_wh, diff_whb, source, + scratch_pool); whb->size = 0; - whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); - whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool); + whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool); + whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, scratch_pool); /* serialize the hash */ - stream = svn_stream_create(whb, pool); + stream = svn_stream_create(whb, scratch_pool); svn_stream_set_write(stream, write_container_handler); - SVN_ERR(writer(stream, collection, pool)); + SVN_ERR(writer(stream, collection, scratch_pool)); SVN_ERR(svn_stream_close(whb->stream)); /* Store the results. */ - SVN_ERR(digests_final(rep, whb->md5_ctx, whb->sha1_ctx, pool)); + SVN_ERR(digests_final(rep, whb->md5_ctx, whb->sha1_ctx, scratch_pool)); /* Check and see if we already have a representation somewhere that's identical to the one we just wrote out. */ - SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, pool)); + SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, scratch_pool, + scratch_pool)); if (old_rep) { SVN_ERR(svn_stream_close(file_stream)); /* We need to erase from the protorev the data we just wrote. */ - SVN_ERR(svn_io_file_trunc(file, offset, pool)); + SVN_ERR(svn_io_file_trunc(file, offset, scratch_pool)); /* Use the old rep for this content. */ memcpy(rep, old_rep, sizeof (*rep)); @@ -2657,24 +2683,26 @@ write_container_delta_rep(representation svn_fs_x__id_part_t noderev_id; /* Write out our cosmetic end marker. */ - SVN_ERR(svn_fs_x__get_file_offset(&rep_end, file, pool)); + SVN_ERR(svn_fs_x__get_file_offset(&rep_end, file, scratch_pool)); SVN_ERR(svn_stream_puts(file_stream, "ENDREP\n")); SVN_ERR(svn_stream_close(file_stream)); - SVN_ERR(allocate_item_index(&rep->id.number, fs, txn_id, pool)); - SVN_ERR(store_l2p_index_entry(fs, txn_id, offset, rep->id.number, pool)); + SVN_ERR(allocate_item_index(&rep->id.number, fs, txn_id, + scratch_pool)); + SVN_ERR(store_l2p_index_entry(fs, txn_id, offset, rep->id.number, + scratch_pool)); noderev_id.change_set = SVN_FS_X__INVALID_CHANGE_SET; noderev_id.number = rep->id.number; entry.offset = offset; - SVN_ERR(svn_fs_x__get_file_offset(&offset, file, pool)); + SVN_ERR(svn_fs_x__get_file_offset(&offset, file, scratch_pool)); entry.size = offset - entry.offset; entry.type = item_type; entry.item_count = 1; entry.items = &noderev_id; - SVN_ERR(store_p2l_index_entry(fs, txn_id, &entry, pool)); + SVN_ERR(store_p2l_index_entry(fs, txn_id, &entry, scratch_pool)); /* update the representation */ rep->expanded_size = whb->size; @@ -2711,7 +2739,7 @@ validate_root_noderev(svn_fs_t *fs, SVN_ERR(svn_fs_x__revision_root(&head_revision, fs, head_revnum, pool)); SVN_ERR(svn_fs_x__node_id(&head_root_id, head_revision, "/", pool)); SVN_ERR(svn_fs_x__get_node_revision(&head_root_noderev, fs, head_root_id, - pool)); + pool, pool)); head_predecessor_count = head_root_noderev->predecessor_count; } @@ -2804,6 +2832,7 @@ write_final_rev(const svn_fs_id_t **new_ svn_fs_x__p2l_entry_t entry; svn_fs_x__change_set_t change_set = svn_fs_x__change_set_by_rev(rev); svn_stream_t *file_stream; + apr_pool_t *subpool; *new_id_p = NULL; @@ -2811,16 +2840,15 @@ write_final_rev(const svn_fs_id_t **new_ if (! svn_fs_x__id_is_txn(id)) return SVN_NO_ERROR; - SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, id, pool)); + subpool = svn_pool_create(pool); + SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, id, pool, subpool)); if (noderev->kind == svn_node_dir) { - apr_pool_t *subpool; apr_array_header_t *entries; int i; /* This is a directory. Write out all the children first. */ - subpool = svn_pool_create(pool); SVN_ERR(svn_fs_x__rep_contents_dir(&entries, fs, noderev, pool, subpool)); @@ -2836,7 +2864,6 @@ write_final_rev(const svn_fs_id_t **new_ if (new_id && (svn_fs_x__id_rev(new_id) == rev)) dirent->id = svn_fs_x__id_copy(new_id, pool); } - svn_pool_destroy(subpool); if (noderev->data_rep && ! svn_fs_x__is_revision(noderev->data_rep->id.change_set)) @@ -2864,14 +2891,16 @@ write_final_rev(const svn_fs_id_t **new_ } } + svn_pool_destroy(subpool); + /* Fix up the property reps. */ if (noderev->prop_rep && svn_fs_x__is_txn(noderev->prop_rep->id.change_set)) { apr_hash_t *proplist; - int item_type = noderev->kind == svn_node_dir - ? SVN_FS_X__ITEM_TYPE_DIR_PROPS - : SVN_FS_X__ITEM_TYPE_FILE_PROPS; + apr_uint32_t item_type = noderev->kind == svn_node_dir + ? SVN_FS_X__ITEM_TYPE_DIR_PROPS + : SVN_FS_X__ITEM_TYPE_FILE_PROPS; SVN_ERR(svn_fs_x__get_proplist(&proplist, fs, noderev, pool)); noderev->prop_rep->id.change_set = change_set; @@ -2947,7 +2976,7 @@ write_final_rev(const svn_fs_id_t **new_ &entry.fnv1_checksum, svn_stream_from_aprfile2(file, TRUE, pool), pool); - SVN_ERR(svn_fs_x__write_noderev(file_stream, noderev, ffd->format, pool)); + SVN_ERR(svn_fs_x__write_noderev(file_stream, noderev, pool)); SVN_ERR(svn_stream_close(file_stream)); /* reference the root noderev from the log-to-phys index */ @@ -3195,6 +3224,47 @@ write_final_revprop(const char **path, return SVN_NO_ERROR; } +svn_error_t * +svn_fs_x__add_index_data(svn_fs_t *fs, + apr_file_t *file, + const char *l2p_proto_index, + const char *p2l_proto_index, + svn_revnum_t revision, + apr_pool_t *pool) +{ + apr_off_t l2p_offset; + apr_off_t p2l_offset; + svn_stringbuf_t *footer; + unsigned char footer_length; + svn_checksum_t *l2p_checksum; + svn_checksum_t *p2l_checksum; + + /* Append the actual index data to the pack file. */ + l2p_offset = 0; + SVN_ERR(svn_io_file_seek(file, APR_END, &l2p_offset, pool)); + SVN_ERR(svn_fs_x__l2p_index_append(&l2p_checksum, fs, file, + l2p_proto_index, revision, + pool, pool)); + + p2l_offset = 0; + SVN_ERR(svn_io_file_seek(file, APR_END, &p2l_offset, pool)); + SVN_ERR(svn_fs_x__p2l_index_append(&p2l_checksum, fs, file, + p2l_proto_index, revision, + pool, pool)); + + /* Append footer. */ + footer = svn_fs_x__unparse_footer(l2p_offset, l2p_checksum, + p2l_offset, p2l_checksum, pool, pool); + SVN_ERR(svn_io_file_write_full(file, footer->data, footer->len, NULL, + pool)); + + footer_length = footer->len; + SVN_ERR_ASSERT(footer_length == footer->len); + SVN_ERR(svn_io_file_write_full(file, &footer_length, 1, NULL, pool)); + + return SVN_NO_ERROR; +} + /* Baton used for commit_body below. */ struct commit_baton { svn_revnum_t *new_rev_p; @@ -3223,6 +3293,22 @@ commit_body(void *baton, apr_pool_t *poo svn_fs_x__txn_id_t txn_id = svn_fs_x__txn_get_id(cb->txn); apr_hash_t *changed_paths; + /* Re-Read the current repository format. All our repo upgrade and + config evaluation strategies are such that existing information in + FS and FFD remains valid. + + Although we don't recommend upgrading hot repositories, people may + still do it and we must make sure to either handle them gracefully + or to error out. + + Committing pre-format 3 txns will fail after upgrade to format 3+ + because the proto-rev cannot be found; no further action needed. + Upgrades from pre-f7 to f7+ means a potential change in addressing + mode for the final rev. We must be sure to detect that cause because + the failure would only manifest once the new revision got committed. + */ + SVN_ERR(svn_fs_x__read_format_file(cb->fs, pool)); + /* Get the current youngest revision. */ SVN_ERR(svn_fs_x__youngest_rev(&old_rev, cb->fs, pool)); @@ -3262,6 +3348,12 @@ commit_body(void *baton, apr_pool_t *poo cb->fs, txn_id, changed_paths, new_rev, pool)); + /* Append the index data to the rev file. */ + SVN_ERR(svn_fs_x__add_index_data(cb->fs, proto_file, + svn_fs_x__path_l2p_proto_index(cb->fs, txn_id, pool), + svn_fs_x__path_p2l_proto_index(cb->fs, txn_id, pool), + new_rev, pool)); + SVN_ERR(svn_io_file_flush_to_disk(proto_file, pool)); SVN_ERR(svn_io_file_close(proto_file, pool)); @@ -3304,17 +3396,6 @@ commit_body(void *baton, apr_pool_t *poo } } - /* Convert the index files from the proto format into their form - in their final location */ - SVN_ERR(svn_fs_x__l2p_index_create(cb->fs, - svn_fs_x__path_l2p_index(cb->fs, new_rev, pool), - svn_fs_x__path_l2p_proto_index(cb->fs, txn_id, pool), - new_rev, pool)); - SVN_ERR(svn_fs_x__p2l_index_create(cb->fs, - svn_fs_x__path_p2l_index(cb->fs, new_rev, pool), - svn_fs_x__path_p2l_proto_index(cb->fs, txn_id, pool), - new_rev, pool)); - /* Move the finished rev file into place. ### This "breaks" the transaction by removing the protorev file @@ -3446,7 +3527,7 @@ svn_fs_x__list_transactions(apr_array_he names = apr_array_make(pool, 1, sizeof(const char *)); /* Get the transactions directory. */ - txn_dir = svn_dirent_join(fs->path, PATH_TXNS_DIR, pool); + txn_dir = svn_fs_x__path_txns_dir(fs, pool); /* Now find a listing of this directory. */ SVN_ERR(svn_io_get_dirents3(&dirents, txn_dir, TRUE, pool, pool)); @@ -3536,7 +3617,7 @@ svn_fs_x__delete_node_revision(svn_fs_t { node_revision_t *noderev; - SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, id, pool)); + SVN_ERR(svn_fs_x__get_node_revision(&noderev, fs, id, pool, pool)); /* Delete any mutable property representation. */ if (noderev->prop_rep
Modified: subversion/branches/move-tracking-2/subversion/libsvn_fs_x/transaction.h URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-2/subversion/libsvn_fs_x/transaction.h?rev=1641647&r1=1641646&r2=1641647&view=diff ============================================================================== --- subversion/branches/move-tracking-2/subversion/libsvn_fs_x/transaction.h (original) +++ subversion/branches/move-tracking-2/subversion/libsvn_fs_x/transaction.h Tue Nov 25 16:26:48 2014 @@ -240,6 +240,18 @@ svn_fs_x__set_proplist(svn_fs_t *fs, apr_hash_t *proplist, apr_pool_t *pool); +/* Append the L2P and P2L indexes given by their proto index file names + * L2P_PROTO_INDEX and P2L_PROTO_INDEX to the revision / pack FILE. + * The latter contains revision(s) starting at REVISION in FS. + * Use POOL for temporary allocations. */ +svn_error_t * +svn_fs_x__add_index_data(svn_fs_t *fs, + apr_file_t *file, + const char *l2p_proto_index, + const char *p2l_proto_index, + svn_revnum_t revision, + apr_pool_t *pool); + /* Commit the transaction TXN in filesystem FS and return its new revision number in *REV. If the transaction is out of date, return the error SVN_ERR_FS_TXN_OUT_OF_DATE. Use POOL for temporary Modified: subversion/branches/move-tracking-2/subversion/libsvn_fs_x/tree.c URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-2/subversion/libsvn_fs_x/tree.c?rev=1641647&r1=1641646&r2=1641647&view=diff ============================================================================== --- subversion/branches/move-tracking-2/subversion/libsvn_fs_x/tree.c (original) +++ subversion/branches/move-tracking-2/subversion/libsvn_fs_x/tree.c Tue Nov 25 16:26:48 2014 @@ -1833,11 +1833,19 @@ merge(svn_stringbuf_t *conflict_p, { node_revision_t *tgt_nr, *anc_nr, *src_nr; svn_boolean_t same; + apr_pool_t *scratch_pool; /* Get node revisions for our id's. */ - SVN_ERR(svn_fs_x__get_node_revision(&tgt_nr, fs, target_id, pool)); - SVN_ERR(svn_fs_x__get_node_revision(&anc_nr, fs, ancestor_id, pool)); - SVN_ERR(svn_fs_x__get_node_revision(&src_nr, fs, source_id, pool)); + scratch_pool = svn_pool_create(pool); + SVN_ERR(svn_fs_x__get_node_revision(&tgt_nr, fs, target_id, pool, + scratch_pool)); + svn_pool_clear(scratch_pool); + SVN_ERR(svn_fs_x__get_node_revision(&anc_nr, fs, ancestor_id, pool, + scratch_pool)); + svn_pool_clear(scratch_pool); + SVN_ERR(svn_fs_x__get_node_revision(&src_nr, fs, source_id, pool, + scratch_pool)); + svn_pool_destroy(scratch_pool); /* Now compare the prop-keys of the skels. Note that just because the keys are different -doesn't- mean the proplists have @@ -4095,10 +4103,12 @@ stringify_node(dag_node_t *node, /* Check metadata sanity on NODE, and on its children. Manually verify information for DAG nodes in revision REV, and trust the metadata - accuracy for nodes belonging to older revisions. */ + accuracy for nodes belonging to older revisions. To detect cycles, + provide all parent dag_node_t * in PARENT_NODES. */ static svn_error_t * verify_node(dag_node_t *node, svn_revnum_t rev, + apr_array_header_t *parent_nodes, apr_pool_t *pool) { svn_boolean_t has_mergeinfo; @@ -4108,6 +4118,18 @@ verify_node(dag_node_t *node, int pred_count; svn_node_kind_t kind; apr_pool_t *iterpool = svn_pool_create(pool); + int i; + + /* Detect (non-)DAG cycles. */ + for (i = 0; i < parent_nodes->nelts; ++i) + { + dag_node_t *parent = APR_ARRAY_IDX(parent_nodes, i, dag_node_t *); + if (svn_fs_x__id_eq(svn_fs_x__dag_get_id(parent), + svn_fs_x__dag_get_id(node))) + return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, + "Node is its own direct or indirect parent '%s'", + stringify_node(node, iterpool)); + } /* Fetch some data. */ SVN_ERR(svn_fs_x__dag_has_mergeinfo(&has_mergeinfo, node)); @@ -4159,8 +4181,8 @@ verify_node(dag_node_t *node, if (kind == svn_node_dir) { apr_array_header_t *entries; - int i; apr_int64_t children_mergeinfo = 0; + APR_ARRAY_PUSH(parent_nodes, dag_node_t*) = node; SVN_ERR(svn_fs_x__dag_dir_entries(&entries, node, pool)); @@ -4179,7 +4201,7 @@ verify_node(dag_node_t *node, { SVN_ERR(svn_fs_x__dag_get_node(&child, fs, dirent->id, iterpool)); - SVN_ERR(verify_node(child, rev, iterpool)); + SVN_ERR(verify_node(child, rev, parent_nodes, iterpool)); SVN_ERR(svn_fs_x__dag_get_mergeinfo_count(&child_mergeinfo, child)); } @@ -4201,6 +4223,10 @@ verify_node(dag_node_t *node, stringify_node(node, iterpool), mergeinfo_count, has_mergeinfo, children_mergeinfo); + + /* If we don't make it here, there was an error / corruption. + * In that case, nobody will need PARENT_NODES anymore. */ + apr_array_pop(parent_nodes); } svn_pool_destroy(iterpool); @@ -4213,6 +4239,7 @@ svn_fs_x__verify_root(svn_fs_root_t *roo { svn_fs_t *fs = root->fs; dag_node_t *root_dir; + apr_array_header_t *parent_nodes; /* Issue #4129: bogus pred-counts and minfo-cnt's on the root node-rev (and elsewhere). This code makes more thorough checks than the @@ -4236,7 +4263,8 @@ svn_fs_x__verify_root(svn_fs_root_t *roo } /* Recursively verify ROOT_DIR. */ - SVN_ERR(verify_node(root_dir, root->rev, pool)); + parent_nodes = apr_array_make(pool, 16, sizeof(dag_node_t *)); + SVN_ERR(verify_node(root_dir, root->rev, parent_nodes, pool)); /* Verify explicitly the predecessor of the root. */ { Modified: subversion/branches/move-tracking-2/subversion/libsvn_fs_x/util.c URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-2/subversion/libsvn_fs_x/util.c?rev=1641647&r1=1641646&r2=1641647&view=diff ============================================================================== --- subversion/branches/move-tracking-2/subversion/libsvn_fs_x/util.c (original) +++ subversion/branches/move-tracking-2/subversion/libsvn_fs_x/util.c Tue Nov 25 16:26:48 2014 @@ -172,24 +172,6 @@ svn_fs_x__path_rev(svn_fs_t *fs, svn_rev } const char * -svn_fs_x__path_l2p_index(svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *pool) -{ - return apr_psprintf(pool, "%s" PATH_EXT_L2P_INDEX, - svn_fs_x__path_rev_absolute(fs, rev, pool)); -} - -const char * -svn_fs_x__path_p2l_index(svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *pool) -{ - return apr_psprintf(pool, "%s" PATH_EXT_P2L_INDEX, - svn_fs_x__path_rev_absolute(fs, rev, pool)); -} - -const char * svn_fs_x__path_rev_absolute(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) @@ -270,11 +252,18 @@ combine_txn_id_string(svn_fs_x__txn_id_t } const char * +svn_fs_x__path_txns_dir(svn_fs_t *fs, + apr_pool_t *pool) +{ + return svn_dirent_join(fs->path, PATH_TXNS_DIR, pool); +} + +const char * svn_fs_x__path_txn_dir(svn_fs_t *fs, svn_fs_x__txn_id_t txn_id, apr_pool_t *pool) { - return svn_dirent_join_many(pool, fs->path, PATH_TXNS_DIR, + return svn_dirent_join_many(pool, svn_fs_x__path_txns_dir(fs, pool), combine_txn_id_string(txn_id, PATH_EXT_TXN, pool), SVN_VA_NULL); @@ -359,6 +348,13 @@ svn_fs_x__path_min_unpacked_rev(svn_fs_t } const char * +svn_fs_x__path_txn_proto_revs(svn_fs_t *fs, + apr_pool_t *pool) +{ + return svn_dirent_join(fs->path, PATH_TXN_PROTOS_DIR, pool); +} + +const char * svn_fs_x__path_txn_item_index(svn_fs_t *fs, svn_fs_x__txn_id_t txn_id, apr_pool_t *pool) @@ -372,7 +368,7 @@ svn_fs_x__path_txn_proto_rev(svn_fs_t *f svn_fs_x__txn_id_t txn_id, apr_pool_t *pool) { - return svn_dirent_join_many(pool, fs->path, PATH_TXN_PROTOS_DIR, + return svn_dirent_join_many(pool, svn_fs_x__path_txn_proto_revs(fs, pool), combine_txn_id_string(txn_id, PATH_EXT_REV, pool), SVN_VA_NULL); @@ -383,7 +379,7 @@ svn_fs_x__path_txn_proto_rev_lock(svn_fs svn_fs_x__txn_id_t txn_id, apr_pool_t *pool) { - return svn_dirent_join_many(pool, fs->path, PATH_TXN_PROTOS_DIR, + return svn_dirent_join_many(pool, svn_fs_x__path_txn_proto_revs(fs, pool), combine_txn_id_string(txn_id, PATH_EXT_REV_LOCK, pool), @@ -462,7 +458,7 @@ svn_fs_x__read_min_unpacked_rev(svn_revn SVN_ERR(svn_io_read_length_line(file, buf, &len, pool)); SVN_ERR(svn_io_file_close(file, pool)); - *min_unpacked_rev = SVN_STR_TO_REV(buf); + SVN_ERR(svn_revnum_parse(min_unpacked_rev, buf, NULL)); return SVN_NO_ERROR; } @@ -501,11 +497,15 @@ svn_fs_x__read_current(svn_revnum_t *rev svn_fs_t *fs, apr_pool_t *pool) { + const char *str; svn_stringbuf_t *content; SVN_ERR(svn_fs_x__read_content(&content, svn_fs_x__path_current(fs, pool), pool)); - SVN_ERR(svn_revnum_parse(rev, content->data, NULL)); + SVN_ERR(svn_revnum_parse(rev, content->data, &str)); + if (*str != '\n') + return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, + _("Corrupt 'current' file")); return SVN_NO_ERROR; } Modified: subversion/branches/move-tracking-2/subversion/libsvn_fs_x/util.h URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-2/subversion/libsvn_fs_x/util.h?rev=1641647&r1=1641646&r2=1641647&view=diff ============================================================================== --- subversion/branches/move-tracking-2/subversion/libsvn_fs_x/util.h (original) +++ subversion/branches/move-tracking-2/subversion/libsvn_fs_x/util.h Tue Nov 25 16:26:48 2014 @@ -78,9 +78,11 @@ svn_boolean_t svn_fs_x__is_packed_revprop(svn_fs_t *fs, svn_revnum_t rev); -/* Return the revision number of the pack / rev file in FS containing REV. */ +/* Return the first revision in the pack / rev file containing REVISION in + * filesystem FS. For non-packed revs, this will simply be REVISION. */ svn_revnum_t -svn_fs_x__packed_base_rev(svn_fs_t *fs, svn_revnum_t rev); +svn_fs_x__packed_base_rev(svn_fs_t *fs, + svn_revnum_t rev); /* Return the number of revisions in the pack / rev file in FS that contains * revision REV. */ @@ -135,16 +137,6 @@ svn_fs_x__path_rev(svn_fs_t *fs, apr_pool_t *pool); const char * -svn_fs_x__path_l2p_index(svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *pool); - -const char * -svn_fs_x__path_p2l_index(svn_fs_t *fs, - svn_revnum_t rev, - apr_pool_t *pool); - -const char * svn_fs_x__path_revprops_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool); @@ -167,11 +159,21 @@ svn_error_t * svn_fs_x__txn_by_name(svn_fs_x__txn_id_t *txn_id, const char *txn_name); +/* Return the path of the directory containing the transaction TXN_ID in FS. + * The result will be allocated in POOL. + */ const char * svn_fs_x__path_txn_dir(svn_fs_t *fs, svn_fs_x__txn_id_t txn_id, apr_pool_t *pool); +/* Return the path of the 'transactions' directory in FS. + * The result will be allocated in POOL. + */ +const char * +svn_fs_x__path_txns_dir(svn_fs_t *fs, + apr_pool_t *pool); + /* Return the name of the sha1->rep mapping file in transaction TXN_ID * within FS for the given SHA1 checksum. Use POOL for allocations. */ @@ -181,6 +183,16 @@ svn_fs_x__path_txn_sha1(svn_fs_t *fs, const unsigned char *sha1, apr_pool_t *pool); +/* Return the path of the 'txn-protorevs' directory in FS, even if that + * folder may not exist in FS. The result will be allocated in POOL. + */ +const char * +svn_fs_x__path_txn_proto_revs(svn_fs_t *fs, + apr_pool_t *pool); + +/* Return the path of the proto-revision file for transaction TXN_ID in FS. + * The result will be allocated in POOL. + */ const char * svn_fs_x__path_txn_changes(svn_fs_t *fs, svn_fs_x__txn_id_t txn_id, Modified: subversion/branches/move-tracking-2/subversion/libsvn_fs_x/verify.c URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-2/subversion/libsvn_fs_x/verify.c?rev=1641647&r1=1641646&r2=1641647&view=diff ============================================================================== --- subversion/branches/move-tracking-2/subversion/libsvn_fs_x/verify.c (original) +++ subversion/branches/move-tracking-2/subversion/libsvn_fs_x/verify.c Tue Nov 25 16:26:48 2014 @@ -133,6 +133,93 @@ verify_rep_cache(svn_fs_t *fs, return SVN_NO_ERROR; } +/* Verify that the MD5 checksum of the data between offsets START and END + * in FILE matches the EXPECTED checksum. If there is a mismatch use the + * indedx NAME in the error message. Supports cancellation with CANCEL_FUNC + * and CANCEL_BATON. SCRATCH_POOL is for temporary allocations. */ +static svn_error_t * +verify_index_checksum(apr_file_t *file, + const char *name, + apr_off_t start, + apr_off_t end, + svn_checksum_t *expected, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + unsigned char buffer[SVN__STREAM_CHUNK_SIZE]; + apr_off_t size = end - start; + svn_checksum_t *actual; + svn_checksum_ctx_t *context + = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool); + + /* Calculate the index checksum. */ + SVN_ERR(svn_io_file_seek(file, APR_SET, &start, scratch_pool)); + while (size > 0) + { + apr_size_t to_read = size > sizeof(buffer) + ? sizeof(buffer) + : (apr_size_t)size; + SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL, + scratch_pool)); + SVN_ERR(svn_checksum_update(context, buffer, to_read)); + size -= to_read; + + if (cancel_func) + SVN_ERR(cancel_func(cancel_baton)); + } + + SVN_ERR(svn_checksum_final(&actual, context, scratch_pool)); + + /* Verify that it matches the expected checksum. */ + if (!svn_checksum_match(expected, actual)) + { + const char *file_name; + + SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); + SVN_ERR(svn_checksum_mismatch_err(expected, actual, scratch_pool, + _("%s checksum mismatch in file %s"), + name, file_name)); + } + + return SVN_NO_ERROR; +} + +/* Verify the MD5 checksums of the index data in the rev / pack file + * containing revision START in FS. If given, invoke CANCEL_FUNC with + * CANCEL_BATON at regular intervals. Use SCRATCH_POOL for temporary + * allocations. + */ +static svn_error_t * +verify_index_checksums(svn_fs_t *fs, + svn_revnum_t start, + svn_cancel_func_t cancel_func, + void *cancel_baton, + apr_pool_t *scratch_pool) +{ + svn_fs_x__revision_file_t *rev_file; + + /* Open the rev / pack file and read the footer */ + SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, + scratch_pool, scratch_pool)); + SVN_ERR(svn_fs_x__auto_read_footer(rev_file)); + + /* Verify the index contents against the checksum from the footer. */ + SVN_ERR(verify_index_checksum(rev_file->file, "L2P index", + rev_file->l2p_offset, rev_file->p2l_offset, + rev_file->l2p_checksum, + cancel_func, cancel_baton, scratch_pool)); + SVN_ERR(verify_index_checksum(rev_file->file, "P2L index", + rev_file->p2l_offset, rev_file->footer_offset, + rev_file->p2l_checksum, + cancel_func, cancel_baton, scratch_pool)); + + /* Done. */ + SVN_ERR(svn_fs_x__close_revision_file(rev_file)); + + return SVN_NO_ERROR; +} + /* Verify that for all log-to-phys index entries for revisions START to * START + COUNT-1 in FS there is a consistent entry in the phys-to-log * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular @@ -150,8 +237,14 @@ compare_l2p_to_p2l_index(svn_fs_t *fs, apr_pool_t *iterpool = svn_pool_create(pool); apr_array_header_t *max_ids; + /* common file access structure */ + svn_fs_x__revision_file_t *rev_file; + SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, pool, + iterpool)); + /* determine the range of items to check for each revision */ - SVN_ERR(svn_fs_x__l2p_get_max_ids(&max_ids, fs, start, count, pool)); + SVN_ERR(svn_fs_x__l2p_get_max_ids(&max_ids, fs, start, count, pool, + iterpool)); /* check all items in all revisions if the given range */ for (i = 0; i < max_ids->nelts; ++i) @@ -171,14 +264,15 @@ compare_l2p_to_p2l_index(svn_fs_t *fs, l2p_item.number = k; /* get L2P entry. Ignore unused entries. */ - SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, &l2p_item, - iterpool)); + SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, rev_file, + &l2p_item, iterpool)); if (offset == -1) continue; /* find the corresponding P2L entry */ - SVN_ERR(svn_fs_x__p2l_item_lookup(&p2l_item, fs, start, - offset, sub_item, iterpool)); + SVN_ERR(svn_fs_x__p2l_item_lookup(&p2l_item, fs, rev_file, + revision, offset, sub_item, + iterpool, iterpool)); if (p2l_item == NULL) return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, @@ -208,6 +302,8 @@ compare_l2p_to_p2l_index(svn_fs_t *fs, svn_pool_destroy(iterpool); + SVN_ERR(svn_fs_x__close_revision_file(rev_file)); + return SVN_NO_ERROR; } @@ -232,8 +328,14 @@ compare_p2l_to_l2p_index(svn_fs_t *fs, apr_off_t max_offset; apr_off_t offset = 0; + /* common file access structure */ + svn_fs_x__revision_file_t *rev_file; + SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, pool, + iterpool)); + /* get the size of the rev / pack file as covered by the P2L index */ - SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, start, pool)); + SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, rev_file, start, + pool)); /* for all offsets in the file, get the P2L index entries and check them against the L2P index */ @@ -244,8 +346,9 @@ compare_p2l_to_l2p_index(svn_fs_t *fs, int i; /* get all entries for the current block */ - SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, start, offset, - ffd->p2l_page_size, iterpool)); + SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, rev_file, start, + offset, ffd->p2l_page_size, + iterpool, iterpool)); if (entries->nelts == 0) return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, NULL, @@ -257,7 +360,7 @@ compare_p2l_to_l2p_index(svn_fs_t *fs, last_entry = &APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_x__p2l_entry_t); offset = last_entry->offset + last_entry->size; - + for (i = 0; i < entries->nelts; ++i) { apr_uint32_t k; @@ -274,7 +377,7 @@ compare_p2l_to_l2p_index(svn_fs_t *fs, = svn_fs_x__get_revnum(p2l_item->change_set); SVN_ERR(svn_fs_x__item_offset(&l2p_offset, &sub_item, fs, - p2l_item, iterpool)); + rev_file, p2l_item, iterpool)); if (sub_item != k || l2p_offset != entry->offset) return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, @@ -297,6 +400,8 @@ compare_p2l_to_l2p_index(svn_fs_t *fs, svn_pool_destroy(iterpool); + SVN_ERR(svn_fs_x__close_revision_file(rev_file)); + return SVN_NO_ERROR; } @@ -385,7 +490,7 @@ expected_checksum(apr_file_t *file, SVN_ERR(svn_io_file_name_get(&file_name, file, pool)); SVN_ERR(svn_io_file_name_get(&file_name, file, pool)); return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, - _("Checksum mismatch item at offset %s of " + _("Checksum mismatch in item at offset %s of " "length %s bytes in file %s"), apr_off_t_toa(pool, entry->offset), apr_off_t_toa(pool, entry->size), file_name); @@ -468,23 +573,26 @@ compare_p2l_to_rev(svn_fs_t *fs, apr_pool_t *iterpool = svn_pool_create(pool); apr_off_t max_offset; apr_off_t offset = 0; - apr_file_t *rev_file; + svn_fs_x__revision_file_t *rev_file; /* open the pack / rev file that is covered by the p2l index */ - SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, pool)); + SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, pool, + iterpool)); /* check file size vs. range covered by index */ - SVN_ERR(svn_io_file_seek(rev_file, APR_END, &offset, pool)); - SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, start, pool)); + SVN_ERR(svn_fs_x__auto_read_footer(rev_file)); + SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, rev_file, start, + pool)); - if (offset != max_offset) + if (rev_file->l2p_offset != max_offset) return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, _("File size of %s for revision r%ld does " "not match p2l index size of %s"), - apr_off_t_toa(pool, offset), start, + apr_off_t_toa(pool, rev_file->l2p_offset), start, apr_off_t_toa(pool, max_offset)); - SVN_ERR(svn_io_file_aligned_seek(rev_file, ffd->block_size, NULL, 0, pool)); + SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, NULL, 0, + pool)); /* for all offsets in the file, get the P2L index entries and check them against the L2P index */ @@ -496,8 +604,14 @@ compare_p2l_to_rev(svn_fs_t *fs, svn_pool_clear(iterpool); /* get all entries for the current block */ - SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, start, offset, - ffd->p2l_page_size, iterpool)); + SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, rev_file, start, + offset, ffd->p2l_page_size, + iterpool, iterpool)); + + /* The above might have moved the file pointer. + * Ensure we actually start reading at OFFSET. */ + SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, + NULL, offset, iterpool)); /* process all entries (and later continue with the next block) */ for (i = 0; i < entries->nelts; ++i) @@ -529,14 +643,16 @@ compare_p2l_to_rev(svn_fs_t *fs, { /* skip filler entry at the end of the p2l index */ if (entry->offset != max_offset) - SVN_ERR(read_all_nul(rev_file, entry->size, pool)); + SVN_ERR(read_all_nul(rev_file->file, entry->size, pool)); } - else if (entry->fnv1_checksum) + else { if (entry->size < STREAM_THRESHOLD) - SVN_ERR(expected_buffered_checksum(rev_file, entry, pool)); + SVN_ERR(expected_buffered_checksum(rev_file->file, entry, + pool)); else - SVN_ERR(expected_streamed_checksum(rev_file, entry, pool)); + SVN_ERR(expected_streamed_checksum(rev_file->file, entry, + pool)); } /* advance offset */ @@ -569,29 +685,59 @@ verify_index_consistency(svn_fs_t *fs, void *cancel_baton, apr_pool_t *pool) { + svn_error_t *err; fs_x_data_t *ffd = fs->fsap_data; - svn_revnum_t revision, pack_start, pack_end; + svn_revnum_t revision, next_revision; apr_pool_t *iterpool = svn_pool_create(pool); - for (revision = start; revision <= end; revision = pack_end) + for (revision = start; revision <= end; revision = next_revision) { - pack_start = svn_fs_x__packed_base_rev(fs, revision); - pack_end = pack_start + svn_fs_x__pack_size(fs, revision); + svn_revnum_t count = svn_fs_x__packed_base_rev(fs, revision); + svn_revnum_t pack_start = count; + svn_revnum_t pack_end = pack_start + svn_fs_x__pack_size(fs, revision); + + svn_pool_clear(iterpool); if (notify_func && (pack_start % ffd->max_files_per_dir == 0)) notify_func(pack_start, notify_baton, iterpool); + /* Check for external corruption to the indexes. */ + err = verify_index_checksums(fs, pack_start, cancel_func, + cancel_baton, iterpool); + /* two-way index check */ - SVN_ERR(compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start, - cancel_func, cancel_baton, iterpool)); - SVN_ERR(compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start, - cancel_func, cancel_baton, iterpool)); + if (!err) + err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start, + cancel_func, cancel_baton, iterpool); + if (!err) + err = compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start, + cancel_func, cancel_baton, iterpool); /* verify in-index checksums and types vs. actual rev / pack files */ - SVN_ERR(compare_p2l_to_rev(fs, pack_start, pack_end - pack_start, - cancel_func, cancel_baton, iterpool)); + if (!err) + err = compare_p2l_to_rev(fs, pack_start, pack_end - pack_start, + cancel_func, cancel_baton, iterpool); + + /* concurrent packing is one of the reasons why verification may fail. + Make sure, we operate on up-to-date information. */ + if (err) + SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, + fs, pool)); - svn_pool_clear(iterpool); + /* retry the whole shard if it got packed in the meantime */ + if (err && count != svn_fs_x__pack_size(fs, revision)) + { + svn_error_clear(err); + + /* We could simply assign revision here but the code below is + more intuitive to maintainers. */ + next_revision = svn_fs_x__packed_base_rev(fs, revision); + } + else + { + SVN_ERR(err); + next_revision = pack_end; + } } svn_pool_destroy(iterpool); Modified: subversion/branches/move-tracking-2/subversion/libsvn_subr/stream.c URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-2/subversion/libsvn_subr/stream.c?rev=1641647&r1=1641646&r2=1641647&view=diff ============================================================================== --- subversion/branches/move-tracking-2/subversion/libsvn_subr/stream.c (original) +++ subversion/branches/move-tracking-2/subversion/libsvn_subr/stream.c Tue Nov 25 16:26:48 2014 @@ -78,17 +78,8 @@ svn_stream_create(void *baton, apr_pool_ { svn_stream_t *stream; - stream = apr_palloc(pool, sizeof(*stream)); + stream = apr_pcalloc(pool, sizeof(*stream)); stream->baton = baton; - stream->read_fn = NULL; - stream->skip_fn = NULL; - stream->write_fn = NULL; - stream->close_fn = NULL; - stream->mark_fn = NULL; - stream->seek_fn = NULL; - stream->data_available_fn = NULL; - stream->is_buffered_fn = NULL; - stream->file = NULL; return stream; } @@ -1126,7 +1117,7 @@ read_helper_gz(svn_stream_t *substream, uInt, but Subversion's API requires apr_size_t. */ apr_size_t apr_len = (apr_size_t) *len; - SVN_ERR(svn_stream_read2(substream, buffer, &apr_len)); + SVN_ERR(svn_stream_read_full(substream, buffer, &apr_len)); /* Type cast back to uInt type that zlib uses. On LP64 platforms apr_size_t will be bigger than uInt. */ Modified: subversion/branches/move-tracking-2/subversion/libsvn_wc/node.c URL: http://svn.apache.org/viewvc/subversion/branches/move-tracking-2/subversion/libsvn_wc/node.c?rev=1641647&r1=1641646&r2=1641647&view=diff ============================================================================== --- subversion/branches/move-tracking-2/subversion/libsvn_wc/node.c (original) +++ subversion/branches/move-tracking-2/subversion/libsvn_wc/node.c Tue Nov 25 16:26:48 2014 @@ -636,7 +636,6 @@ svn_wc__node_get_base(svn_node_kind_t *k svn_wc_context_t *wc_ctx, const char *local_abspath, svn_boolean_t ignore_enoent, - svn_boolean_t show_hidden, apr_pool_t *result_pool, apr_pool_t *scratch_pool) { @@ -656,9 +655,8 @@ svn_wc__node_get_base(svn_node_kind_t *k if (err && err->apr_err != SVN_ERR_WC_PATH_NOT_FOUND) return svn_error_trace(err); else if (err - || (!err && !show_hidden - && (status != svn_wc__db_status_normal - && status != svn_wc__db_status_incomplete))) + || (status != svn_wc__db_status_normal + && status != svn_wc__db_status_incomplete)) { if (!ignore_enoent) {
