Author: fdmanana Date: Wed Dec 8 12:17:14 2010 New Revision: 1043381 URL: http://svn.apache.org/viewvc?rev=1043381&view=rev Log: Removed all the code that upgrades database files from version 0.9. Also reorganized the export declarations in couch_file and couch_stream to be more readable and not exceed 80 characters per line.
Modified: couchdb/trunk/src/couchdb/couch_db.erl couchdb/trunk/src/couchdb/couch_db_updater.erl couchdb/trunk/src/couchdb/couch_doc.erl couchdb/trunk/src/couchdb/couch_file.erl couchdb/trunk/src/couchdb/couch_stream.erl couchdb/trunk/src/couchdb/couch_view_group.erl Modified: couchdb/trunk/src/couchdb/couch_db.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.erl?rev=1043381&r1=1043380&r2=1043381&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_db.erl (original) +++ couchdb/trunk/src/couchdb/couch_db.erl Wed Dec 8 12:17:14 2010 @@ -1138,9 +1138,6 @@ doc_meta_info(#doc_info{high_seq=Seq,rev true -> [{local_seq, Seq}] end. -read_doc(#db{fd=Fd}, OldStreamPointer) when is_tuple(OldStreamPointer) -> - % 09 UPGRADE CODE - couch_stream:old_read_term(Fd, OldStreamPointer); read_doc(#db{fd=Fd}, Pos) -> couch_file:pread_term(Fd, Pos). Modified: couchdb/trunk/src/couchdb/couch_db_updater.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db_updater.erl?rev=1043381&r1=1043380&r2=1043381&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_db_updater.erl (original) +++ couchdb/trunk/src/couchdb/couch_db_updater.erl Wed Dec 8 12:17:14 2010 @@ -30,7 +30,6 @@ init({MainPid, DbName, Filepath, Fd, Opt RootDir = couch_config:get("couchdb", "database_dir", "."), couch_file:delete(RootDir, Filepath ++ ".compact"); false -> - ok = couch_file:upgrade_old_header(Fd, <<$g, $m, $k, 0>>), % 09 UPGRADE CODE case couch_file:read_header(Fd) of {ok, Header} -> ok; @@ -298,19 +297,7 @@ btree_by_seq_join(KeySeq, {Id, RevInfos, [#rev_info{rev=Rev,seq=Seq,deleted=false,body_sp = Bp} || {Rev, Seq, Bp} <- RevInfos] ++ [#rev_info{rev=Rev,seq=Seq,deleted=true,body_sp = Bp} || - {Rev, Seq, Bp} <- DeletedRevInfos]}; -btree_by_seq_join(KeySeq,{Id, Rev, Bp, Conflicts, DelConflicts, Deleted}) -> - % 09 UPGRADE CODE - % this is the 0.9.0 and earlier by_seq record. It's missing the body pointers - % and individual seq nums for conflicts that are currently in the index, - % meaning the filtered _changes api will not work except for on main docs. - % Simply compact a 0.9.0 database to upgrade the index. - #doc_info{ - id=Id, - high_seq=KeySeq, - revs = [#rev_info{rev=Rev,seq=KeySeq,deleted=Deleted,body_sp=Bp}] ++ - [#rev_info{rev=Rev1,seq=KeySeq,deleted=false} || Rev1 <- Conflicts] ++ - [#rev_info{rev=Rev2,seq=KeySeq,deleted=true} || Rev2 <- DelConflicts]}. + {Rev, Seq, Bp} <- DeletedRevInfos]}. btree_by_id_split(#full_doc_info{id=Id, update_seq=Seq, deleted=Deleted, rev_tree=Tree}) -> @@ -329,14 +316,7 @@ btree_by_id_join(Id, {HighSeq, Deleted, fun(_RevId, {IsDeleted, BodyPointer, UpdateSeq}) -> {IsDeleted == 1, BodyPointer, UpdateSeq}; (_RevId, ?REV_MISSING) -> - ?REV_MISSING; - (_RevId, {IsDeleted, BodyPointer}) -> - % 09 UPGRADE CODE - % this is the 0.9.0 and earlier rev info record. It's missing the seq - % nums, which means couchdb will sometimes reexamine unchanged - % documents with the _changes API. - % This is fixed by compacting the database. - {IsDeleted == 1, BodyPointer, HighSeq} + ?REV_MISSING end, DiskTree), #full_doc_info{id=Id, update_seq=HighSeq, deleted=Deleted==1, rev_tree=Tree}. @@ -363,14 +343,16 @@ simple_upgrade_record(Old, New) when tup lists:sublist(tuple_to_list(New), OldSz + 1, tuple_size(New) - OldSz), list_to_tuple(tuple_to_list(Old) ++ NewValuesTail). +-define(OLD_DISK_VERSION_ERROR, + "Database files from versions smaller than 0.10.0 are no longer supported"). init_db(DbName, Filepath, Fd, ReaderFd, Header0, Options) -> Header1 = simple_upgrade_record(Header0, #db_header{}), Header = case element(2, Header1) of - 1 -> Header1#db_header{unused = 0, security_ptr = nil}; % 0.9 - 2 -> Header1#db_header{unused = 0, security_ptr = nil}; % post 0.9 and pre 0.10 - 3 -> Header1#db_header{security_ptr = nil}; % post 0.9 and pre 0.10 + 1 -> throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); + 2 -> throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); + 3 -> throw({database_disk_version_error, ?OLD_DISK_VERSION_ERROR}); 4 -> Header1#db_header{security_ptr = nil}; % 0.10 and pre 0.11 ?LATEST_DISK_VERSION -> Header1; _ -> throw({database_disk_version_error, "Incorrect disk header version"}) @@ -736,28 +718,11 @@ commit_data(Db, _) -> end. -copy_doc_attachments(#db{updater_fd = SrcFd} = SrcDb, {Pos,_RevId}, - SrcSp, DestFd) -> +copy_doc_attachments(#db{updater_fd = SrcFd} = SrcDb, SrcSp, DestFd) -> {ok, {BodyData, BinInfos}} = couch_db:read_doc(SrcDb, SrcSp), % copy the bin values NewBinInfos = lists:map( - fun({Name, {Type, BinSp, AttLen}}) when is_tuple(BinSp) orelse BinSp == null -> - % 09 UPGRADE CODE - {NewBinSp, AttLen, AttLen, Md5, _IdentityMd5} = - couch_stream:old_copy_to_new_stream(SrcFd, BinSp, AttLen, DestFd), - {Name, Type, NewBinSp, AttLen, AttLen, Pos, Md5, identity}; - ({Name, {Type, BinSp, AttLen}}) -> - % 09 UPGRADE CODE - {NewBinSp, AttLen, AttLen, Md5, _IdentityMd5} = - couch_stream:copy_to_new_stream(SrcFd, BinSp, DestFd), - {Name, Type, NewBinSp, AttLen, AttLen, Pos, Md5, identity}; - ({Name, Type, BinSp, AttLen, _RevPos, <<>>}) when - is_tuple(BinSp) orelse BinSp == null -> - % 09 UPGRADE CODE - {NewBinSp, AttLen, AttLen, Md5, _IdentityMd5} = - couch_stream:old_copy_to_new_stream(SrcFd, BinSp, AttLen, DestFd), - {Name, Type, NewBinSp, AttLen, AttLen, AttLen, Md5, identity}; - ({Name, Type, BinSp, AttLen, RevPos, Md5}) -> + fun({Name, Type, BinSp, AttLen, RevPos, Md5}) -> % 010 UPGRADE CODE {NewBinSp, AttLen, AttLen, Md5, _IdentityMd5} = couch_stream:copy_to_new_stream(SrcFd, BinSp, DestFd), @@ -781,8 +746,8 @@ copy_doc_attachments(#db{updater_fd = Sr copy_rev_tree_attachments(SrcDb, DestFd, Tree) -> couch_key_tree:map( - fun(Rev, {IsDel, Sp, Seq}, leaf) -> - DocBody = copy_doc_attachments(SrcDb, Rev, Sp, DestFd), + fun(_Rev, {IsDel, Sp, Seq}, leaf) -> + DocBody = copy_doc_attachments(SrcDb, Sp, DestFd), {IsDel, DocBody, Seq}; (_, _, branch) -> ?REV_MISSING Modified: couchdb/trunk/src/couchdb/couch_doc.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_doc.erl?rev=1043381&r1=1043380&r2=1043381&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_doc.erl (original) +++ couchdb/trunk/src/couchdb/couch_doc.erl Wed Dec 8 12:17:14 2010 @@ -310,9 +310,6 @@ to_doc_info_path(#full_doc_info{id=Id,re att_foldl(#att{data=Bin}, Fun, Acc) when is_binary(Bin) -> Fun(Bin, Acc); -att_foldl(#att{data={Fd,Sp},att_len=Len}, Fun, Acc) when is_tuple(Sp) orelse Sp == null -> - % 09 UPGRADE CODE - couch_stream:old_foldl(Fd, Sp, Len, Fun, Acc); att_foldl(#att{data={Fd,Sp},md5=Md5}, Fun, Acc) -> couch_stream:foldl(Fd, Sp, Md5, Fun, Acc); att_foldl(#att{data=DataFun,att_len=Len}, Fun, Acc) when is_function(DataFun) -> Modified: couchdb/trunk/src/couchdb/couch_file.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_file.erl?rev=1043381&r1=1043380&r2=1043381&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_file.erl (original) +++ couchdb/trunk/src/couchdb/couch_file.erl Wed Dec 8 12:17:14 2010 @@ -18,16 +18,20 @@ -define(SIZE_BLOCK, 4096). -record(file, { - fd, - tail_append_begin = 0 % 09 UPGRADE CODE - }). - --export([open/1, open/2, close/1, bytes/1, sync/1, append_binary/2,old_pread/3]). --export([append_term/2, pread_term/2, pread_iolist/2, write_header/2]). --export([pread_binary/2, read_header/1, truncate/2, upgrade_old_header/2]). --export([append_term_md5/2,append_binary_md5/2]). --export([init/1, terminate/2, handle_call/3, handle_cast/2, code_change/3, handle_info/2]). --export([delete/2,delete/3,init_delete_dir/1]). + fd +}). + +% public API +-export([open/1, open/2, close/1, bytes/1, sync/1, truncate/2]). +-export([pread_term/2, pread_iolist/2, pread_binary/2]). +-export([append_binary/2, append_binary_md5/2]). +-export([append_term/2, append_term_md5/2]). +-export([write_header/2, read_header/1]). +-export([delete/2, delete/3, init_delete_dir/1]). + +% gen_server callbacks +-export([init/1, terminate/2, code_change/3]). +-export([handle_call/3, handle_cast/2, handle_info/2]). %%---------------------------------------------------------------------- %% Args: Valid Options are [create] and [create,overwrite]. @@ -190,16 +194,6 @@ init_delete_dir(RootDir) -> end, ok). -% 09 UPGRADE CODE -old_pread(Fd, Pos, Len) -> - {ok, <<RawBin:Len/binary>>, false} = gen_server:call(Fd, {pread, Pos, Len}, infinity), - {ok, RawBin}. - -% 09 UPGRADE CODE -upgrade_old_header(Fd, Sig) -> - gen_server:call(Fd, {upgrade_old_header, Sig}, infinity). - - read_header(Fd) -> case gen_server:call(Fd, find_header, infinity) of {ok, Bin} -> @@ -313,16 +307,17 @@ handle_call({pread_iolist, Pos}, _From, IoList = maybe_read_more_iolist(RestRawData, Len, NextPos, File), {reply, {ok, IoList}, File} end; -handle_call({pread, Pos, Bytes}, _From, #file{fd=Fd,tail_append_begin=TailAppendBegin}=File) -> - {ok, Bin} = file:pread(Fd, Pos, Bytes), - {reply, {ok, Bin, Pos >= TailAppendBegin}, File}; + handle_call(bytes, _From, #file{fd = Fd} = File) -> {reply, file:position(Fd, eof), File}; + handle_call(sync, _From, #file{fd=Fd}=File) -> {reply, file:sync(Fd), File}; + handle_call({truncate, Pos}, _From, #file{fd=Fd}=File) -> {ok, Pos} = file:position(Fd, Pos), {reply, file:truncate(Fd), File}; + handle_call({append_bin, Bin}, _From, #file{fd = Fd} = File) -> {ok, Pos} = file:position(Fd, eof), Blocks = make_blocks(Pos rem ?SIZE_BLOCK, Bin), @@ -332,6 +327,7 @@ handle_call({append_bin, Bin}, _From, #f Error -> {reply, Error, File} end; + handle_call({write_header, Bin}, _From, #file{fd = Fd} = File) -> {ok, Pos} = file:position(Fd, eof), BinSize = byte_size(Bin), @@ -344,135 +340,13 @@ handle_call({write_header, Bin}, _From, FinalBin = [Padding, <<1, BinSize:32/integer>> | make_blocks(5, [Bin])], {reply, file:write(Fd, FinalBin), File}; -handle_call({upgrade_old_header, Prefix}, _From, #file{fd=Fd}=File) -> - case (catch read_old_header(Fd, Prefix)) of - {ok, Header} -> - {ok, TailAppendBegin} = file:position(Fd, eof), - Bin = term_to_binary(Header), - Md5 = couch_util:md5(Bin), - % now we assemble the final header binary and write to disk - FinalBin = <<Md5/binary, Bin/binary>>, - {reply, ok, _} = handle_call({write_header, FinalBin}, ok, File), - ok = write_old_header(Fd, <<"upgraded">>, TailAppendBegin), - {reply, ok, File#file{tail_append_begin=TailAppendBegin}}; - _Error -> - case (catch read_old_header(Fd, <<"upgraded">>)) of - {ok, TailAppendBegin} -> - {reply, ok, File#file{tail_append_begin = TailAppendBegin}}; - _Error2 -> - {reply, ok, File} - end - end; - - handle_call(find_header, _From, #file{fd = Fd} = File) -> {ok, Pos} = file:position(Fd, eof), {reply, find_header(Fd, Pos div ?SIZE_BLOCK), File}. -% 09 UPGRADE CODE --define(HEADER_SIZE, 2048). % size of each segment of the doubly written header - -% 09 UPGRADE CODE -read_old_header(Fd, Prefix) -> - {ok, Bin} = file:pread(Fd, 0, 2*(?HEADER_SIZE)), - <<Bin1:(?HEADER_SIZE)/binary, Bin2:(?HEADER_SIZE)/binary>> = Bin, - Result = - % read the first header - case extract_header(Prefix, Bin1) of - {ok, Header1} -> - case extract_header(Prefix, Bin2) of - {ok, Header2} -> - case Header1 == Header2 of - true -> - % Everything is completely normal! - {ok, Header1}; - false -> - % To get here we must have two different header versions with signatures intact. - % It's weird but possible (a commit failure right at the 2k boundary). Log it and take the first. - ?LOG_INFO("Header version differences.~nPrimary Header: ~p~nSecondary Header: ~p", [Header1, Header2]), - {ok, Header1} - end; - Error -> - % error reading second header. It's ok, but log it. - ?LOG_INFO("Secondary header corruption (error: ~p). Using primary header.", [Error]), - {ok, Header1} - end; - Error -> - % error reading primary header - case extract_header(Prefix, Bin2) of - {ok, Header2} -> - % log corrupt primary header. It's ok since the secondary is still good. - ?LOG_INFO("Primary header corruption (error: ~p). Using secondary header.", [Error]), - {ok, Header2}; - _ -> - % error reading secondary header too - % return the error, no need to log anything as the caller will be responsible for dealing with the error. - Error - end - end, - case Result of - {ok, {pointer_to_header_data, Ptr}} -> - pread_term(Fd, Ptr); - _ -> - Result - end. - -% 09 UPGRADE CODE -extract_header(Prefix, Bin) -> - SizeOfPrefix = size(Prefix), - SizeOfTermBin = ?HEADER_SIZE - - SizeOfPrefix - - 16, % md5 sig - - <<HeaderPrefix:SizeOfPrefix/binary, TermBin:SizeOfTermBin/binary, Sig:16/binary>> = Bin, - - % check the header prefix - case HeaderPrefix of - Prefix -> - % check the integrity signature - case couch_util:md5(TermBin) == Sig of - true -> - Header = binary_to_term(TermBin), - {ok, Header}; - false -> - header_corrupt - end; - _ -> - unknown_header_type - end. - - -% 09 UPGRADE CODE -write_old_header(Fd, Prefix, Data) -> - TermBin = term_to_binary(Data), - % the size of all the bytes written to the header, including the md5 signature (16 bytes) - FilledSize = byte_size(Prefix) + byte_size(TermBin) + 16, - {TermBin2, FilledSize2} = - case FilledSize > ?HEADER_SIZE of - true -> - % too big! - {ok, Pos} = append_binary(Fd, TermBin), - PtrBin = term_to_binary({pointer_to_header_data, Pos}), - {PtrBin, byte_size(Prefix) + byte_size(PtrBin) + 16}; - false -> - {TermBin, FilledSize} - end, - ok = file:sync(Fd), - % pad out the header with zeros, then take the md5 hash - PadZeros = <<0:(8*(?HEADER_SIZE - FilledSize2))>>, - Sig = couch_util:md5([TermBin2, PadZeros]), - % now we assemble the final header binary and write to disk - WriteBin = <<Prefix/binary, TermBin2/binary, PadZeros/binary, Sig/binary>>, - ?HEADER_SIZE = size(WriteBin), % sanity check - DblWriteBin = [WriteBin, WriteBin], - ok = file:pwrite(Fd, 0, DblWriteBin), - ok = file:sync(Fd). - - handle_cast(close, Fd) -> {stop,normal,Fd}. - code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -523,17 +397,11 @@ maybe_read_more_iolist(Buffer, DataSize, {Data::iolist(), CurPos::non_neg_integer()}. read_raw_iolist_int(Fd, {Pos, _Size}, Len) -> % 0110 UPGRADE CODE read_raw_iolist_int(Fd, Pos, Len); -read_raw_iolist_int(#file{fd=Fd, tail_append_begin=TAB}, Pos, Len) -> +read_raw_iolist_int(#file{fd = Fd}, Pos, Len) -> BlockOffset = Pos rem ?SIZE_BLOCK, TotalBytes = calculate_total_read_len(BlockOffset, Len), {ok, <<RawBin:TotalBytes/binary>>} = file:pread(Fd, Pos, TotalBytes), - if Pos >= TAB -> - {remove_block_prefixes(BlockOffset, RawBin), Pos + TotalBytes}; - true -> - % 09 UPGRADE CODE - <<ReturnBin:Len/binary, _/binary>> = RawBin, - {[ReturnBin], Pos + Len} - end. + {remove_block_prefixes(BlockOffset, RawBin), Pos + TotalBytes}. -spec extract_md5(iolist()) -> {binary(), iolist()}. extract_md5(FullIoList) -> Modified: couchdb/trunk/src/couchdb/couch_stream.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_stream.erl?rev=1043381&r1=1043380&r2=1043381&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_stream.erl (original) +++ couchdb/trunk/src/couchdb/couch_stream.erl Wed Dec 8 12:17:14 2010 @@ -24,11 +24,14 @@ -define(DEFAULT_STREAM_CHUNK, 16#00100000). % 1 meg chunks when streaming data --export([open/1, open/3, close/1, write/2, foldl/4, foldl/5, range_foldl/6, foldl_decode/6, - old_foldl/5,old_copy_to_new_stream/4]). --export([copy_to_new_stream/3,old_read_term/2]). --export([init/1, terminate/2, handle_call/3]). --export([handle_cast/2,code_change/3,handle_info/2]). +% public API +-export([open/1, open/3, close/1]). +-export([foldl/4, foldl/5, foldl_decode/6, range_foldl/6]). +-export([copy_to_new_stream/3, write/2]). + +% gen_server callbacks +-export([init/1, terminate/2, code_change/3]). +-export([handle_cast/2, handle_call/3, handle_info/2]). -include("couch_db.hrl"). @@ -68,23 +71,6 @@ copy_to_new_stream(Fd, PosList, DestFd) end, ok), close(Dest). - -% 09 UPGRADE CODE -old_copy_to_new_stream(Fd, Pos, Len, DestFd) -> - {ok, Dest} = open(DestFd), - old_foldl(Fd, Pos, Len, - fun(Bin, _) -> - ok = write(Dest, Bin) - end, ok), - close(Dest). - -% 09 UPGRADE CODE -old_foldl(_Fd, null, 0, _Fun, Acc) -> - Acc; -old_foldl(Fd, OldPointer, Len, Fun, Acc) when is_tuple(OldPointer)-> - {ok, Acc2, _} = old_stream_data(Fd, OldPointer, Len, ?DEFAULT_STREAM_CHUNK, Fun, Acc), - Acc2. - foldl(_Fd, [], _Fun, Acc) -> Acc; foldl(Fd, [Pos|Rest], Fun, Acc) -> @@ -317,41 +303,3 @@ code_change(_OldVsn, State, _Extra) -> handle_info(_Info, State) -> {noreply, State}. - - - -% 09 UPGRADE CODE -old_read_term(Fd, Sp) -> - {ok, <<TermLen:(?STREAM_OFFSET_BITS)>>, Sp2} - = old_read(Fd, Sp, ?STREAM_OFFSET_BYTES), - {ok, Bin, _Sp3} = old_read(Fd, Sp2, TermLen), - {ok, binary_to_term(Bin)}. - -old_read(Fd, Sp, Num) -> - {ok, RevBin, Sp2} = old_stream_data(Fd, Sp, Num, ?HUGE_CHUNK, fun(Bin, Acc) -> [Bin | Acc] end, []), - Bin = list_to_binary(lists:reverse(RevBin)), - {ok, Bin, Sp2}. - -% 09 UPGRADE CODE -old_stream_data(_Fd, Sp, 0, _MaxChunk, _Fun, Acc) -> - {ok, Acc, Sp}; -old_stream_data(Fd, {Pos, 0}, Num, MaxChunk, Fun, Acc) -> - {ok, <<NextPos:(?FILE_POINTER_BITS), NextOffset:(?STREAM_OFFSET_BITS)>>} - = couch_file:old_pread(Fd, Pos, ?FILE_POINTER_BYTES + ?STREAM_OFFSET_BYTES), - Sp = {NextPos, NextOffset}, - % Check NextPos is past current Pos (this is always true in a stream) - % Guards against potential infinite loops caused by corruption. - case NextPos > Pos of - true -> ok; - false -> throw({error, stream_corruption}) - end, - old_stream_data(Fd, Sp, Num, MaxChunk, Fun, Acc); -old_stream_data(Fd, {Pos, Offset}, Num, MaxChunk, Fun, Acc) -> - ReadAmount = lists:min([MaxChunk, Num, Offset]), - {ok, Bin} = couch_file:old_pread(Fd, Pos, ReadAmount), - Sp = {Pos + ReadAmount, Offset - ReadAmount}, - old_stream_data(Fd, Sp, Num - ReadAmount, MaxChunk, Fun, Fun(Bin, Acc)). - - -% Tests moved to tests/etap/050-stream.t - Modified: couchdb/trunk/src/couchdb/couch_view_group.erl URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_view_group.erl?rev=1043381&r1=1043380&r2=1043381&view=diff ============================================================================== --- couchdb/trunk/src/couchdb/couch_view_group.erl (original) +++ couchdb/trunk/src/couchdb/couch_view_group.erl Wed Dec 8 12:17:14 2010 @@ -396,8 +396,6 @@ prepare_group({RootDir, DbName, #group{s % this can happen if we missed a purge {ok, reset_file(Db, Fd, DbName, Group)}; true -> - % 09 UPGRADE CODE - ok = couch_file:upgrade_old_header(Fd, <<$r, $c, $k, 0>>), case (catch couch_file:read_header(Fd)) of {ok, {Sig, HeaderInfo}} -> % sigs match!