This is an automated email from the ASF dual-hosted git repository. rnewson pushed a commit to branch database_encryption in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit be4f0fb3f831a563f9e72ae9c7a9196896ae8ad6 Author: Robert Newson <[email protected]> AuthorDate: Thu May 5 14:39:37 2022 +0100 Database Encryption Support CouchDB can optionally encrypt databases and views. We use AES in Counter Mode, which ensures we can encrypt and decrypt any section of the file without padding or alignment. The ciphertext is the same length as the plaintext. This mode provides confidentiality but not authentication. Key management is configurable, a system administrator can write a module implementing the aegis_key_manager behaviour with any implementation. CouchDB ships with an implementation that stores keys in the config file as an example, but this is not suitable for production. --- src/couch/src/couch_bt_engine.erl | 7 +- src/couch/src/couch_bt_engine_compactor.erl | 17 +- src/couch/src/couch_db_updater.erl | 6 +- src/couch/src/couch_encryption_manager.erl | 26 +++ src/couch/src/couch_file.erl | 299 +++++++++++++++++++++--- src/couch/src/couch_util.erl | 42 +++- src/couch_mrview/src/couch_mrview_compactor.erl | 6 +- src/couch_mrview/src/couch_mrview_index.erl | 6 +- src/couch_mrview/src/couch_mrview_util.erl | 8 +- 9 files changed, 362 insertions(+), 55 deletions(-) diff --git a/src/couch/src/couch_bt_engine.erl b/src/couch/src/couch_bt_engine.erl index 486ed7cb0..c7dcf0d0d 100644 --- a/src/couch/src/couch_bt_engine.erl +++ b/src/couch/src/couch_bt_engine.erl @@ -827,14 +827,15 @@ copy_props(#st{header = Header} = St, Props) -> needs_commit = true }}. -open_db_file(FilePath, Options) -> - case couch_file:open(FilePath, Options) of +open_db_file(FilePath, Options0) -> + case couch_file:open(FilePath, Options0) of {ok, Fd} -> {ok, Fd}; {error, enoent} -> % Couldn't find file. is there a compact version? This ca % happen (rarely) if we crashed during the file switch. - case couch_file:open(FilePath ++ ".compact", [nologifmissing]) of + Options1 = couch_encryption_manager:encryption_options(Options0), + case couch_file:open(FilePath ++ ".compact", [nologifmissing | Options1]) of {ok, Fd} -> Fmt = "Recovering from compaction file: ~s~s", couch_log:info(Fmt, [FilePath, ".compact"]), diff --git a/src/couch/src/couch_bt_engine_compactor.erl b/src/couch/src/couch_bt_engine_compactor.erl index 8ed55b5c3..e39ab76c1 100644 --- a/src/couch/src/couch_bt_engine_compactor.erl +++ b/src/couch/src/couch_bt_engine_compactor.erl @@ -54,7 +54,11 @@ start(#st{} = St, DbName, Options, Parent) -> couch_db_engine:trigger_on_compact(DbName), ?COMP_EVENT(init), - {ok, InitCompSt} = open_compaction_files(DbName, St, Options), + EncryptionOptions = case couch_encryption_manager:get_key(DbName) of + false -> []; + KEK -> [{kek, KEK}] + end, + {ok, InitCompSt} = open_compaction_files(DbName, St, Options ++ EncryptionOptions), ?COMP_EVENT(files_opened), Stages = [ @@ -94,8 +98,9 @@ open_compaction_files(DbName, OldSt, Options) -> } = OldSt, DataFile = DbFilePath ++ ".compact.data", MetaFile = DbFilePath ++ ".compact.meta", - {ok, DataFd, DataHdr} = open_compaction_file(DataFile), - {ok, MetaFd, MetaHdr} = open_compaction_file(MetaFile), + EncryptionOptions = couch_encryption_manager:encryption_options(Options), + {ok, DataFd, DataHdr} = open_compaction_file(DataFile, EncryptionOptions), + {ok, MetaFd, MetaHdr} = open_compaction_file(MetaFile, EncryptionOptions), DataHdrIsDbHdr = couch_bt_engine_header:is_header(DataHdr), CompSt = case {DataHdr, MetaHdr} of @@ -623,15 +628,15 @@ compact_final_sync(#comp_st{new_st = St0} = CompSt) -> new_st = St1 }. -open_compaction_file(FilePath) -> - case couch_file:open(FilePath, [nologifmissing]) of +open_compaction_file(FilePath, FileOpenOptions) -> + case couch_file:open(FilePath, [nologifmissing | FileOpenOptions]) of {ok, Fd} -> case couch_file:read_header(Fd) of {ok, Header} -> {ok, Fd, Header}; no_valid_header -> {ok, Fd, nil} end; {error, enoent} -> - {ok, Fd} = couch_file:open(FilePath, [create]), + {ok, Fd} = couch_file:open(FilePath, [create | FileOpenOptions]), {ok, Fd, nil} end. diff --git a/src/couch/src/couch_db_updater.erl b/src/couch/src/couch_db_updater.erl index 17a1e9160..eca81c7f0 100644 --- a/src/couch/src/couch_db_updater.erl +++ b/src/couch/src/couch_db_updater.erl @@ -37,7 +37,11 @@ init({Engine, DbName, FilePath, Options0}) -> erlang:put(io_priority, {db_update, DbName}), update_idle_limit_from_config(), DefaultSecObj = default_security_object(DbName), - Options = [{default_security_object, DefaultSecObj} | Options0], + Options = [{default_security_object, DefaultSecObj} | Options0] ++ + case couch_encryption_manager:get_key(DbName) of + false -> []; + KEK -> [{kek, KEK}] + end, try {ok, EngineState} = couch_db_engine:init(Engine, FilePath, Options), Db = init_db(DbName, FilePath, EngineState, Options), diff --git a/src/couch/src/couch_encryption_manager.erl b/src/couch/src/couch_encryption_manager.erl new file mode 100644 index 000000000..ca736324e --- /dev/null +++ b/src/couch/src/couch_encryption_manager.erl @@ -0,0 +1,26 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_encryption_manager). + +-export([get_key/1, encryption_options/1]). + +-spec get_key(DbName :: binary()) -> KEK :: binary() | false. +get_key(_DbName) -> + <<0:256>>. + +%% Extract just the encryption related options from an options list. +encryption_options(Options) -> + case lists:keyfind(kek, 1, Options) of + false -> []; + {kek, KEK} -> [{kek, KEK}] + end. \ No newline at end of file diff --git a/src/couch/src/couch_file.erl b/src/couch/src/couch_file.erl index ba8d9c42f..879070016 100644 --- a/src/couch/src/couch_file.erl +++ b/src/couch/src/couch_file.erl @@ -23,6 +23,19 @@ -define(IS_OLD_STATE(S), is_pid(S#file.db_monitor)). -define(PREFIX_SIZE, 5). -define(DEFAULT_READ_COUNT, 1024). +-define(ENCRYPTED_HEADER, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15). + +%% Database encryption design details + +%% If, at couch_file creation time, encryption is enabled, couch_file +%% generates a random 256-bit AES key and a random 128 bit +%% initialisation vector. The currently configured key manager +%% module is asked to wrap the key and return it as a +%% binary. couch_file stores the IV and the wrapped key in a special +%% header block, always block 0, in the first 4 KiB of the file. All +%% data, including all headers except the encryption header at block +%% 0, are encrypted using AES in Counter Mode, where the counter is +%% calculated as IV plus the file offset, in AES block increments. -type block_id() :: non_neg_integer(). -type location() :: non_neg_integer(). @@ -33,9 +46,55 @@ is_sys, eof = 0, db_monitor, - pread_limit = 0 + pread_limit = 0, + iv, + dek, + wek }). +-define(encrypt_ctr(File, Pos, Data), + crypto:stream_encrypt( + crypto:stream_init(aes_ctr, File#file.dek, aes_ctr(File#file.iv, Pos)), Data + ) +). +-define(decrypt_ctr(File, Pos, Data), + crypto:stream_decrypt( + crypto:stream_init(aes_ctr, File#file.dek, aes_ctr(File#file.iv, Pos)), Data + ) +). + +-define(aes_gcm_encrypt(Key, IV, AAD, Data), + crypto:block_encrypt(aes_gcm, Key, IV, {AAD, Data, 16})), + +-define(aes_gcm_decrypt(Key, IV, AAD, CipherText, CipherTag), + crypto:block_decrypt(aes_gcm, Key, IV, {AAD, CipherText, CipherTag})). + +-ifdef(OTP_RELEASE). +-if(?OTP_RELEASE >= 22). + +-undef(encrypt_ctr). +-define(encrypt_ctr(File, Pos, Data), + crypto:crypto_one_time(aes_256_ctr, File#file.dek, aes_ctr(File#file.iv, Pos), Data, true) +). + +-undef(decrypt_ctr). +-define(decrypt_ctr(File, Pos, Data), + crypto:crypto_one_time(aes_256_ctr, File#file.dek, aes_ctr(File#file.iv, Pos), Data, false) +). + +-undef(aes_gcm_encrypt). +-define(aes_gcm_encrypt(Key, IV, AAD, Data), + crypto:crypto_one_time_aead(aes_256_gcm, Key, IV, Data, AAD, 16, true) +). + +-undef(aes_gcm_decrypt). +-define(aes_gcm_decrypt(Key, IV, AAD, CipherText, CipherTag), + crypto:crypto_one_time_aead(aes_256_gcm, Key, IV, CipherText, AAD, CipherTag, false) +). + +-endif. +-endif. + % public API -export([open/1, open/2, close/1, bytes/1, sync/1, truncate/2, set_db_pid/2]). -export([pread_term/2, pread_iolist/2, pread_binary/2]). @@ -439,7 +498,12 @@ init({Filepath, Options, ReturnPid, Ref}) -> ok = file:sync(Fd), maybe_track_open_os_files(Options), erlang:send_after(?INITIAL_WAIT, self(), maybe_close), - {ok, #file{fd = Fd, is_sys = IsSys, pread_limit = Limit}}; + init_crypto( + #file{ + fd = Fd, is_sys = IsSys, pread_limit = Limit + }, + Options + ); false -> ok = file:close(Fd), init_status_error(ReturnPid, Ref, {error, eexist}) @@ -447,7 +511,12 @@ init({Filepath, Options, ReturnPid, Ref}) -> false -> maybe_track_open_os_files(Options), erlang:send_after(?INITIAL_WAIT, self(), maybe_close), - {ok, #file{fd = Fd, is_sys = IsSys, pread_limit = Limit}} + init_crypto( + #file{ + fd = Fd, is_sys = IsSys, pread_limit = Limit + }, + Options + ) end; Error -> init_status_error(ReturnPid, Ref, Error) @@ -464,7 +533,12 @@ init({Filepath, Options, ReturnPid, Ref}) -> maybe_track_open_os_files(Options), {ok, Eof} = file:position(Fd, eof), erlang:send_after(?INITIAL_WAIT, self(), maybe_close), - {ok, #file{fd = Fd, eof = Eof, is_sys = IsSys, pread_limit = Limit}}; + init_crypto( + #file{ + fd = Fd, eof = Eof, is_sys = IsSys, pread_limit = Limit + }, + Options + ); Error -> init_status_error(ReturnPid, Ref, Error) end; @@ -567,20 +641,25 @@ handle_call({truncate, Pos}, _From, #file{fd = Fd} = File) -> {ok, Pos} = file:position(Fd, Pos), case file:truncate(Fd) of ok -> - {reply, ok, File#file{eof = Pos}}; + case init_crypto(File#file{eof = Pos}, []) of + {ok, File1} -> + {reply, ok, File1}; + {error, Reason} -> + {error, Reason} + end; Error -> {reply, Error, File} end; -handle_call({append_bin, Bin}, _From, #file{fd = Fd, eof = Pos} = File) -> +handle_call({append_bin, Bin}, _From, #file{eof = Pos} = File) -> Blocks = make_blocks(Pos rem ?SIZE_BLOCK, Bin), Size = iolist_size(Blocks), - case file:write(Fd, Blocks) of + case encrypted_write(File, Blocks) of ok -> {reply, {ok, Pos, Size}, File#file{eof = Pos + Size}}; Error -> {reply, Error, reset_eof(File)} end; -handle_call({append_bins, Bins}, _From, #file{fd = Fd, eof = Pos} = File) -> +handle_call({append_bins, Bins}, _From, #file{eof = Pos} = File) -> {BlockResps, FinalPos} = lists:mapfoldl( fun(Bin, PosAcc) -> Blocks = make_blocks(PosAcc rem ?SIZE_BLOCK, Bin), @@ -591,13 +670,13 @@ handle_call({append_bins, Bins}, _From, #file{fd = Fd, eof = Pos} = File) -> Bins ), {AllBlocks, Resps} = lists:unzip(BlockResps), - case file:write(Fd, AllBlocks) of + case encrypted_write(File, AllBlocks) of ok -> {reply, {ok, Resps}, File#file{eof = FinalPos}}; Error -> {reply, Error, reset_eof(File)} end; -handle_call({write_header, Bin}, _From, #file{fd = Fd, eof = Pos} = File) -> +handle_call({write_header, Bin}, _From, #file{eof = Pos} = File) -> BinSize = byte_size(Bin), case Pos rem ?SIZE_BLOCK of 0 -> @@ -606,14 +685,14 @@ handle_call({write_header, Bin}, _From, #file{fd = Fd, eof = Pos} = File) -> Padding = <<0:(8 * (?SIZE_BLOCK - BlockOffset))>> end, FinalBin = [Padding, <<1, BinSize:32/integer>> | make_blocks(5, [Bin])], - case file:write(Fd, FinalBin) of + case encrypted_write(File, FinalBin) of ok -> {reply, ok, File#file{eof = Pos + iolist_size(FinalBin)}}; Error -> {reply, Error, reset_eof(File)} end; -handle_call(find_header, _From, #file{fd = Fd, eof = Pos} = File) -> - {reply, find_header(Fd, Pos div ?SIZE_BLOCK), File}. +handle_call(find_header, _From, #file{eof = Pos} = File) -> + {reply, find_header(File, Pos div ?SIZE_BLOCK), File}. handle_cast(close, Fd) -> {stop, normal, Fd}. @@ -641,26 +720,26 @@ format_status(_Opt, [PDict, #file{} = File]) -> {_Fd, FilePath} = couch_util:get_value(couch_file_fd, PDict), [{data, [{"State", File}, {"InitialFilePath", FilePath}]}]. -find_header(Fd, Block) -> - case (catch load_header(Fd, Block)) of +find_header(#file{} = File, Block) -> + case (catch load_header(File, Block)) of {ok, Bin} -> {ok, Bin}; _Error -> ReadCount = config:get_integer( "couchdb", "find_header_read_count", ?DEFAULT_READ_COUNT ), - find_header(Fd, Block - 1, ReadCount) + find_header(File, Block - 1, ReadCount) end. -load_header(Fd, Block) -> +load_header(#file{} = File, Block) -> {ok, <<1, HeaderLen:32/integer, RestBlock/binary>>} = - file:pread(Fd, Block * ?SIZE_BLOCK, ?SIZE_BLOCK), - load_header(Fd, Block * ?SIZE_BLOCK, HeaderLen, RestBlock). + encrypted_pread(File, Block * ?SIZE_BLOCK, ?SIZE_BLOCK), + load_header(File, Block * ?SIZE_BLOCK, HeaderLen, RestBlock). -load_header(Fd, Pos, HeaderLen) -> - load_header(Fd, Pos, HeaderLen, <<>>). +load_header(#file{} = File, Pos, HeaderLen) -> + load_header(File, Pos, HeaderLen, <<>>). -load_header(Fd, Pos, HeaderLen, RestBlock) -> +load_header(#file{} = File, Pos, HeaderLen, RestBlock) -> TotalBytes = calculate_total_read_len(?PREFIX_SIZE, HeaderLen), RawBin = case TotalBytes =< byte_size(RestBlock) of @@ -670,7 +749,7 @@ load_header(Fd, Pos, HeaderLen, RestBlock) -> false -> ReadStart = Pos + ?PREFIX_SIZE + byte_size(RestBlock), ReadLen = TotalBytes - byte_size(RestBlock), - {ok, Missing} = file:pread(Fd, ReadStart, ReadLen), + {ok, Missing} = encrypted_pread(File, ReadStart, ReadLen), <<RestBlock/binary, Missing/binary>> end, <<Md5Sig:16/binary, HeaderBin/binary>> = @@ -681,12 +760,12 @@ load_header(Fd, Pos, HeaderLen, RestBlock) -> %% Read multiple block locations using a single file:pread/2. -spec find_header(file:fd(), block_id(), non_neg_integer()) -> {ok, binary()} | no_valid_header. -find_header(_Fd, Block, _ReadCount) when Block < 0 -> +find_header(_File, Block, _ReadCount) when Block < 0 -> no_valid_header; -find_header(Fd, Block, ReadCount) -> +find_header(#file{} = File, Block, ReadCount) -> FirstBlock = max(0, Block - ReadCount + 1), BlockLocations = [?SIZE_BLOCK * B || B <- lists:seq(FirstBlock, Block)], - {ok, DataL} = file:pread(Fd, [{L, ?PREFIX_SIZE} || L <- BlockLocations]), + {ok, DataL} = encrypted_pread(File, [{L, ?PREFIX_SIZE} || L <- BlockLocations]), %% Since BlockLocations are ordered from oldest to newest, we rely %% on lists:foldl/3 to reverse the order, making HeaderLocations %% correctly ordered from newest to oldest. @@ -700,27 +779,27 @@ find_header(Fd, Block, ReadCount) -> [], lists:zip(BlockLocations, DataL) ), - case find_newest_header(Fd, HeaderLocations) of + case find_newest_header(File, HeaderLocations) of {ok, _Location, HeaderBin} -> {ok, HeaderBin}; _ -> ok = file:advise( - Fd, hd(BlockLocations), ReadCount * ?SIZE_BLOCK, dont_need + File#file.fd, hd(BlockLocations), ReadCount * ?SIZE_BLOCK, dont_need ), NextBlock = hd(BlockLocations) div ?SIZE_BLOCK - 1, - find_header(Fd, NextBlock, ReadCount) + find_header(File, NextBlock, ReadCount) end. -spec find_newest_header(file:fd(), [{location(), header_size()}]) -> {ok, location(), binary()} | not_found. -find_newest_header(_Fd, []) -> +find_newest_header(_File, []) -> not_found; -find_newest_header(Fd, [{Location, Size} | LocationSizes]) -> - case (catch load_header(Fd, Location, Size)) of +find_newest_header(#file{} = File, [{Location, Size} | LocationSizes]) -> + case (catch load_header(File, Location, Size)) of {ok, HeaderBin} -> {ok, Location, HeaderBin}; _Error -> - find_newest_header(Fd, LocationSizes) + find_newest_header(File, LocationSizes) end. -spec read_raw_iolist_int(#file{}, Pos :: non_neg_integer(), Len :: non_neg_integer()) -> @@ -728,9 +807,9 @@ find_newest_header(Fd, [{Location, Size} | LocationSizes]) -> % 0110 UPGRADE CODE read_raw_iolist_int(Fd, {Pos, _Size}, Len) -> read_raw_iolist_int(Fd, Pos, Len); -read_raw_iolist_int(#file{fd = Fd} = File, Pos, Len) -> +read_raw_iolist_int(#file{} = File, Pos, Len) -> {Pos, TotalBytes} = get_pread_locnum(File, Pos, Len), - case catch file:pread(Fd, Pos, TotalBytes) of + case catch encrypted_pread(File, Pos, TotalBytes) of {ok, <<RawBin:TotalBytes/binary>>} -> {remove_block_prefixes(Pos rem ?SIZE_BLOCK, RawBin), Pos + TotalBytes}; Else -> @@ -744,15 +823,15 @@ read_raw_iolist_int(#file{fd = Fd} = File, Pos, Len) -> throw({file_truncate_error, Else, Filepath}) end. -% TODO: check if this is really unused -read_multi_raw_iolists_int(#file{fd = Fd} = File, PosLens) -> +% used in couch_bt_engine_compactor.erl via pread_terms/2 +read_multi_raw_iolists_int(#file{} = File, PosLens) -> LocNums = lists:map( fun({Pos, Len}) -> get_pread_locnum(File, Pos, Len) end, PosLens ), - {ok, Bins} = file:pread(Fd, LocNums), + {ok, Bins} = encrypted_pread(File, LocNums), lists:zipwith( fun({Pos, TotalBytes}, Bin) -> <<RawBin:TotalBytes/binary>> = Bin, @@ -905,6 +984,150 @@ reset_eof(#file{} = File) -> {ok, Eof} = file:position(File#file.fd, eof), File#file{eof = Eof}. +%% new file. +init_crypto(#file{eof = 0, dek = undefined, wek = undefined} = File0, Options) -> + case lists:keyfind(kek, 1, Options) of + {kek, KEK} -> + DEK = crypto:strong_rand_bytes(32), + WEK = wrap_key(KEK, DEK), + IV = crypto:strong_rand_bytes(16), + case write_encryption_header(File0, WEK, IV) of + {ok, File1} -> + ok = file:sync(File1#file.fd), + {ok, init_crypto(File1, WEK, DEK, IV)}; + {error, Reason} -> + {error, Reason} + end; + false -> + {ok, File0} + end; +%% truncated file. +init_crypto(#file{eof = Pos, dek = DEK, wek = WEK} = File0, _Options) when + Pos < ?SIZE_BLOCK, is_binary(DEK), is_binary(WEK) +-> + IV = crypto:strong_rand_bytes(16), + case write_encryption_header(File0, WEK, IV) of + {ok, File1} -> + ok = file:sync(File1#file.fd), + {ok, init_crypto(File1, WEK, DEK, IV)}; + {error, Reason} -> + {error, Reason} + end; +%% we're opening an existing file and need to unwrap the key if file is encrypted. +init_crypto(#file{eof = Pos, dek = undefined} = File, Options) when Pos >= ?SIZE_BLOCK -> + case read_encryption_header(File) of + {ok, WEK, IV} -> + case lists:keyfind(kek, 1, Options) of + {kek, KEK} -> + case unwrap_key(KEK, WEK) of + error -> + {error, <<"failed to unwrap encryption key">>}; + DEK when is_binary(DEK) -> + {ok, init_crypto(File, WEK, DEK, IV)} + end; + false -> + {error, <<"required encryption key not supplied">>} + end; + not_encrypted -> + {ok, File}; + {error, Reason} -> + {error, Reason} + end. + +init_crypto(#file{} = File, WEK, DEK, IV) when is_binary(WEK), is_binary(DEK), is_binary(IV) -> + File#file{iv = crypto:bytes_to_integer(IV), wek = WEK, dek = DEK}. + +wrap_key(KEK, DEK) when is_binary(KEK), is_binary(DEK) -> + IV = crypto:strong_rand_bytes(16), + {<<_:32/binary>> = CipherText, <<_:16/binary>> = CipherTag} = ?aes_gcm_encrypt( + KEK, IV, <<>>, DEK + ), + <<IV:16/binary, CipherText/binary, CipherTag/binary>>. + +unwrap_key(KEK, <<IV:16/binary, CipherText:32/binary, CipherTag:16/binary>>) when is_binary(KEK) -> + ?aes_gcm_decrypt(KEK, IV, <<>>, CipherText, CipherTag). + +write_encryption_header(#file{eof = Pos} = File, WrappedKey, IV) when + Pos < ?SIZE_BLOCK, byte_size(WrappedKey) < 1024, bit_size(IV) == 128 +-> + Header = [<<?ENCRYPTED_HEADER>>, IV, <<(byte_size(WrappedKey)):16>>, WrappedKey], + PaddedHeader = [Header, <<0:((?SIZE_BLOCK - iolist_size(Header) - 32) * 8)>>], + DigestHeader = [PaddedHeader, crypto:hash(sha256, PaddedHeader)], + ?SIZE_BLOCK = iolist_size(DigestHeader), + case file:pwrite(File#file.fd, 0, DigestHeader) of + ok -> + {ok, File#file{eof = ?SIZE_BLOCK}}; + {error, Reason} -> + {error, Reason} + end. + +read_encryption_header(#file{} = File) -> + case file:pread(File#file.fd, 0, ?SIZE_BLOCK) of + {ok, + <<?ENCRYPTED_HEADER, IV:16/binary, WrappedKeyLen:16, WrappedKey:(WrappedKeyLen)/binary, + _/binary>> = DigestHeader} -> + Header = binary:part(DigestHeader, 0, ?SIZE_BLOCK - 32), + Digest = binary:part(DigestHeader, ?SIZE_BLOCK - 32, 32), + case Digest == crypto:hash(sha256, Header) of + true -> + {ok, WrappedKey, IV}; + false -> + {error, corrupted_encryption_header} + end; + {ok, _} -> + not_encrypted; + {error, Reason} -> + {error, Reason} + end. + +%% We can encrypt any section of the file but we must make +%% sure we align with the key stream. +encrypted_write(#file{dek = undefined} = File, Data) -> + file:write(File#file.fd, Data); +encrypted_write(#file{} = File, Data) -> + CipherText = ?encrypt_ctr(File, File#file.eof, pad(File#file.eof, Data)), + file:write(File#file.fd, unpad(File#file.eof, CipherText)). + +encrypted_pread(#file{dek = undefined} = File, LocNums) -> + file:pread(File#file.fd, LocNums); +encrypted_pread(#file{} = File, LocNums) -> + case file:pread(File#file.fd, LocNums) of + {ok, DataL} -> + {ok, + lists:zipwith( + fun({Pos, _Len}, CipherText) -> + PlainText = ?decrypt_ctr(File, Pos, pad(Pos, CipherText)), + unpad(Pos, PlainText) + end, + LocNums, + DataL + )}; + Else -> + Else + end. + +encrypted_pread(#file{dek = undefined} = File, Pos, Len) -> + file:pread(File#file.fd, Pos, Len); +encrypted_pread(#file{} = File, Pos, Len) -> + case file:pread(File#file.fd, Pos, Len) of + {ok, CipherText} -> + PlainText = ?decrypt_ctr(File, Pos, pad(Pos, CipherText)), + {ok, unpad(Pos, PlainText)}; + Else -> + Else + end. + +aes_ctr(IV, Pos) -> + <<(IV + (Pos div 16)):128>>. + +pad(Pos, IOData) -> + [<<0:(Pos rem 16 * 8)>>, IOData]. + +unpad(Pos, Bin) when is_binary(Bin) -> + Size = Pos rem 16 * 8, + <<_:Size, Result/binary>> = Bin, + Result. + -ifdef(TEST). -include_lib("couch/include/couch_eunit.hrl"). diff --git a/src/couch/src/couch_util.erl b/src/couch/src/couch_util.erl index 912c6dd8a..fd1c30072 100644 --- a/src/couch/src/couch_util.erl +++ b/src/couch/src/couch_util.erl @@ -17,7 +17,7 @@ -export([rand32/0, implode/2]). -export([abs_pathname/1, abs_pathname/2, trim/1, drop_dot_couch_ext/1]). -export([encodeBase64Url/1, decodeBase64Url/1]). --export([validate_utf8/1, to_hex/1, parse_term/1, dict_find/3]). +-export([validate_utf8/1, to_hex/1, from_hex/1, parse_term/1, dict_find/3]). -export([get_nested_json_value/2, json_user_ctx/1]). -export([proplist_apply_field/2, json_apply_field/2]). -export([to_binary/1, to_integer/1, to_list/1, url_encode/1]). @@ -236,6 +236,46 @@ nibble_to_hex(13) -> $d; nibble_to_hex(14) -> $e; nibble_to_hex(15) -> $f. +from_hex(<<Hi:8, Lo:8, Rest/binary>>) -> + iolist_to_binary([<<(hex_to_nibble(Hi)):4, (hex_to_nibble(Lo)):4>> | from_hex(Rest)]); +from_hex(<<>>) -> + []; +from_hex(List) when is_list(List) -> + from_hex(list_to_binary(List)). + +hex_to_nibble($0) -> + 0; +hex_to_nibble($1) -> + 1; +hex_to_nibble($2) -> + 2; +hex_to_nibble($3) -> + 3; +hex_to_nibble($4) -> + 4; +hex_to_nibble($5) -> + 5; +hex_to_nibble($6) -> + 6; +hex_to_nibble($7) -> + 7; +hex_to_nibble($8) -> + 8; +hex_to_nibble($9) -> + 9; +hex_to_nibble($a) -> + 10; +hex_to_nibble($b) -> + 11; +hex_to_nibble($c) -> + 12; +hex_to_nibble($d) -> + 13; +hex_to_nibble($e) -> + 14; +hex_to_nibble($f) -> + 15. + parse_term(Bin) when is_binary(Bin) -> parse_term(binary_to_list(Bin)); parse_term(List) -> diff --git a/src/couch_mrview/src/couch_mrview_compactor.erl b/src/couch_mrview/src/couch_mrview_compactor.erl index 28e5a9b3d..be67adbcb 100644 --- a/src/couch_mrview/src/couch_mrview_compactor.erl +++ b/src/couch_mrview/src/couch_mrview_compactor.erl @@ -47,7 +47,11 @@ compact(State) -> {EmptyState, NumDocIds} = couch_util:with_db(DbName, fun(Db) -> CompactFName = couch_mrview_util:compaction_file(DbName, Sig), - {ok, Fd} = couch_mrview_util:open_file(CompactFName), + Options = case couch_encryption_manager:get_key(DbName) of + false -> []; + KEK -> [{kek, KEK}] + end, + {ok, Fd} = couch_mrview_util:open_file(CompactFName, Options), ESt = couch_mrview_util:reset_index(Db, Fd, State), {ok, Count} = couch_db:get_doc_count(Db), diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl index 1bfdb2818..bcbb19ce5 100644 --- a/src/couch_mrview/src/couch_mrview_index.erl +++ b/src/couch_mrview/src/couch_mrview_index.erl @@ -120,7 +120,11 @@ open(Db, State0) -> OldSig = couch_mrview_util:maybe_update_index_file(State), - case couch_mrview_util:open_file(IndexFName) of + Options = case couch_encryption_manager:get_key(DbName) of + false -> []; + KEK -> [{kek, KEK}] + end, + case couch_mrview_util:open_file(IndexFName, Options) of {ok, Fd} -> case couch_file:read_header(Fd) of % upgrade code for <= 2.x diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl index 9e3d292ed..f9c533376 100644 --- a/src/couch_mrview/src/couch_mrview_util.erl +++ b/src/couch_mrview/src/couch_mrview_util.erl @@ -17,7 +17,7 @@ -export([verify_view_filename/1, get_signature_from_filename/1]). -export([ddoc_to_mrst/2, init_state/4, reset_index/3]). -export([make_header/1]). --export([index_file/2, compaction_file/2, open_file/1]). +-export([index_file/2, compaction_file/2, open_file/2]). -export([delete_files/2, delete_index_file/2, delete_compaction_file/2]). -export([get_row_count/1, all_docs_reduce_to_count/1, reduce_to_count/1]). -export([all_docs_key_opts/1, all_docs_key_opts/2, key_opts/1, key_opts/2]). @@ -791,10 +791,10 @@ compaction_file(DbName, Sig) -> FileName = couch_index_util:hexsig(Sig) ++ ".compact.view", couch_index_util:index_file(mrview, DbName, FileName). -open_file(FName) -> - case couch_file:open(FName, [nologifmissing]) of +open_file(FName, Options) -> + case couch_file:open(FName, [nologifmissing | Options]) of {ok, Fd} -> {ok, Fd}; - {error, enoent} -> couch_file:open(FName, [create]); + {error, enoent} -> couch_file:open(FName, [create | Options]); Error -> Error end.
