Paul,
Here:
+stream_chunked_response(Req, ReqId, Resp) ->
+ receive
+ {ibrowse_async_response, ReqId, Chunk} ->
+ couch_httpd:send_chunk(Resp, Chunk),
+ ibrowse:stream_next(ReqId),
+ stream_chunked_response(Req, ReqId, Resp);
+ {ibrowse_async_response, ReqId, {error, Reason}} ->
+ throw({error, Reason});
+ {ibrowse_async_response_end, ReqId} ->
+ couch_httpd:last_chunk(Resp)
+ end.
+
+
+stream_length_response(Req, ReqId, Resp) ->
+ receive
+ {ibrowse_async_response, ReqId, Chunk} ->
+ couch_httpd:send(Resp, Chunk),
+ ibrowse:stream_next(ReqId),
+ stream_length_response(Req, ReqId, Resp);
+ {ibrowse_async_response, {error, Reason}} ->
+ throw({error, Reason});
+ {ibrowse_async_response_end, ReqId} ->
+ ok
+ end.
The " {ibrowse_async_response, ReqId, {error, Reason}} " clauses
should come before the " {ibrowse_async_response, ReqId, Chunk} "
clauses, since the last always mask the former.
Also, in the stream_length_response function you're missing the ReqId
element before the {error, Reason} element.
Good work :D
On Fri, Nov 5, 2010 at 11:26 PM, <[email protected]> wrote:
> Author: davisp
> Date: Fri Nov 5 23:26:21 2010
> New Revision: 1031877
>
> URL: http://svn.apache.org/viewvc?rev=1031877&view=rev
> Log:
> HTTP proxy handler.
>
> The second of two new features to replace the _externals protocols. This
> allows users to configure CouchDB to proxy requests to an external HTTP
> server. The external HTTP server is not required to be on the same host
> running CouchDB.
>
> The configuration looks like such:
>
> [httpd_global_handlers]
> _google = {couch_httpd_proxy, handle_proxy_req, <<"http://www.google.com";>>}
>
> You can then hit this proxy at the url:
>
> http://127.0.0.1:5984/_google
>
> If you add any path after the proxy name, or make a request with a query
> string, those will be appended to the URL specified in the configuration.
>
> Ie:
>
> http://127.0.0.1:5984/_google/search?q=plankton
>
> would translate to:
>
> http://www.google.com/search?q=plankton
>
> Obviously, request bodies are handled as expected.
>
>
> Added:
> couchdb/trunk/src/couchdb/couch_httpd_proxy.erl
> couchdb/trunk/test/etap/180-http-proxy.ini
> couchdb/trunk/test/etap/180-http-proxy.t
> couchdb/trunk/test/etap/test_web.erl
> Modified:
> couchdb/trunk/etc/couchdb/local.ini
> couchdb/trunk/share/www/script/test/basics.js
> couchdb/trunk/src/couchdb/Makefile.am
> couchdb/trunk/src/couchdb/couch_httpd.erl
> couchdb/trunk/test/etap/ (props changed)
> couchdb/trunk/test/etap/Makefile.am
>
> Modified: couchdb/trunk/etc/couchdb/local.ini
> URL:
> http://svn.apache.org/viewvc/couchdb/trunk/etc/couchdb/local.ini?rev=1031877&r1=1031876&r2=1031877&view=diff
> ==============================================================================
> --- couchdb/trunk/etc/couchdb/local.ini (original)
> +++ couchdb/trunk/etc/couchdb/local.ini Fri Nov 5 23:26:21 2010
> @@ -20,6 +20,9 @@
> ; the whitelist.
> ;config_whitelist = [{httpd,config_whitelist}, {log,level}, {etc,etc}]
>
> +[httpd_global_handlers]
> +;_google = {couch_httpd_proxy, handle_proxy_req, <<"http://www.google.com";>>}
> +
> [couch_httpd_auth]
> ; If you set this to true, you should also uncomment the WWW-Authenticate
> line
> ; above. If you don't configure a WWW-Authenticate header, CouchDB will send
>
> Modified: couchdb/trunk/share/www/script/test/basics.js
> URL:
> http://svn.apache.org/viewvc/couchdb/trunk/share/www/script/test/basics.js?rev=1031877&r1=1031876&r2=1031877&view=diff
> ==============================================================================
> --- couchdb/trunk/share/www/script/test/basics.js (original)
> +++ couchdb/trunk/share/www/script/test/basics.js Fri Nov 5 23:26:21 2010
> @@ -159,8 +159,8 @@ couchTests.basics = function(debug) {
> var loc = xhr.getResponseHeader("Location");
> T(loc, "should have a Location header");
> var locs = loc.split('/');
> - T(locs[4] == resp.id);
> - T(locs[3] == "test_suite_db");
> + T(locs[locs.length-1] == resp.id);
> + T(locs[locs.length-2] == "test_suite_db");
>
> // test that that POST's with an _id aren't overriden with a UUID.
> var xhr = CouchDB.request("POST", "/test_suite_db", {
>
> Modified: couchdb/trunk/src/couchdb/Makefile.am
> URL:
> http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/Makefile.am?rev=1031877&r1=1031876&r2=1031877&view=diff
> ==============================================================================
> --- couchdb/trunk/src/couchdb/Makefile.am (original)
> +++ couchdb/trunk/src/couchdb/Makefile.am Fri Nov 5 23:26:21 2010
> @@ -50,6 +50,7 @@ source_files = \
> couch_httpd_show.erl \
> couch_httpd_view.erl \
> couch_httpd_misc_handlers.erl \
> + couch_httpd_proxy.erl \
> couch_httpd_rewrite.erl \
> couch_httpd_stats_handlers.erl \
> couch_httpd_vhost.erl \
> @@ -107,6 +108,7 @@ compiled_files = \
> couch_httpd_db.beam \
> couch_httpd_auth.beam \
> couch_httpd_oauth.beam \
> + couch_httpd_proxy.beam \
> couch_httpd_external.beam \
> couch_httpd_show.beam \
> couch_httpd_view.beam \
>
> Modified: couchdb/trunk/src/couchdb/couch_httpd.erl
> URL:
> http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd.erl?rev=1031877&r1=1031876&r2=1031877&view=diff
> ==============================================================================
> --- couchdb/trunk/src/couchdb/couch_httpd.erl (original)
> +++ couchdb/trunk/src/couchdb/couch_httpd.erl Fri Nov 5 23:26:21 2010
> @@ -22,7 +22,7 @@
> -export([parse_form/1,json_body/1,json_body_obj/1,body/1,doc_etag/1,
> make_etag/1, etag_respond/3]).
> -export([primary_header_value/2,partition/1,serve_file/3,serve_file/4,
> server_header/0]).
> -export([start_chunked_response/3,send_chunk/2,log_request/2]).
> --export([start_response_length/4, send/2]).
> +-export([start_response_length/4, start_response/3, send/2]).
> -export([start_json_response/2, start_json_response/3, end_json_response/1]).
> -export([send_response/4,send_method_not_allowed/2,send_error/4,
> send_redirect/2,send_chunked_error/2]).
> -export([send_json/2,send_json/3,send_json/4,last_chunk/1,parse_multipart_request/3]).
> @@ -526,6 +526,18 @@ start_response_length(#httpd{mochi_req=M
> end,
> {ok, Resp}.
>
> +start_response(#httpd{mochi_req=MochiReq}=Req, Code, Headers) ->
> + log_request(Req, Code),
> + couch_stats_collector:increment({httpd_status_cdes, Code}),
> + CookieHeader = couch_httpd_auth:cookie_auth_header(Req, Headers),
> + Headers2 = Headers ++ server_header() ++ CookieHeader,
> + Resp = MochiReq:start_response({Code, Headers2}),
> + case MochiReq:get(method) of
> + 'HEAD' -> throw({http_head_abort, Resp});
> + _ -> ok
> + end,
> + {ok, Resp}.
> +
> send(Resp, Data) ->
> Resp:send(Data),
> {ok, Resp}.
>
> Added: couchdb/trunk/src/couchdb/couch_httpd_proxy.erl
> URL:
> http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_proxy.erl?rev=1031877&view=auto
> ==============================================================================
> --- couchdb/trunk/src/couchdb/couch_httpd_proxy.erl (added)
> +++ couchdb/trunk/src/couchdb/couch_httpd_proxy.erl Fri Nov 5 23:26:21 2010
> @@ -0,0 +1,425 @@
> +% Licensed under the Apache License, Version 2.0 (the "License"); you may not
> +% use this file except in compliance with the License. You may obtain a copy
> of
> +% the License at
> +%
> +% http://www.apache.org/licenses/LICENSE-2.0
> +%
> +% Unless required by applicable law or agreed to in writing, software
> +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
> +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
> +% License for the specific language governing permissions and limitations
> under
> +% the License.
> +-module(couch_httpd_proxy).
> +
> +-export([handle_proxy_req/2]).
> +
> +-include("couch_db.hrl").
> +-include("../ibrowse/ibrowse.hrl").
> +
> +-define(TIMEOUT, infinity).
> +-define(PKT_SIZE, 4096).
> +
> +
> +handle_proxy_req(Req, ProxyDest) ->
> +
> + %% Bug in Mochiweb?
> + %% Reported here: http://github.com/mochi/mochiweb/issues/issue/16
> + erase(mochiweb_request_body_length),
> +
> + Method = get_method(Req),
> + Url = get_url(Req, ProxyDest),
> + Version = get_version(Req),
> + Headers = get_headers(Req),
> + Body = get_body(Req),
> + Options = [
> + {http_vsn, Version},
> + {headers_as_is, true},
> + {response_format, binary},
> + {stream_to, {self(), once}}
> + ],
> + case ibrowse:send_req(Url, Headers, Method, Body, Options, ?TIMEOUT) of
> + {ibrowse_req_id, ReqId} ->
> + stream_response(Req, ProxyDest, ReqId);
> + {error, Reason} ->
> + throw({error, Reason})
> + end.
> +
> +
> +get_method(#httpd{mochi_req=MochiReq}) ->
> + case MochiReq:get(method) of
> + Method when is_atom(Method) ->
> + list_to_atom(string:to_lower(atom_to_list(Method)));
> + Method when is_list(Method) ->
> + list_to_atom(string:to_lower(Method));
> + Method when is_binary(Method) ->
> + list_to_atom(string:to_lower(?b2l(Method)))
> + end.
> +
> +
> +get_url(Req, ProxyDest) when is_binary(ProxyDest) ->
> + get_url(Req, ?b2l(ProxyDest));
> +get_url(#httpd{mochi_req=MochiReq}=Req, ProxyDest) ->
> + BaseUrl = case mochiweb_util:partition(ProxyDest, "/") of
> + {[], "/", _} -> couch_httpd:absolute_uri(Req, ProxyDest);
> + _ -> ProxyDest
> + end,
> + ProxyPrefix = "/" ++ ?b2l(hd(Req#httpd.path_parts)),
> + RequestedPath = MochiReq:get(raw_path),
> + case mochiweb_util:partition(RequestedPath, ProxyPrefix) of
> + {[], ProxyPrefix, []} ->
> + BaseUrl;
> + {[], ProxyPrefix, [$/ | DestPath]} ->
> + remove_trailing_slash(BaseUrl) ++ "/" ++ DestPath;
> + {[], ProxyPrefix, DestPath} ->
> + remove_trailing_slash(BaseUrl) ++ "/" ++ DestPath;
> + _Else ->
> + throw({invalid_url_path, {ProxyPrefix, RequestedPath}})
> + end.
> +
> +get_version(#httpd{mochi_req=MochiReq}) ->
> + MochiReq:get(version).
> +
> +
> +get_headers(#httpd{mochi_req=MochiReq}) ->
> + to_ibrowse_headers(mochiweb_headers:to_list(MochiReq:get(headers)), []).
> +
> +to_ibrowse_headers([], Acc) ->
> + lists:reverse(Acc);
> +to_ibrowse_headers([{K, V} | Rest], Acc) when is_atom(K) ->
> + to_ibrowse_headers([{atom_to_list(K), V} | Rest], Acc);
> +to_ibrowse_headers([{K, V} | Rest], Acc) when is_list(K) ->
> + case string:to_lower(K) of
> + "content-length" ->
> + to_ibrowse_headers(Rest, [{content_length, V} | Acc]);
> + % This appears to make ibrowse too smart.
> + %"transfer-encoding" ->
> + % to_ibrowse_headers(Rest, [{transfer_encoding, V} | Acc]);
> + _ ->
> + to_ibrowse_headers(Rest, [{K, V} | Acc])
> + end.
> +
> +get_body(#httpd{method='GET'}) ->
> + fun() -> eof end;
> +get_body(#httpd{method='HEAD'}) ->
> + fun() -> eof end;
> +get_body(#httpd{method='DELETE'}) ->
> + fun() -> eof end;
> +get_body(#httpd{mochi_req=MochiReq}) ->
> + case MochiReq:get(body_length) of
> + undefined ->
> + <<>>;
> + {unknown_transfer_encoding, Unknown} ->
> + exit({unknown_transfer_encoding, Unknown});
> + chunked ->
> + {fun stream_chunked_body/1, {init, MochiReq, 0}};
> + 0 ->
> + <<>>;
> + Length when is_integer(Length) andalso Length > 0 ->
> + {fun stream_length_body/1, {init, MochiReq, Length}};
> + Length ->
> + exit({invalid_body_length, Length})
> + end.
> +
> +
> +remove_trailing_slash(Url) ->
> + rem_slash(lists:reverse(Url)).
> +
> +rem_slash([]) ->
> + [];
> +rem_slash([$\s | RevUrl]) ->
> + rem_slash(RevUrl);
> +rem_slash([$\t | RevUrl]) ->
> + rem_slash(RevUrl);
> +rem_slash([$\r | RevUrl]) ->
> + rem_slash(RevUrl);
> +rem_slash([$\n | RevUrl]) ->
> + rem_slash(RevUrl);
> +rem_slash([$/ | RevUrl]) ->
> + rem_slash(RevUrl);
> +rem_slash(RevUrl) ->
> + lists:reverse(RevUrl).
> +
> +
> +stream_chunked_body({init, MReq, 0}) ->
> + % First chunk, do expect-continue dance.
> + init_body_stream(MReq),
> + stream_chunked_body({stream, MReq, 0, [], ?PKT_SIZE});
> +stream_chunked_body({stream, MReq, 0, Buf, BRem}) ->
> + % Finished a chunk, get next length. If next length
> + % is 0, its time to try and read trailers.
> + {CRem, Data} = read_chunk_length(MReq),
> + case CRem of
> + 0 ->
> + BodyData = iolist_to_binary(lists:reverse(Buf, Data)),
> + {ok, BodyData, {trailers, MReq, [], ?PKT_SIZE}};
> + _ ->
> + stream_chunked_body(
> + {stream, MReq, CRem, [Data | Buf], BRem-size(Data)}
> + )
> + end;
> +stream_chunked_body({stream, MReq, CRem, Buf, BRem}) when BRem =< 0 ->
> + % Time to empty our buffers to the upstream socket.
> + BodyData = iolist_to_binary(lists:reverse(Buf)),
> + {ok, BodyData, {stream, MReq, CRem, [], ?PKT_SIZE}};
> +stream_chunked_body({stream, MReq, CRem, Buf, BRem}) ->
> + % Buffer some more data from the client.
> + Length = lists:min([CRem, BRem]),
> + Socket = MReq:get(socket),
> + NewState = case mochiweb_socket:recv(Socket, Length, ?TIMEOUT) of
> + {ok, Data} when size(Data) == CRem ->
> + case mochiweb_socket:recv(Socket, 2, ?TIMEOUT) of
> + {ok, <<"\r\n">>} ->
> + {stream, MReq, 0, [<<"\r\n">>, Data | Buf],
> BRem-Length-2};
> + _ ->
> + exit(normal)
> + end;
> + {ok, Data} ->
> + {stream, MReq, CRem-Length, [Data | Buf], BRem-Length};
> + _ ->
> + exit(normal)
> + end,
> + stream_chunked_body(NewState);
> +stream_chunked_body({trailers, MReq, Buf, BRem}) when BRem =< 0 ->
> + % Empty our buffers and send data upstream.
> + BodyData = iolist_to_binary(lists:reverse(Buf)),
> + {ok, BodyData, {trailers, MReq, [], ?PKT_SIZE}};
> +stream_chunked_body({trailers, MReq, Buf, BRem}) ->
> + % Read another trailer into the buffer or stop on an
> + % empty line.
> + Socket = MReq:get(socket),
> + mochiweb_socket:setopts(Socket, [{packet, line}]),
> + case mochiweb_socket:recv(Socket, 0, ?TIMEOUT) of
> + {ok, <<"\r\n">>} ->
> + mochiweb_socket:setopts(Socket, [{packet, raw}]),
> + BodyData = iolist_to_binary(lists:reverse(Buf, <<"\r\n">>)),
> + {ok, BodyData, eof};
> + {ok, Footer} ->
> + mochiweb_socket:setopts(Socket, [{packet, raw}]),
> + NewState = {trailers, MReq, [Footer | Buf], BRem-size(Footer)},
> + stream_chunked_body(NewState);
> + _ ->
> + exit(normal)
> + end;
> +stream_chunked_body(eof) ->
> + % Tell ibrowse we're done sending data.
> + eof.
> +
> +
> +stream_length_body({init, MochiReq, Length}) ->
> + % Do the expect-continue dance
> + init_body_stream(MochiReq),
> + stream_length_body({stream, MochiReq, Length});
> +stream_length_body({stream, _MochiReq, 0}) ->
> + % Finished streaming.
> + eof;
> +stream_length_body({stream, MochiReq, Length}) ->
> + BufLen = lists:min([Length, ?PKT_SIZE]),
> + case MochiReq:recv(BufLen) of
> + <<>> -> eof;
> + Bin -> {ok, Bin, {stream, MochiReq, Length-BufLen}}
> + end.
> +
> +
> +init_body_stream(MochiReq) ->
> + Expect = case MochiReq:get_header_value("expect") of
> + undefined ->
> + undefined;
> + Value when is_list(Value) ->
> + string:to_lower(Value)
> + end,
> + case Expect of
> + "100-continue" ->
> + MochiReq:start_raw_response({100, gb_trees:empty()});
> + _Else ->
> + ok
> + end.
> +
> +
> +read_chunk_length(MochiReq) ->
> + Socket = MochiReq:get(socket),
> + mochiweb_socket:setopts(Socket, [{packet, line}]),
> + case mochiweb_socket:recv(Socket, 0, ?TIMEOUT) of
> + {ok, Header} ->
> + mochiweb_socket:setopts(Socket, [{packet, raw}]),
> + Splitter = fun(C) ->
> + C =/= $\r andalso C =/= $\n andalso C =/= $\s
> + end,
> + {Hex, _Rest} = lists:splitwith(Splitter, ?b2l(Header)),
> + {mochihex:to_int(Hex), Header};
> + _ ->
> + exit(normal)
> + end.
> +
> +
> +stream_response(Req, ProxyDest, ReqId) ->
> + receive
> + {ibrowse_async_headers, ReqId, "100", _} ->
> + % ibrowse doesn't handle 100 Continue responses which
> + % means we have to discard them so the proxy client
> + % doesn't get confused.
> + ibrowse:stream_next(ReqId),
> + stream_response(Req, ProxyDest, ReqId);
> + {ibrowse_async_headers, ReqId, Status, Headers} ->
> + {Source, Dest} = get_urls(Req, ProxyDest),
> + FixedHeaders = fix_headers(Source, Dest, Headers, []),
> + case body_length(FixedHeaders) of
> + chunked ->
> + {ok, Resp} = couch_httpd:start_chunked_response(
> + Req, list_to_integer(Status), FixedHeaders
> + ),
> + ibrowse:stream_next(ReqId),
> + stream_chunked_response(Req, ReqId, Resp),
> + {ok, Resp};
> + Length when is_integer(Length) ->
> + {ok, Resp} = couch_httpd:start_response_length(
> + Req, list_to_integer(Status), FixedHeaders, Length
> + ),
> + ibrowse:stream_next(ReqId),
> + stream_length_response(Req, ReqId, Resp),
> + {ok, Resp};
> + _ ->
> + {ok, Resp} = couch_httpd:start_response(
> + Req, list_to_integer(Status), FixedHeaders
> + ),
> + ibrowse:stream_next(ReqId),
> + stream_length_response(Req, ReqId, Resp),
> + % XXX: MochiWeb apparently doesn't look at the
> + % response to see if it must force close the
> + % connection. So we help it out here.
> + erlang:put(mochiweb_request_force_close, true),
> + {ok, Resp}
> + end
> + end.
> +
> +
> +stream_chunked_response(Req, ReqId, Resp) ->
> + receive
> + {ibrowse_async_response, ReqId, Chunk} ->
> + couch_httpd:send_chunk(Resp, Chunk),
> + ibrowse:stream_next(ReqId),
> + stream_chunked_response(Req, ReqId, Resp);
> + {ibrowse_async_response, ReqId, {error, Reason}} ->
> + throw({error, Reason});
> + {ibrowse_async_response_end, ReqId} ->
> + couch_httpd:last_chunk(Resp)
> + end.
> +
> +
> +stream_length_response(Req, ReqId, Resp) ->
> + receive
> + {ibrowse_async_response, ReqId, Chunk} ->
> + couch_httpd:send(Resp, Chunk),
> + ibrowse:stream_next(ReqId),
> + stream_length_response(Req, ReqId, Resp);
> + {ibrowse_async_response, {error, Reason}} ->
> + throw({error, Reason});
> + {ibrowse_async_response_end, ReqId} ->
> + ok
> + end.
> +
> +
> +get_urls(Req, ProxyDest) ->
> + SourceUrl = couch_httpd:absolute_uri(Req, "/" ++
> hd(Req#httpd.path_parts)),
> + Source = parse_url(?b2l(iolist_to_binary(SourceUrl))),
> + case (catch parse_url(ProxyDest)) of
> + Dest when is_record(Dest, url) ->
> + {Source, Dest};
> + _ ->
> + DestUrl = couch_httpd:absolute_uri(Req, ProxyDest),
> + {Source, parse_url(DestUrl)}
> + end.
> +
> +
> +fix_headers(_, _, [], Acc) ->
> + lists:reverse(Acc);
> +fix_headers(Source, Dest, [{K, V} | Rest], Acc) ->
> + Fixed = case string:to_lower(K) of
> + "location" -> rewrite_location(Source, Dest, V);
> + "content-location" -> rewrite_location(Source, Dest, V);
> + "uri" -> rewrite_location(Source, Dest, V);
> + "destination" -> rewrite_location(Source, Dest, V);
> + "set-cookie" -> rewrite_cookie(Source, Dest, V);
> + _ -> V
> + end,
> + fix_headers(Source, Dest, Rest, [{K, Fixed} | Acc]).
> +
> +
> +rewrite_location(Source, #url{host=Host, port=Port, protocol=Proto}, Url) ->
> + case (catch parse_url(Url)) of
> + #url{host=Host, port=Port, protocol=Proto} = Location ->
> + DestLoc = #url{
> + protocol=Source#url.protocol,
> + host=Source#url.host,
> + port=Source#url.port,
> + path=join_url_path(Source#url.path, Location#url.path)
> + },
> + url_to_url(DestLoc);
> + #url{} ->
> + Url;
> + _ ->
> + url_to_url(Source#url{path=join_url_path(Source#url.path, Url)})
> + end.
> +
> +
> +rewrite_cookie(_Source, _Dest, Cookie) ->
> + Cookie.
> +
> +
> +parse_url(Url) when is_binary(Url) ->
> + ibrowse_lib:parse_url(?b2l(Url));
> +parse_url(Url) when is_list(Url) ->
> + ibrowse_lib:parse_url(?b2l(iolist_to_binary(Url))).
> +
> +
> +join_url_path(Src, Dst) ->
> + Src2 = case lists:reverse(Src) of
> + "/" ++ RestSrc -> lists:reverse(RestSrc);
> + _ -> Src
> + end,
> + Dst2 = case Dst of
> + "/" ++ RestDst -> RestDst;
> + _ -> Dst
> + end,
> + Src2 ++ "/" ++ Dst2.
> +
> +
> +url_to_url(#url{host=Host, port=Port, path=Path, protocol=Proto}) ->
> + LPort = case {Proto, Port} of
> + {http, 80} -> "";
> + {https, 443} -> "";
> + _ -> ":" ++ integer_to_list(Port)
> + end,
> + LPath = case Path of
> + "/" ++ _RestPath -> Path;
> + _ -> "/" ++ Path
> + end,
> + atom_to_list(Proto) ++ "://" ++ Host ++ LPort ++ LPath.
> +
> +
> +body_length(Headers) ->
> + case is_chunked(Headers) of
> + true -> chunked;
> + _ -> content_length(Headers)
> + end.
> +
> +
> +is_chunked([]) ->
> + false;
> +is_chunked([{K, V} | Rest]) ->
> + case string:to_lower(K) of
> + "transfer-encoding" ->
> + string:to_lower(V) == "chunked";
> + _ ->
> + is_chunked(Rest)
> + end.
> +
> +content_length([]) ->
> + undefined;
> +content_length([{K, V} | Rest]) ->
> + case string:to_lower(K) of
> + "content-length" ->
> + list_to_integer(V);
> + _ ->
> + content_length(Rest)
> + end.
> +
>
> Propchange: couchdb/trunk/test/etap/
> ------------------------------------------------------------------------------
> --- svn:ignore (original)
> +++ svn:ignore Fri Nov 5 23:26:21 2010
> @@ -3,4 +3,5 @@ Makefile
> Makefile.in
> test_util.erl
> test_util.beam
> +test_web.beam
> run
>
> Added: couchdb/trunk/test/etap/180-http-proxy.ini
> URL:
> http://svn.apache.org/viewvc/couchdb/trunk/test/etap/180-http-proxy.ini?rev=1031877&view=auto
> ==============================================================================
> --- couchdb/trunk/test/etap/180-http-proxy.ini (added)
> +++ couchdb/trunk/test/etap/180-http-proxy.ini Fri Nov 5 23:26:21 2010
> @@ -0,0 +1,20 @@
> +; Licensed to the Apache Software Foundation (ASF) under one
> +; or more contributor license agreements. See the NOTICE file
> +; distributed with this work for additional information
> +; regarding copyright ownership. The ASF licenses this file
> +; to you under the Apache License, Version 2.0 (the
> +; "License"); you may not use this file except in compliance
> +; with the License. You may obtain a copy of the License at
> +;
> +; http://www.apache.org/licenses/LICENSE-2.0
> +;
> +; Unless required by applicable law or agreed to in writing,
> +; software distributed under the License is distributed on an
> +; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> +; KIND, either express or implied. See the License for the
> +; specific language governing permissions and limitations
> +; under the License.
> +
> +[httpd_global_handlers]
> +_test = {couch_httpd_proxy, handle_proxy_req, <<"http://127.0.0.1:5985/";>>}
> +_error = {couch_httpd_proxy, handle_proxy_req, <<"http://127.0.0.1:5986/";>>}
> \ No newline at end of file
>
> Added: couchdb/trunk/test/etap/180-http-proxy.t
> URL:
> http://svn.apache.org/viewvc/couchdb/trunk/test/etap/180-http-proxy.t?rev=1031877&view=auto
> ==============================================================================
> --- couchdb/trunk/test/etap/180-http-proxy.t (added)
> +++ couchdb/trunk/test/etap/180-http-proxy.t Fri Nov 5 23:26:21 2010
> @@ -0,0 +1,357 @@
> +#!/usr/bin/env escript
> +% Licensed under the Apache License, Version 2.0 (the "License"); you may not
> +% use this file except in compliance with the License. You may obtain a copy
> of
> +% the License at
> +%
> +% http://www.apache.org/licenses/LICENSE-2.0
> +%
> +% Unless required by applicable law or agreed to in writing, software
> +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
> +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
> +% License for the specific language governing permissions and limitations
> under
> +% the License.
> +
> +-record(req, {method=get, path="", headers=[], body="", opts=[]}).
> +
> +default_config() ->
> + [
> + test_util:build_file("etc/couchdb/default_dev.ini"),
> + test_util:source_file("test/etap/180-http-proxy.ini")
> + ].
> +
> +server() -> "http://127.0.0.1:5984/_test/";.
> +proxy() -> "http://127.0.0.1:5985/";.
> +external() -> "https://www.google.com/";.
> +
> +main(_) ->
> + test_util:init_code_path(),
> +
> + etap:plan(61),
> + case (catch test()) of
> + ok ->
> + etap:end_tests();
> + Other ->
> + etap:diag("Test died abnormally: ~p", [Other]),
> + etap:bail("Bad return value.")
> + end,
> + ok.
> +
> +check_request(Name, Req, Remote, Local) ->
> + case Remote of
> + no_remote -> ok;
> + _ -> test_web:set_assert(Remote)
> + end,
> + Url = case proplists:lookup(url, Req#req.opts) of
> + none -> server() ++ Req#req.path;
> + {url, DestUrl} -> DestUrl
> + end,
> + Opts = [{headers_as_is, true} | Req#req.opts],
> + Resp =ibrowse:send_req(
> + Url, Req#req.headers, Req#req.method, Req#req.body, Opts
> + ),
> + %etap:diag("ibrowse response: ~p", [Resp]),
> + case Local of
> + no_local -> ok;
> + _ -> etap:fun_is(Local, Resp, Name)
> + end,
> + case {Remote, Local} of
> + {no_remote, _} ->
> + ok;
> + {_, no_local} ->
> + ok;
> + _ ->
> + etap:is(test_web:check_last(), was_ok, Name ++ " - request
> handled")
> + end,
> + Resp.
> +
> +test() ->
> + couch_server_sup:start_link(default_config()),
> + ibrowse:start(),
> + crypto:start(),
> + test_web:start_link(),
> +
> + test_basic(),
> + test_alternate_status(),
> + test_trailing_slash(),
> + test_passes_header(),
> + test_passes_host_header(),
> + test_passes_header_back(),
> + test_rewrites_location_headers(),
> + test_doesnt_rewrite_external_locations(),
> + test_rewrites_relative_location(),
> + test_uses_same_version(),
> + test_passes_body(),
> + test_passes_eof_body_back(),
> + test_passes_chunked_body(),
> + test_passes_chunked_body_back(),
> +
> + test_connect_error(),
> +
> + ok.
> +
> +test_basic() ->
> + Remote = fun(Req) ->
> + 'GET' = Req:get(method),
> + "/" = Req:get(path),
> + undefined = Req:get(body_length),
> + undefined = Req:recv_body(),
> + {ok, {200, [{"Content-Type", "text/plain"}], "ok"}}
> + end,
> + Local = fun({ok, "200", _, "ok"}) -> true; (_) -> false end,
> + check_request("Basic proxy test", #req{}, Remote, Local).
> +
> +test_alternate_status() ->
> + Remote = fun(Req) ->
> + "/alternate_status" = Req:get(path),
> + {ok, {201, [], "ok"}}
> + end,
> + Local = fun({ok, "201", _, "ok"}) -> true; (_) -> false end,
> + Req = #req{path="alternate_status"},
> + check_request("Alternate status", Req, Remote, Local).
> +
> +test_trailing_slash() ->
> + Remote = fun(Req) ->
> + "/trailing_slash/" = Req:get(path),
> + {ok, {200, [], "ok"}}
> + end,
> + Local = fun({ok, "200", _, "ok"}) -> true; (_) -> false end,
> + Req = #req{path="trailing_slash/"},
> + check_request("Trailing slash", Req, Remote, Local).
> +
> +test_passes_header() ->
> + Remote = fun(Req) ->
> + "/passes_header" = Req:get(path),
> + "plankton" = Req:get_header_value("X-CouchDB-Ralph"),
> + {ok, {200, [], "ok"}}
> + end,
> + Local = fun({ok, "200", _, "ok"}) -> true; (_) -> false end,
> + Req = #req{
> + path="passes_header",
> + headers=[{"X-CouchDB-Ralph", "plankton"}]
> + },
> + check_request("Passes header", Req, Remote, Local).
> +
> +test_passes_host_header() ->
> + Remote = fun(Req) ->
> + "/passes_host_header" = Req:get(path),
> + "www.google.com" = Req:get_header_value("Host"),
> + {ok, {200, [], "ok"}}
> + end,
> + Local = fun({ok, "200", _, "ok"}) -> true; (_) -> false end,
> + Req = #req{
> + path="passes_host_header",
> + headers=[{"Host", "www.google.com"}]
> + },
> + check_request("Passes host header", Req, Remote, Local).
> +
> +test_passes_header_back() ->
> + Remote = fun(Req) ->
> + "/passes_header_back" = Req:get(path),
> + {ok, {200, [{"X-CouchDB-Plankton", "ralph"}], "ok"}}
> + end,
> + Local = fun
> + ({ok, "200", Headers, "ok"}) ->
> + lists:member({"X-CouchDB-Plankton", "ralph"}, Headers);
> + (_) ->
> + false
> + end,
> + Req = #req{path="passes_header_back"},
> + check_request("Passes header back", Req, Remote, Local).
> +
> +test_rewrites_location_headers() ->
> + etap:diag("Testing location header rewrites."),
> + do_rewrite_tests([
> + {"Location", proxy() ++ "foo/bar", server() ++ "foo/bar"},
> + {"Content-Location", proxy() ++ "bing?q=2", server() ++ "bing?q=2"},
> + {"Uri", proxy() ++ "zip#frag", server() ++ "zip#frag"},
> + {"Destination", proxy(), server()}
> + ]).
> +
> +test_doesnt_rewrite_external_locations() ->
> + etap:diag("Testing no rewrite of external locations."),
> + do_rewrite_tests([
> + {"Location", external() ++ "search", external() ++ "search"},
> + {"Content-Location", external() ++ "s?q=2", external() ++ "s?q=2"},
> + {"Uri", external() ++ "f#f", external() ++ "f#f"},
> + {"Destination", external() ++ "f?q=2#f", external() ++ "f?q=2#f"}
> + ]).
> +
> +test_rewrites_relative_location() ->
> + etap:diag("Testing relative rewrites."),
> + do_rewrite_tests([
> + {"Location", "/foo", server() ++ "foo"},
> + {"Content-Location", "bar", server() ++ "bar"},
> + {"Uri", "/zing?q=3", server() ++ "zing?q=3"},
> + {"Destination", "bing?q=stuff#yay", server() ++ "bing?q=stuff#yay"}
> + ]).
> +
> +do_rewrite_tests(Tests) ->
> + lists:foreach(fun({Header, Location, Url}) ->
> + do_rewrite_test(Header, Location, Url)
> + end, Tests).
> +
> +do_rewrite_test(Header, Location, Url) ->
> + Remote = fun(Req) ->
> + "/rewrite_test" = Req:get(path),
> + {ok, {302, [{Header, Location}], "ok"}}
> + end,
> + Local = fun
> + ({ok, "302", Headers, "ok"}) ->
> + etap:is(
> + couch_util:get_value(Header, Headers),
> + Url,
> + "Header rewritten correctly."
> + ),
> + true;
> + (_) ->
> + false
> + end,
> + Req = #req{path="rewrite_test"},
> + Label = "Rewrite test for ",
> + check_request(Label ++ Header, Req, Remote, Local).
> +
> +test_uses_same_version() ->
> + Remote = fun(Req) ->
> + "/uses_same_version" = Req:get(path),
> + {1, 0} = Req:get(version),
> + {ok, {200, [], "ok"}}
> + end,
> + Local = fun({ok, "200", _, "ok"}) -> true; (_) -> false end,
> + Req = #req{
> + path="uses_same_version",
> + opts=[{http_vsn, {1, 0}}]
> + },
> + check_request("Uses same version", Req, Remote, Local).
> +
> +test_passes_body() ->
> + Remote = fun(Req) ->
> + 'PUT' = Req:get(method),
> + "/passes_body" = Req:get(path),
> + <<"Hooray!">> = Req:recv_body(),
> + {ok, {201, [], "ok"}}
> + end,
> + Local = fun({ok, "201", _, "ok"}) -> true; (_) -> false end,
> + Req = #req{
> + method=put,
> + path="passes_body",
> + body="Hooray!"
> + },
> + check_request("Passes body", Req, Remote, Local).
> +
> +test_passes_eof_body_back() ->
> + BodyChunks = [<<"foo">>, <<"bar">>, <<"bazinga">>],
> + Remote = fun(Req) ->
> + 'GET' = Req:get(method),
> + "/passes_eof_body" = Req:get(path),
> + {raw, {200, [{"Connection", "close"}], BodyChunks}}
> + end,
> + Local = fun({ok, "200", _, "foobarbazinga"}) -> true; (_) -> false end,
> + Req = #req{path="passes_eof_body"},
> + check_request("Passes eof body", Req, Remote, Local).
> +
> +test_passes_chunked_body() ->
> + BodyChunks = [<<"foo">>, <<"bar">>, <<"bazinga">>],
> + Remote = fun(Req) ->
> + 'POST' = Req:get(method),
> + "/passes_chunked_body" = Req:get(path),
> + RecvBody = fun
> + ({Length, Chunk}, [Chunk | Rest]) ->
> + Length = size(Chunk),
> + Rest;
> + ({0, []}, []) ->
> + ok
> + end,
> + ok = Req:stream_body(1024*1024, RecvBody, BodyChunks),
> + {ok, {201, [], "ok"}}
> + end,
> + Local = fun({ok, "201", _, "ok"}) -> true; (_) -> false end,
> + Req = #req{
> + method=post,
> + path="passes_chunked_body",
> + headers=[{"Transfer-Encoding", "chunked"}],
> + body=mk_chunked_body(BodyChunks)
> + },
> + check_request("Passes chunked body", Req, Remote, Local).
> +
> +test_passes_chunked_body_back() ->
> + Name = "Passes chunked body back",
> + Remote = fun(Req) ->
> + 'GET' = Req:get(method),
> + "/passes_chunked_body_back" = Req:get(path),
> + BodyChunks = [<<"foo">>, <<"bar">>, <<"bazinga">>],
> + {chunked, {200, [{"Transfer-Encoding", "chunked"}], BodyChunks}}
> + end,
> + Req = #req{
> + path="passes_chunked_body_back",
> + opts=[{stream_to, self()}]
> + },
> +
> + Resp = check_request(Name, Req, Remote, no_local),
> +
> + etap:fun_is(
> + fun({ibrowse_req_id, _}) -> true; (_) -> false end,
> + Resp,
> + "Received an ibrowse request id."
> + ),
> + {_, ReqId} = Resp,
> +
> + % Grab headers from response
> + receive
> + {ibrowse_async_headers, ReqId, "200", Headers} ->
> + etap:is(
> + proplists:get_value("Transfer-Encoding", Headers),
> + "chunked",
> + "Response included the Transfer-Encoding: chunked header"
> + ),
> + ibrowse:stream_next(ReqId)
> + after 1000 ->
> + throw({error, timeout})
> + end,
> +
> + % Check body received
> + % TODO: When we upgrade to ibrowse >= 2.0.0 this check needs to
> + % check that the chunks returned are what we sent from the
> + % Remote test.
> + etap:diag("TODO: UPGRADE IBROWSE"),
> + etap:is(recv_body(ReqId, []), <<"foobarbazinga">>, "Decoded chunked
> body."),
> +
> + % Check test_web server.
> + etap:is(test_web:check_last(), was_ok, Name ++ " - request handled").
> +
> +test_connect_error() ->
> + Local = fun({ok, "500", _Headers, _Body}) -> true; (_) -> false end,
> + Req = #req{opts=[{url, "http://127.0.0.1:5984/_error"}]},
> + check_request("Connect error", Req, no_remote, Local).
> +
> +
> +mk_chunked_body(Chunks) ->
> + mk_chunked_body(Chunks, []).
> +
> +mk_chunked_body([], Acc) ->
> + iolist_to_binary(lists:reverse(Acc, "0\r\n\r\n"));
> +mk_chunked_body([Chunk | Rest], Acc) ->
> + Size = to_hex(size(Chunk)),
> + mk_chunked_body(Rest, ["\r\n", Chunk, "\r\n", Size | Acc]).
> +
> +to_hex(Val) ->
> + to_hex(Val, []).
> +
> +to_hex(0, Acc) ->
> + Acc;
> +to_hex(Val, Acc) ->
> + to_hex(Val div 16, [hex_char(Val rem 16) | Acc]).
> +
> +hex_char(V) when V < 10 -> $0 + V;
> +hex_char(V) -> $A + V - 10.
> +
> +recv_body(ReqId, Acc) ->
> + receive
> + {ibrowse_async_response, ReqId, Data} ->
> + recv_body(ReqId, [Data | Acc]);
> + {ibrowse_async_response_end, ReqId} ->
> + iolist_to_binary(lists:reverse(Acc));
> + Else ->
> + throw({error, unexpected_mesg, Else})
> + after 5000 ->
> + throw({error, timeout})
> + end.
>
> Modified: couchdb/trunk/test/etap/Makefile.am
> URL:
> http://svn.apache.org/viewvc/couchdb/trunk/test/etap/Makefile.am?rev=1031877&r1=1031876&r2=1031877&view=diff
> ==============================================================================
> --- couchdb/trunk/test/etap/Makefile.am (original)
> +++ couchdb/trunk/test/etap/Makefile.am Fri Nov 5 23:26:21 2010
> @@ -11,7 +11,7 @@
> ## the License.
>
> noinst_SCRIPTS = run
> -noinst_DATA = test_util.beam
> +noinst_DATA = test_util.beam test_web.beam
>
> %.beam: %.erl
> $(ERLC) $<
> @@ -27,6 +27,7 @@ DISTCLEANFILES = temp.*
>
> EXTRA_DIST = \
> run.tpl \
> + test_web.erl \
> 001-load.t \
> 002-icu-driver.t \
> 010-file-basics.t \
> @@ -77,4 +78,6 @@ EXTRA_DIST = \
> 172-os-daemon-errors.4.es \
> 172-os-daemon-errors.t \
> 173-os-daemon-cfg-register.es \
> - 173-os-daemon-cfg-register.t
> + 173-os-daemon-cfg-register.t \
> + 180-http-proxy.ini \
> + 180-http-proxy.t
>
> Added: couchdb/trunk/test/etap/test_web.erl
> URL:
> http://svn.apache.org/viewvc/couchdb/trunk/test/etap/test_web.erl?rev=1031877&view=auto
> ==============================================================================
> --- couchdb/trunk/test/etap/test_web.erl (added)
> +++ couchdb/trunk/test/etap/test_web.erl Fri Nov 5 23:26:21 2010
> @@ -0,0 +1,99 @@
> +% Licensed under the Apache License, Version 2.0 (the "License"); you may not
> +% use this file except in compliance with the License. You may obtain a copy
> of
> +% the License at
> +%
> +% http://www.apache.org/licenses/LICENSE-2.0
> +%
> +% Unless required by applicable law or agreed to in writing, software
> +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
> +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
> +% License for the specific language governing permissions and limitations
> under
> +% the License.
> +
> +-module(test_web).
> +-behaviour(gen_server).
> +
> +-export([start_link/0, loop/1, get_port/0, set_assert/1, check_last/0]).
> +-export([init/1, terminate/2, code_change/3]).
> +-export([handle_call/3, handle_cast/2, handle_info/2]).
> +
> +-define(SERVER, test_web_server).
> +-define(HANDLER, test_web_handler).
> +
> +start_link() ->
> + gen_server:start({local, ?HANDLER}, ?MODULE, [], []),
> + mochiweb_http:start([
> + {name, ?SERVER},
> + {loop, {?MODULE, loop}},
> + {port, 5985}
> + ]).
> +
> +loop(Req) ->
> + %etap:diag("Handling request: ~p", [Req]),
> + case gen_server:call(?HANDLER, {check_request, Req}) of
> + {ok, RespInfo} ->
> + {ok, Req:respond(RespInfo)};
> + {raw, {Status, Headers, BodyChunks}} ->
> + Resp = Req:start_response({Status, Headers}),
> + lists:foreach(fun(C) -> Resp:send(C) end, BodyChunks),
> + erlang:put(mochiweb_request_force_close, true),
> + {ok, Resp};
> + {chunked, {Status, Headers, BodyChunks}} ->
> + Resp = Req:respond({Status, Headers, chunked}),
> + timer:sleep(500),
> + lists:foreach(fun(C) -> Resp:write_chunk(C) end, BodyChunks),
> + Resp:write_chunk([]),
> + {ok, Resp};
> + {error, Reason} ->
> + etap:diag("Error: ~p", [Reason]),
> + Body = lists:flatten(io_lib:format("Error: ~p", [Reason])),
> + {ok, Req:respond({200, [], Body})}
> + end.
> +
> +get_port() ->
> + mochiweb_socket_server:get(?SERVER, port).
> +
> +set_assert(Fun) ->
> + ok = gen_server:call(?HANDLER, {set_assert, Fun}).
> +
> +check_last() ->
> + gen_server:call(?HANDLER, last_status).
> +
> +init(_) ->
> + {ok, nil}.
> +
> +terminate(_Reason, _State) ->
> + ok.
> +
> +handle_call({check_request, Req}, _From, State) when is_function(State, 1) ->
> + Resp2 = case (catch State(Req)) of
> + {ok, Resp} -> {reply, {ok, Resp}, was_ok};
> + {raw, Resp} -> {reply, {raw, Resp}, was_ok};
> + {chunked, Resp} -> {reply, {chunked, Resp}, was_ok};
> + Error -> {reply, {error, Error}, not_ok}
> + end,
> + Req:cleanup(),
> + Resp2;
> +handle_call({check_request, _Req}, _From, _State) ->
> + {reply, {error, no_assert_function}, not_ok};
> +handle_call(last_status, _From, State) when is_atom(State) ->
> + {reply, State, nil};
> +handle_call(last_status, _From, State) ->
> + {reply, {error, not_checked}, State};
> +handle_call({set_assert, Fun}, _From, nil) ->
> + {reply, ok, Fun};
> +handle_call({set_assert, _}, _From, State) ->
> + {reply, {error, assert_function_set}, State};
> +handle_call(Msg, _From, State) ->
> + {reply, {ignored, Msg}, State}.
> +
> +handle_cast(Msg, State) ->
> + etap:diag("Ignoring cast message: ~p", [Msg]),
> + {noreply, State}.
> +
> +handle_info(Msg, State) ->
> + etap:diag("Ignoring info message: ~p", [Msg]),
> + {noreply, State}.
> +
> +code_change(_OldVsn, State, _Extra) ->
> + {ok, State}.
>
>
>
--
Filipe David Manana,
[email protected], [email protected]
"Reasonable men adapt themselves to the world.
Unreasonable men adapt the world to themselves.
That's why all progress depends on unreasonable men."
