Return-Path: X-Original-To: apmail-couchdb-commits-archive@www.apache.org Delivered-To: apmail-couchdb-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 325A311738 for ; Fri, 25 Apr 2014 12:14:54 +0000 (UTC) Received: (qmail 54432 invoked by uid 500); 25 Apr 2014 12:14:50 -0000 Delivered-To: apmail-couchdb-commits-archive@couchdb.apache.org Received: (qmail 54343 invoked by uid 500); 25 Apr 2014 12:14:47 -0000 Mailing-List: contact commits-help@couchdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@couchdb.apache.org Delivered-To: mailing list commits@couchdb.apache.org Received: (qmail 53967 invoked by uid 99); 25 Apr 2014 12:14:38 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 25 Apr 2014 12:14:38 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 521B895136F; Fri, 25 Apr 2014 12:14:37 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: rnewson@apache.org To: commits@couchdb.apache.org Date: Fri, 25 Apr 2014 12:14:42 -0000 Message-Id: <6bf782d720c74f0086aaa6d210e15194@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [6/6] ibrowse commit: updated refs/heads/master to 4af2d40 OTP shape Project: http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/repo Commit: http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/commit/4af2d408 Tree: http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/tree/4af2d408 Diff: http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/diff/4af2d408 Branch: refs/heads/master Commit: 4af2d408607874d124414ac45df1edbe3961d1cd Parents: 1167b0e Author: Robert Newson Authored: Fri Apr 25 11:41:45 2014 +0100 Committer: Robert Newson Committed: Fri Apr 25 11:41:45 2014 +0100 ---------------------------------------------------------------------- Makefile.am | 51 - ibrowse.app.in | 7 - ibrowse.erl | 929 ------------------ ibrowse.hrl | 21 - ibrowse_app.erl | 63 -- ibrowse_http_client.erl | 1945 -------------------------------------- ibrowse_lb.erl | 252 ----- ibrowse_lib.erl | 442 --------- ibrowse_socks5.erl | 109 --- ibrowse_sup.erl | 63 -- ibrowse_test.erl | 625 ------------ include/ibrowse.hrl | 21 + src/ibrowse.app.src | 7 + src/ibrowse.erl | 929 ++++++++++++++++++ src/ibrowse_app.erl | 63 ++ src/ibrowse_http_client.erl | 1945 ++++++++++++++++++++++++++++++++++++++ src/ibrowse_lb.erl | 252 +++++ src/ibrowse_lib.erl | 442 +++++++++ src/ibrowse_socks5.erl | 109 +++ src/ibrowse_sup.erl | 63 ++ src/ibrowse_test.erl | 625 ++++++++++++ 21 files changed, 4456 insertions(+), 4507 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/4af2d408/Makefile.am ---------------------------------------------------------------------- diff --git a/Makefile.am b/Makefile.am deleted file mode 100644 index 7c48169..0000000 --- a/Makefile.am +++ /dev/null @@ -1,51 +0,0 @@ -## Licensed under the Apache License, Version 2.0 (the "License"); you may not -## use this file except in compliance with the License. You may obtain a copy of -## the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -## License for the specific language governing permissions and limitations under -## the License. - -ibrowseebindir = $(localerlanglibdir)/ibrowse-2.2.0/ebin - -ibrowse_file_collection = \ - ibrowse.app.in \ - ibrowse.erl \ - ibrowse_app.erl \ - ibrowse_http_client.erl \ - ibrowse_lb.erl \ - ibrowse_lib.erl \ - ibrowse_socks5.erl \ - ibrowse_sup.erl \ - ibrowse_test.erl - -ibrowseebin_make_generated_file_list = \ - ibrowse.app \ - ibrowse.beam \ - ibrowse_app.beam \ - ibrowse_http_client.beam \ - ibrowse_lb.beam \ - ibrowse_lib.beam \ - ibrowse_socks5.beam \ - ibrowse_sup.beam \ - ibrowse_test.beam - -ibrowseebin_DATA = \ - $(ibrowseebin_make_generated_file_list) - -EXTRA_DIST = \ - $(ibrowse_file_collection) \ - ibrowse.hrl - -CLEANFILES = \ - $(ibrowseebin_make_generated_file_list) - -%.app: %.app.in - cp $< $@ - -%.beam: %.erl - $(ERLC) $(ERLC_FLAGS) $< http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/4af2d408/ibrowse.app.in ---------------------------------------------------------------------- diff --git a/ibrowse.app.in b/ibrowse.app.in deleted file mode 100644 index 1d88084..0000000 --- a/ibrowse.app.in +++ /dev/null @@ -1,7 +0,0 @@ -{application, ibrowse, - [{description, "Erlang HTTP client application"}, - {vsn, "4.0.1"}, - {registered, [ibrowse_sup, ibrowse]}, - {applications, [kernel,stdlib]}, - {env, []}, - {mod, {ibrowse_app, []}}]}. http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/4af2d408/ibrowse.erl ---------------------------------------------------------------------- diff --git a/ibrowse.erl b/ibrowse.erl deleted file mode 100644 index 80a4282..0000000 --- a/ibrowse.erl +++ /dev/null @@ -1,929 +0,0 @@ -%%%------------------------------------------------------------------- -%%% File : ibrowse.erl -%%% Author : Chandrashekhar Mullaparthi -%%% Description : Load balancer process for HTTP client connections. -%%% -%%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi -%%%------------------------------------------------------------------- -%% @author Chandrashekhar Mullaparthi -%% @copyright 2005-2012 Chandrashekhar Mullaparthi -%% @doc The ibrowse application implements an HTTP 1.1 client in erlang. This -%% module implements the API of the HTTP client. There is one named -%% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is -%% one process to handle one TCP connection to a webserver -%% (implemented in the module ibrowse_http_client). Multiple connections to a -%% webserver are setup based on the settings for each webserver. The -%% ibrowse process also determines which connection to pipeline a -%% certain request on. The functions to call are send_req/3, -%% send_req/4, send_req/5, send_req/6. -%% -%%

Here are a few sample invocations.

-%% -%% -%% ibrowse:send_req("http://intranet/messenger/", [], get). -%%

-%% -%% ibrowse:send_req("http://www.google.com/", [], get, [], -%% [{proxy_user, "XXXXX"}, -%% {proxy_password, "XXXXX"}, -%% {proxy_host, "proxy"}, -%% {proxy_port, 8080}], 1000). -%%

-%% -%%ibrowse:send_req("http://www.erlang.org/download/otp_src_R10B-3.tar.gz", [], get, [], -%% [{proxy_user, "XXXXX"}, -%% {proxy_password, "XXXXX"}, -%% {proxy_host, "proxy"}, -%% {proxy_port, 8080}, -%% {save_response_to_file, true}], 1000). -%%

-%% -%% ibrowse:send_req("http://www.erlang.org", [], head). -%% -%%

-%% ibrowse:send_req("http://www.sun.com", [], options). -%% -%%

-%% ibrowse:send_req("http://www.bbc.co.uk", [], trace). -%% -%%

-%% ibrowse:send_req("http://www.google.com", [], get, [], -%% [{stream_to, self()}]). -%% -%% - --module(ibrowse). --behaviour(gen_server). -%%-------------------------------------------------------------------- -%% Include files -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% External exports --export([start_link/0, start/0, stop/0]). - -%% gen_server callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - -%% API interface --export([ - rescan_config/0, - rescan_config/1, - add_config/1, - get_config_value/1, - get_config_value/2, - spawn_worker_process/1, - spawn_worker_process/2, - spawn_link_worker_process/1, - spawn_link_worker_process/2, - stop_worker_process/1, - send_req/3, - send_req/4, - send_req/5, - send_req/6, - send_req_direct/4, - send_req_direct/5, - send_req_direct/6, - send_req_direct/7, - stream_next/1, - stream_close/1, - set_max_sessions/3, - set_max_pipeline_size/3, - set_dest/3, - trace_on/0, - trace_off/0, - trace_on/2, - trace_off/2, - all_trace_off/0, - show_dest_status/0, - show_dest_status/1, - show_dest_status/2, - get_metrics/0, - get_metrics/2 - ]). - --ifdef(debug). --compile(export_all). --endif. - --import(ibrowse_lib, [ - parse_url/1, - get_value/3, - do_trace/2 - ]). - --record(state, {trace = false}). - --include("ibrowse.hrl"). --include_lib("stdlib/include/ms_transform.hrl"). - --define(DEF_MAX_SESSIONS,10). --define(DEF_MAX_PIPELINE_SIZE,10). - -%%==================================================================== -%% External functions -%%==================================================================== -%%-------------------------------------------------------------------- -%% Function: start_link/0 -%% Description: Starts the server -%%-------------------------------------------------------------------- -%% @doc Starts the ibrowse process linked to the calling process. Usually invoked by the supervisor ibrowse_sup -%% @spec start_link() -> {ok, pid()} -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -%% @doc Starts the ibrowse process without linking. Useful when testing using the shell -start() -> - gen_server:start({local, ?MODULE}, ?MODULE, [], [{debug, []}]). - -%% @doc Stop the ibrowse process. Useful when testing using the shell. -stop() -> - case catch gen_server:call(ibrowse, stop) of - {'EXIT',{noproc,_}} -> - ok; - Res -> - Res - end. - -%% @doc This is the basic function to send a HTTP request. -%% The Status return value indicates the HTTP status code returned by the webserver -%% @spec send_req(Url::string(), Headers::headerList(), Method::method()) -> response() -%% headerList() = [{header(), value()}] -%% header() = atom() | string() -%% value() = term() -%% method() = get | post | head | options | put | delete | trace | mkcol | propfind | proppatch | lock | unlock | move | copy -%% Status = string() -%% ResponseHeaders = [respHeader()] -%% respHeader() = {headerName(), headerValue()} -%% headerName() = string() -%% headerValue() = string() -%% response() = {ok, Status, ResponseHeaders, ResponseBody} | {ibrowse_req_id, req_id() } | {error, Reason} -%% req_id() = term() -%% ResponseBody = string() | {file, Filename} -%% Reason = term() -send_req(Url, Headers, Method) -> - send_req(Url, Headers, Method, [], []). - -%% @doc Same as send_req/3. -%% If a list is specified for the body it has to be a flat list. The body can also be a fun/0 or a fun/1.
-%% If fun/0, the connection handling process will repeatdely call the fun until it returns an error or eof. 
Fun() = {ok, Data} | eof

-%% If fun/1, the connection handling process will repeatedly call the fun with the supplied state until it returns an error or eof. 
Fun(State) = {ok, Data} | {ok, Data, NewState} | eof
-%% @spec send_req(Url, Headers, Method::method(), Body::body()) -> response() -%% body() = [] | string() | binary() | fun_arity_0() | {fun_arity_1(), initial_state()} -%% initial_state() = term() -send_req(Url, Headers, Method, Body) -> - send_req(Url, Headers, Method, Body, []). - -%% @doc Same as send_req/4. -%% For a description of SSL Options, look in the ssl manpage. If the -%% HTTP Version to use is not specified, the default is 1.1. -%%
-%%
    -%%
  • The host_header option is useful in the case where ibrowse is -%% connecting to a component such as stunnel which then sets up a -%% secure connection to a webserver. In this case, the URL supplied to -%% ibrowse must have the stunnel host/port details, but that won't -%% make sense to the destination webserver. This option can then be -%% used to specify what should go in the Host header in -%% the request.
    • -%%
  • The stream_to option can be used to have the HTTP -%% response streamed to a process as messages as data arrives on the -%% socket. If the calling process wishes to control the rate at which -%% data is received from the server, the option {stream_to, -%% {process(), once}} can be specified. The calling process -%% will have to invoke ibrowse:stream_next(Request_id) to -%% receive the next packet.
    • -%% -%%
  • When both the options save_response_to_file and stream_to -%% are specified, the former takes precedence.
    • -%% -%%
  • For the save_response_to_file option, the response body is saved to -%% file only if the status code is in the 200-299 range. If not, the response body is returned -%% as a string.
    • -%%
  • Whenever an error occurs in the processing of a request, ibrowse will return as much -%% information as it has, such as HTTP Status Code and HTTP Headers. When this happens, the response -%% is of the form {error, {Reason, {stat_code, StatusCode}, HTTP_headers}}
    • -%% -%%
  • The inactivity_timeout option is useful when -%% dealing with large response bodies and/or slow links. In these -%% cases, it might be hard to estimate how long a request will take to -%% complete. In such cases, the client might want to timeout if no -%% data has been received on the link for a certain time interval. -%% -%% This value is also used to close connections which are not in use for -%% the specified timeout value. -%%
    • -%% -%%
  • -%% The connect_timeout option is to specify how long the -%% client process should wait for connection establishment. This is -%% useful in scenarios where connections to servers are usually setup -%% very fast, but responses might take much longer compared to -%% connection setup. In such cases, it is better for the calling -%% process to timeout faster if there is a problem (DNS lookup -%% delays/failures, network routing issues, etc). The total timeout -%% value specified for the request will enforced. To illustrate using -%% an example: -%% -%% ibrowse:send_req("http://www.example.com/cgi-bin/request", [], get, [], [{connect_timeout, 100}], 1000). -%% -%% In the above invocation, if the connection isn't established within -%% 100 milliseconds, the request will fail with -%% {error, conn_failed}.
    -%% If connection setup succeeds, the total time allowed for the -%% request to complete will be 1000 milliseconds minus the time taken -%% for connection setup. -%%
    • -%% -%%
  • The socket_options option can be used to set -%% specific options on the socket. The {active, true | false | once} -%% and {packet_type, Packet_type} will be filtered out by ibrowse.
    • -%% -%%
  • The headers_as_is option is to enable the caller -%% to send headers exactly as specified in the request without ibrowse -%% adding some of its own. Required for some picky servers apparently.
    • -%% -%%
  • The give_raw_headers option is to enable the -%% caller to get access to the raw status line and raw unparsed -%% headers. Not quite sure why someone would want this, but one of my -%% users asked for it, so here it is.
    • -%% -%%
  • The preserve_chunked_encoding option enables the caller -%% to receive the raw data stream when the Transfer-Encoding of the server -%% response is Chunked. -%%
    • -%%
-%% -%% @spec send_req(Url::string(), Headers::headerList(), Method::method(), Body::body(), Options::optionList()) -> response() -%% optionList() = [option()] -%% option() = {max_sessions, integer()} | -%% {response_format,response_format()}| -%% {stream_chunk_size, integer()} | -%% {max_pipeline_size, integer()} | -%% {trace, boolean()} | -%% {is_ssl, boolean()} | -%% {ssl_options, [SSLOpt]} | -%% {pool_name, atom()} | -%% {proxy_host, string()} | -%% {proxy_port, integer()} | -%% {proxy_user, string()} | -%% {proxy_password, string()} | -%% {use_absolute_uri, boolean()} | -%% {basic_auth, {username(), password()}} | -%% {cookie, string()} | -%% {content_length, integer()} | -%% {content_type, string()} | -%% {save_response_to_file, srtf()} | -%% {stream_to, stream_to()} | -%% {http_vsn, {MajorVsn, MinorVsn}} | -%% {host_header, string()} | -%% {inactivity_timeout, integer()} | -%% {connect_timeout, integer()} | -%% {socket_options, Sock_opts} | -%% {transfer_encoding, {chunked, ChunkSize}} | -%% {headers_as_is, boolean()} | -%% {give_raw_headers, boolean()} | -%% {preserve_chunked_encoding,boolean()} | -%% {workaround, head_response_with_body} -%% -%% stream_to() = process() | {process(), once} -%% process() = pid() | atom() -%% username() = string() -%% password() = string() -%% SSLOpt = term() -%% Sock_opts = [Sock_opt] -%% Sock_opt = term() -%% ChunkSize = integer() -%% srtf() = boolean() | filename() | {append, filename()} -%% filename() = string() -%% response_format() = list | binary -send_req(Url, Headers, Method, Body, Options) -> - send_req(Url, Headers, Method, Body, Options, 30000). - -%% @doc Same as send_req/5. -%% All timeout values are in milliseconds. -%% @spec send_req(Url, Headers::headerList(), Method::method(), Body::body(), Options::optionList(), Timeout) -> response() -%% Timeout = integer() | infinity -send_req(Url, Headers, Method, Body, Options, Timeout) -> - case catch parse_url(Url) of - #url{host = Host, - port = Port, - protocol = Protocol} = Parsed_url -> - Lb_pid = case ets:lookup(ibrowse_lb, {Host, Port}) of - [] -> - get_lb_pid(Parsed_url); - [#lb_pid{pid = Lb_pid_1}] -> - Lb_pid_1 - end, - Max_sessions = get_max_sessions(Host, Port, Options), - Max_pipeline_size = get_max_pipeline_size(Host, Port, Options), - Options_1 = merge_options(Host, Port, Options), - {SSLOptions, IsSSL} = - case (Protocol == https) orelse - get_value(is_ssl, Options_1, false) of - false -> {[], false}; - true -> {get_value(ssl_options, Options_1, []), true} - end, - try_routing_request(Lb_pid, Parsed_url, - Max_sessions, - Max_pipeline_size, - {SSLOptions, IsSSL}, - Headers, Method, Body, Options_1, Timeout, 0); - Err -> - {error, {url_parsing_failed, Err}} - end. - -try_routing_request(Lb_pid, Parsed_url, - Max_sessions, - Max_pipeline_size, - {SSLOptions, IsSSL}, - Headers, Method, Body, Options_1, Timeout, Try_count) when Try_count < 3 -> - case ibrowse_lb:spawn_connection(Lb_pid, Parsed_url, - Max_sessions, - Max_pipeline_size, - {SSLOptions, IsSSL}) of - {ok, Conn_Pid} -> - case do_send_req(Conn_Pid, Parsed_url, Headers, - Method, Body, Options_1, Timeout) of - {error, sel_conn_closed} -> - try_routing_request(Lb_pid, Parsed_url, - Max_sessions, - Max_pipeline_size, - {SSLOptions, IsSSL}, - Headers, Method, Body, Options_1, Timeout, Try_count + 1); - Res -> - Res - end; - Err -> - Err - end; -try_routing_request(_, _, _, _, _, _, _, _, _, _, _) -> - {error, retry_later}. - -merge_options(Host, Port, Options) -> - Config_options = get_config_value({options, Host, Port}, []) ++ - get_config_value({options, global}, []), - lists:foldl( - fun({Key, Val}, Acc) -> - case lists:keysearch(Key, 1, Options) of - false -> - [{Key, Val} | Acc]; - _ -> - Acc - end - end, Options, Config_options). - -get_lb_pid(Url) -> - gen_server:call(?MODULE, {get_lb_pid, Url}). - -get_max_sessions(Host, Port, Options) -> - get_value(max_sessions, Options, - get_config_value({max_sessions, Host, Port}, - default_max_sessions())). - -get_max_pipeline_size(Host, Port, Options) -> - get_value(max_pipeline_size, Options, - get_config_value({max_pipeline_size, Host, Port}, - default_max_pipeline_size())). - -default_max_sessions() -> - safe_get_env(ibrowse, default_max_sessions, ?DEF_MAX_SESSIONS). - -default_max_pipeline_size() -> - safe_get_env(ibrowse, default_max_pipeline_size, ?DEF_MAX_PIPELINE_SIZE). - -safe_get_env(App, Key, Def_val) -> - case application:get_env(App, Key) of - undefined -> - Def_val; - {ok, Val} -> - Val - end. - -%% @doc Deprecated. Use set_max_sessions/3 and set_max_pipeline_size/3 -%% for achieving the same effect. -set_dest(Host, Port, [{max_sessions, Max} | T]) -> - set_max_sessions(Host, Port, Max), - set_dest(Host, Port, T); -set_dest(Host, Port, [{max_pipeline_size, Max} | T]) -> - set_max_pipeline_size(Host, Port, Max), - set_dest(Host, Port, T); -set_dest(Host, Port, [{trace, Bool} | T]) when Bool == true; Bool == false -> - ibrowse ! {trace, true, Host, Port}, - set_dest(Host, Port, T); -set_dest(_Host, _Port, [H | _]) -> - exit({invalid_option, H}); -set_dest(_, _, []) -> - ok. - -%% @doc Set the maximum number of connections allowed to a specific Host:Port. -%% @spec set_max_sessions(Host::string(), Port::integer(), Max::integer()) -> ok -set_max_sessions(Host, Port, Max) when is_integer(Max), Max > 0 -> - gen_server:call(?MODULE, {set_config_value, {max_sessions, Host, Port}, Max}). - -%% @doc Set the maximum pipeline size for each connection to a specific Host:Port. -%% @spec set_max_pipeline_size(Host::string(), Port::integer(), Max::integer()) -> ok -set_max_pipeline_size(Host, Port, Max) when is_integer(Max), Max > 0 -> - gen_server:call(?MODULE, {set_config_value, {max_pipeline_size, Host, Port}, Max}). - -do_send_req(Conn_Pid, Parsed_url, Headers, Method, Body, Options, Timeout) -> - case catch ibrowse_http_client:send_req(Conn_Pid, Parsed_url, - Headers, Method, ensure_bin(Body), - Options, Timeout) of - {'EXIT', {timeout, _}} -> - {error, req_timedout}; - {'EXIT', {noproc, {gen_server, call, [Conn_Pid, _, _]}}} -> - {error, sel_conn_closed}; - {'EXIT', {normal, _}} -> - {error, req_timedout}; - {error, connection_closed} -> - {error, sel_conn_closed}; - {'EXIT', Reason} -> - {error, {'EXIT', Reason}}; - {ok, St_code, Headers, Body} = Ret when is_binary(Body) -> - case get_value(response_format, Options, list) of - list -> - {ok, St_code, Headers, binary_to_list(Body)}; - binary -> - Ret - end; - Ret -> - Ret - end. - -ensure_bin(L) when is_list(L) -> list_to_binary(L); -ensure_bin(B) when is_binary(B) -> B; -ensure_bin(Fun) when is_function(Fun) -> Fun; -ensure_bin({Fun}) when is_function(Fun) -> Fun; -ensure_bin({Fun, _} = Body) when is_function(Fun) -> Body. - -%% @doc Creates a HTTP client process to the specified Host:Port which -%% is not part of the load balancing pool. This is useful in cases -%% where some requests to a webserver might take a long time whereas -%% some might take a very short time. To avoid getting these quick -%% requests stuck in the pipeline behind time consuming requests, use -%% this function to get a handle to a connection process.
-%% Note: Calling this function only creates a worker process. No connection -%% is setup. The connection attempt is made only when the first -%% request is sent via any of the send_req_direct/4,5,6,7 functions.
-%% Note: It is the responsibility of the calling process to control -%% pipeline size on such connections. -%% -%% @spec spawn_worker_process(Url::string()) -> {ok, pid()} -spawn_worker_process(Url) -> - ibrowse_http_client:start(Url). - -%% @doc Same as spawn_worker_process/1 but takes as input a Host and Port -%% instead of a URL. -%% @spec spawn_worker_process(Host::string(), Port::integer()) -> {ok, pid()} -spawn_worker_process(Host, Port) -> - ibrowse_http_client:start({Host, Port}). - -%% @doc Same as spawn_worker_process/1 except the the calling process -%% is linked to the worker process which is spawned. -%% @spec spawn_link_worker_process(Url::string()) -> {ok, pid()} -spawn_link_worker_process(Url) -> - ibrowse_http_client:start_link(Url). - -%% @doc Same as spawn_worker_process/2 except the the calling process -%% is linked to the worker process which is spawned. -%% @spec spawn_link_worker_process(Host::string(), Port::integer()) -> {ok, pid()} -spawn_link_worker_process(Host, Port) -> - ibrowse_http_client:start_link({Host, Port}). - -%% @doc Terminate a worker process spawned using -%% spawn_worker_process/2 or spawn_link_worker_process/2. Requests in -%% progress will get the error response 
{error, closing_on_request}
-%% @spec stop_worker_process(Conn_pid::pid()) -> ok -stop_worker_process(Conn_pid) -> - ibrowse_http_client:stop(Conn_pid). - -%% @doc Same as send_req/3 except that the first argument is the PID -%% returned by spawn_worker_process/2 or spawn_link_worker_process/2 -send_req_direct(Conn_pid, Url, Headers, Method) -> - send_req_direct(Conn_pid, Url, Headers, Method, [], []). - -%% @doc Same as send_req/4 except that the first argument is the PID -%% returned by spawn_worker_process/2 or spawn_link_worker_process/2 -send_req_direct(Conn_pid, Url, Headers, Method, Body) -> - send_req_direct(Conn_pid, Url, Headers, Method, Body, []). - -%% @doc Same as send_req/5 except that the first argument is the PID -%% returned by spawn_worker_process/2 or spawn_link_worker_process/2 -send_req_direct(Conn_pid, Url, Headers, Method, Body, Options) -> - send_req_direct(Conn_pid, Url, Headers, Method, Body, Options, 30000). - -%% @doc Same as send_req/6 except that the first argument is the PID -%% returned by spawn_worker_process/2 or spawn_link_worker_process/2 -send_req_direct(Conn_pid, Url, Headers, Method, Body, Options, Timeout) -> - case catch parse_url(Url) of - #url{host = Host, - port = Port} = Parsed_url -> - Options_1 = merge_options(Host, Port, Options), - case do_send_req(Conn_pid, Parsed_url, Headers, Method, Body, Options_1, Timeout) of - {error, {'EXIT', {noproc, _}}} -> - {error, worker_is_dead}; - Ret -> - Ret - end; - Err -> - {error, {url_parsing_failed, Err}} - end. - -%% @doc Tell ibrowse to stream the next chunk of data to the -%% caller. Should be used in conjunction with the -%% stream_to option -%% @spec stream_next(Req_id :: req_id()) -> ok | {error, unknown_req_id} -stream_next(Req_id) -> - case ets:lookup(ibrowse_stream, {req_id_pid, Req_id}) of - [] -> - {error, unknown_req_id}; - [{_, Pid}] -> - catch Pid ! {stream_next, Req_id}, - ok - end. - -%% @doc Tell ibrowse to close the connection associated with the -%% specified stream. Should be used in conjunction with the -%% stream_to option. Note that all requests in progress on -%% the connection which is serving this Req_id will be aborted, and an -%% error returned. -%% @spec stream_close(Req_id :: req_id()) -> ok | {error, unknown_req_id} -stream_close(Req_id) -> - case ets:lookup(ibrowse_stream, {req_id_pid, Req_id}) of - [] -> - {error, unknown_req_id}; - [{_, Pid}] -> - catch Pid ! {stream_close, Req_id}, - ok - end. - -%% @doc Turn tracing on for the ibrowse process -trace_on() -> - ibrowse ! {trace, true}. -%% @doc Turn tracing off for the ibrowse process -trace_off() -> - ibrowse ! {trace, false}. - -%% @doc Turn tracing on for all connections to the specified HTTP -%% server. Host is whatever is specified as the domain name in the URL -%% @spec trace_on(Host, Port) -> ok -%% Host = string() -%% Port = integer() -trace_on(Host, Port) -> - ibrowse ! {trace, true, Host, Port}, - ok. - -%% @doc Turn tracing OFF for all connections to the specified HTTP -%% server. -%% @spec trace_off(Host, Port) -> ok -trace_off(Host, Port) -> - ibrowse ! {trace, false, Host, Port}, - ok. - -%% @doc Turn Off ALL tracing -%% @spec all_trace_off() -> ok -all_trace_off() -> - ibrowse ! all_trace_off, - ok. - -%% @doc Shows some internal information about load balancing. Info -%% about workers spawned using spawn_worker_process/2 or -%% spawn_link_worker_process/2 is not included. -show_dest_status() -> - io:format("~-40.40s | ~-5.5s | ~-10.10s | ~s~n", - ["Server:port", "ETS", "Num conns", "LB Pid"]), - io:format("~80.80.=s~n", [""]), - Metrics = get_metrics(), - lists:foreach( - fun({Host, Port, Lb_pid, Tid, Size}) -> - io:format("~40.40s | ~-5.5s | ~-5.5s | ~p~n", - [Host ++ ":" ++ integer_to_list(Port), - integer_to_list(Tid), - integer_to_list(Size), - Lb_pid]) - end, Metrics). - -show_dest_status(Url) -> - #url{host = Host, port = Port} = ibrowse_lib:parse_url(Url), - show_dest_status(Host, Port). - -%% @doc Shows some internal information about load balancing to a -%% specified Host:Port. Info about workers spawned using -%% spawn_worker_process/2 or spawn_link_worker_process/2 is not -%% included. -show_dest_status(Host, Port) -> - case get_metrics(Host, Port) of - {Lb_pid, MsgQueueSize, Tid, Size, - {{First_p_sz, First_speculative_sz}, - {Last_p_sz, Last_speculative_sz}}} -> - io:format("Load Balancer Pid : ~p~n" - "LB process msg q size : ~p~n" - "LB ETS table id : ~p~n" - "Num Connections : ~p~n" - "Smallest pipeline : ~p:~p~n" - "Largest pipeline : ~p:~p~n", - [Lb_pid, MsgQueueSize, Tid, Size, - First_p_sz, First_speculative_sz, - Last_p_sz, Last_speculative_sz]); - _Err -> - io:format("Metrics not available~n", []) - end. - -get_metrics() -> - Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host), - is_integer(Port) -> - true; - (_) -> - false - end, ets:tab2list(ibrowse_lb)), - All_ets = ets:all(), - lists:map(fun({lb_pid, {Host, Port}, Lb_pid}) -> - case lists:dropwhile( - fun(Tid) -> - ets:info(Tid, owner) /= Lb_pid - end, All_ets) of - [] -> - {Host, Port, Lb_pid, unknown, 0}; - [Tid | _] -> - Size = case catch (ets:info(Tid, size)) of - N when is_integer(N) -> N; - _ -> 0 - end, - {Host, Port, Lb_pid, Tid, Size} - end - end, Dests). - -get_metrics(Host, Port) -> - case ets:lookup(ibrowse_lb, {Host, Port}) of - [] -> - no_active_processes; - [#lb_pid{pid = Lb_pid}] -> - MsgQueueSize = (catch process_info(Lb_pid, message_queue_len)), - %% {Lb_pid, MsgQueueSize, - case lists:dropwhile( - fun(Tid) -> - ets:info(Tid, owner) /= Lb_pid - end, ets:all()) of - [] -> - {Lb_pid, MsgQueueSize, unknown, 0, unknown}; - [Tid | _] -> - try - Size = ets:info(Tid, size), - case Size of - 0 -> - ok; - _ -> - First = ets:first(Tid), - Last = ets:last(Tid), - [{_, First_p_sz, First_speculative_sz}] = ets:lookup(Tid, First), - [{_, Last_p_sz, Last_speculative_sz}] = ets:lookup(Tid, Last), - {Lb_pid, MsgQueueSize, Tid, Size, - {{First_p_sz, First_speculative_sz}, {Last_p_sz, Last_speculative_sz}}} - end - catch _:_ -> - not_available - end - end - end. - -%% @doc Clear current configuration for ibrowse and load from the file -%% ibrowse.conf in the IBROWSE_EBIN/../priv directory. Current -%% configuration is cleared only if the ibrowse.conf file is readable -%% using file:consult/1 -rescan_config() -> - gen_server:call(?MODULE, rescan_config). - -%% Clear current configuration for ibrowse and load from the specified -%% file. Current configuration is cleared only if the specified -%% file is readable using file:consult/1 -rescan_config([{_,_}|_]=Terms) -> - gen_server:call(?MODULE, {rescan_config_terms, Terms}); -rescan_config(File) when is_list(File) -> - gen_server:call(?MODULE, {rescan_config, File}). - -%% @doc Add additional configuration elements at runtime. -add_config([{_,_}|_]=Terms) -> - gen_server:call(?MODULE, {add_config_terms, Terms}). - -%%==================================================================== -%% Server functions -%%==================================================================== - -%%-------------------------------------------------------------------- -%% Function: init/1 -%% Description: Initiates the server -%% Returns: {ok, State} | -%% {ok, State, Timeout} | -%% ignore | -%% {stop, Reason} -%%-------------------------------------------------------------------- -init(_) -> - process_flag(trap_exit, true), - State = #state{}, - put(my_trace_flag, State#state.trace), - put(ibrowse_trace_token, "ibrowse"), - ibrowse_lb = ets:new(ibrowse_lb, [named_table, public, {keypos, 2}]), - ibrowse_conf = ets:new(ibrowse_conf, [named_table, protected, {keypos, 2}]), - ibrowse_stream = ets:new(ibrowse_stream, [named_table, public]), - import_config(), - {ok, #state{}}. - -import_config() -> - case code:priv_dir(ibrowse) of - {error, _} -> - ok; - PrivDir -> - Filename = filename:join(PrivDir, "ibrowse.conf"), - import_config(Filename) - end. - -import_config(Filename) -> - case file:consult(Filename) of - {ok, Terms} -> - apply_config(Terms); - _Err -> - ok - end. - -apply_config(Terms) -> - ets:delete_all_objects(ibrowse_conf), - insert_config(Terms). - -insert_config(Terms) -> - Fun = fun({dest, Host, Port, MaxSess, MaxPipe, Options}) - when is_list(Host), is_integer(Port), - is_integer(MaxSess), MaxSess > 0, - is_integer(MaxPipe), MaxPipe > 0, is_list(Options) -> - I = [{{max_sessions, Host, Port}, MaxSess}, - {{max_pipeline_size, Host, Port}, MaxPipe}, - {{options, Host, Port}, Options}], - lists:foreach( - fun({X, Y}) -> - ets:insert(ibrowse_conf, - #ibrowse_conf{key = X, - value = Y}) - end, I); - ({K, V}) -> - ets:insert(ibrowse_conf, - #ibrowse_conf{key = K, - value = V}); - (X) -> - io:format("Skipping unrecognised term: ~p~n", [X]) - end, - lists:foreach(Fun, Terms). - -%% @doc Internal export -get_config_value(Key) -> - try - [#ibrowse_conf{value = V}] = ets:lookup(ibrowse_conf, Key), - V - catch - error:badarg -> - throw({error, ibrowse_not_running}) - end. - -%% @doc Internal export -get_config_value(Key, DefVal) -> - try - case ets:lookup(ibrowse_conf, Key) of - [] -> - DefVal; - [#ibrowse_conf{value = V}] -> - V - end - catch - error:badarg -> - throw({error, ibrowse_not_running}) - end. - -set_config_value(Key, Val) -> - ets:insert(ibrowse_conf, #ibrowse_conf{key = Key, value = Val}). -%%-------------------------------------------------------------------- -%% Function: handle_call/3 -%% Description: Handling call messages -%% Returns: {reply, Reply, State} | -%% {reply, Reply, State, Timeout} | -%% {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, Reply, State} | (terminate/2 is called) -%% {stop, Reason, State} (terminate/2 is called) -%%-------------------------------------------------------------------- -handle_call({get_lb_pid, #url{host = Host, port = Port} = Url}, _From, State) -> - Pid = do_get_connection(Url, ets:lookup(ibrowse_lb, {Host, Port})), - {reply, Pid, State}; - -handle_call(stop, _From, State) -> - do_trace("IBROWSE shutting down~n", []), - ets:foldl(fun(#lb_pid{pid = Pid}, Acc) -> - ibrowse_lb:stop(Pid), - Acc - end, [], ibrowse_lb), - {stop, normal, ok, State}; - -handle_call({set_config_value, Key, Val}, _From, State) -> - set_config_value(Key, Val), - {reply, ok, State}; - -handle_call(rescan_config, _From, State) -> - Ret = (catch import_config()), - {reply, Ret, State}; - -handle_call({rescan_config, File}, _From, State) -> - Ret = (catch import_config(File)), - {reply, Ret, State}; - -handle_call({rescan_config_terms, Terms}, _From, State) -> - Ret = (catch apply_config(Terms)), - {reply, Ret, State}; - -handle_call({add_config_terms, Terms}, _From, State) -> - Ret = (catch insert_config(Terms)), - {reply, Ret, State}; - -handle_call(Request, _From, State) -> - Reply = {unknown_request, Request}, - {reply, Reply, State}. - -%%-------------------------------------------------------------------- -%% Function: handle_cast/2 -%% Description: Handling cast messages -%% Returns: {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, State} (terminate/2 is called) -%%-------------------------------------------------------------------- - -handle_cast(_Msg, State) -> - {noreply, State}. - -%%-------------------------------------------------------------------- -%% Function: handle_info/2 -%% Description: Handling all non call/cast messages -%% Returns: {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, State} (terminate/2 is called) -%%-------------------------------------------------------------------- -handle_info(all_trace_off, State) -> - Mspec = [{{ibrowse_conf,{trace,'$1','$2'},true},[],[{{'$1','$2'}}]}], - Trace_on_dests = ets:select(ibrowse_conf, Mspec), - Fun = fun(#lb_pid{host_port = {H, P}, pid = Pid}, _) -> - case lists:member({H, P}, Trace_on_dests) of - false -> - ok; - true -> - catch Pid ! {trace, false} - end; - (_, Acc) -> - Acc - end, - ets:foldl(Fun, undefined, ibrowse_lb), - ets:select_delete(ibrowse_conf, [{{ibrowse_conf,{trace,'$1','$2'},true},[],['true']}]), - {noreply, State}; - -handle_info({trace, Bool}, State) -> - put(my_trace_flag, Bool), - {noreply, State}; - -handle_info({trace, Bool, Host, Port}, State) -> - Fun = fun(#lb_pid{host_port = {H, P}, pid = Pid}, _) - when H == Host, - P == Port -> - catch Pid ! {trace, Bool}; - (_, Acc) -> - Acc - end, - ets:foldl(Fun, undefined, ibrowse_lb), - ets:insert(ibrowse_conf, #ibrowse_conf{key = {trace, Host, Port}, - value = Bool}), - {noreply, State}; - -handle_info(_Info, State) -> - {noreply, State}. - -%%-------------------------------------------------------------------- -%% Function: terminate/2 -%% Description: Shutdown the server -%% Returns: any (ignored by gen_server) -%%-------------------------------------------------------------------- -terminate(_Reason, _State) -> - ok. - -%%-------------------------------------------------------------------- -%% Func: code_change/3 -%% Purpose: Convert process state when code is changed -%% Returns: {ok, NewState} -%%-------------------------------------------------------------------- -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%%% Internal functions -%%-------------------------------------------------------------------- -do_get_connection(#url{host = Host, port = Port}, []) -> - {ok, Pid} = ibrowse_lb:start_link([Host, Port]), - ets:insert(ibrowse_lb, #lb_pid{host_port = {Host, Port}, pid = Pid}), - Pid; -do_get_connection(_Url, [#lb_pid{pid = Pid}]) -> - Pid. http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/4af2d408/ibrowse.hrl ---------------------------------------------------------------------- diff --git a/ibrowse.hrl b/ibrowse.hrl deleted file mode 100644 index 18dde82..0000000 --- a/ibrowse.hrl +++ /dev/null @@ -1,21 +0,0 @@ --ifndef(IBROWSE_HRL). --define(IBROWSE_HRL, "ibrowse.hrl"). - --record(url, { - abspath, - host, - port, - username, - password, - path, - protocol, - host_type % 'hostname', 'ipv4_address' or 'ipv6_address' -}). - --record(lb_pid, {host_port, pid}). - --record(client_conn, {key, cur_pipeline_size = 0, reqs_served = 0}). - --record(ibrowse_conf, {key, value}). - --endif. http://git-wip-us.apache.org/repos/asf/couchdb-ibrowse/blob/4af2d408/ibrowse_app.erl ---------------------------------------------------------------------- diff --git a/ibrowse_app.erl b/ibrowse_app.erl deleted file mode 100644 index d3a0f7b..0000000 --- a/ibrowse_app.erl +++ /dev/null @@ -1,63 +0,0 @@ -%%%------------------------------------------------------------------- -%%% File : ibrowse_app.erl -%%% Author : Chandrashekhar Mullaparthi -%%% Description : -%%% -%%% Created : 15 Oct 2003 by Chandrashekhar Mullaparthi -%%%------------------------------------------------------------------- --module(ibrowse_app). - --behaviour(application). -%%-------------------------------------------------------------------- -%% Include files -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% External exports -%%-------------------------------------------------------------------- --export([ - start/2, - stop/1 - ]). - -%%-------------------------------------------------------------------- -%% Internal exports -%%-------------------------------------------------------------------- --export([ - ]). - -%%-------------------------------------------------------------------- -%% Macros -%%-------------------------------------------------------------------- - -%%-------------------------------------------------------------------- -%% Records -%%-------------------------------------------------------------------- - -%%==================================================================== -%% External functions -%%==================================================================== -%%-------------------------------------------------------------------- -%% Func: start/2 -%% Returns: {ok, Pid} | -%% {ok, Pid, State} | -%% {error, Reason} -%%-------------------------------------------------------------------- -start(_Type, _StartArgs) -> - case ibrowse_sup:start_link() of - {ok, Pid} -> - {ok, Pid}; - Error -> - Error - end. - -%%-------------------------------------------------------------------- -%% Func: stop/1 -%% Returns: any -%%-------------------------------------------------------------------- -stop(_State) -> - ok. - -%%==================================================================== -%% Internal functions -%%====================================================================