jungerl/lib/ibrowse/src ibrowse.erl, 1.6, 1.7 ibrowse_http_client.erl, 1.17, 1.18 ibrowse_test.erl, 1.2, 1.3

[Chandru <chandrusf@us...>]

Update of /cvsroot/jungerl/jungerl/lib/ibrowse/src
In directory sc8-pr-cvs17.sourceforge.net:/tmp/cvs-serv16700/src

Modified Files:
	ibrowse.erl ibrowse_http_client.erl ibrowse_test.erl 
Log Message:
ibrowse was not reading all the options from its config file.

Index: ibrowse.erl
===================================================================
RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse.erl,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- ibrowse.erl	27 Mar 2008 01:35:50 -0000	1.6
+++ ibrowse.erl	21 May 2008 15:28:11 -0000	1.7
@@ -239,10 +239,11 @@
 		     end,
 	    Max_sessions = get_max_sessions(Host, Port, Options),
 	    Max_pipeline_size = get_max_pipeline_size(Host, Port, Options),
+	    Options_1 = merge_options(Host, Port, Options),
 	    {SSLOptions, IsSSL} =
-		case get_value(is_ssl, Options, false) of
+		case get_value(is_ssl, Options_1, false) of
 		    false -> {[], false};
-		    true -> {get_value(ssl_options, Options), true}
+		    true -> {get_value(ssl_options, Options_1), true}
 		end,
 	    case ibrowse_lb:spawn_connection(Lb_pid, Parsed_url,
 					     Max_sessions, 
@@ -250,7 +251,7 @@
 					     {SSLOptions, IsSSL}) of
 		{ok, Conn_Pid} ->
 		    do_send_req(Conn_Pid, Parsed_url, Headers,
-				Method, Body, Options, Timeout);
+				Method, Body, Options_1, Timeout);
 		Err ->
 		    Err
 	    end;
@@ -258,6 +259,18 @@
 	    {error, {url_parsing_failed, Err}}
     end.
 
+merge_options(Host, Port, Options) ->
+    Config_options = get_config_value({options, Host, Port}, []),
+    lists:foldl(
+      fun({Key, Val}, Acc) ->
+			case lists:keysearch(Key, 1, Options) of
+			    false ->
+				[{Key, Val} | Acc];
+			    _ ->
+				Acc
+			end
+      end, Options, Config_options).
+
 get_lb_pid(Url) ->
     gen_server:call(?MODULE, {get_lb_pid, Url}).
 
@@ -354,8 +367,10 @@
 %% returned by spawn_worker_process/2 or spawn_link_worker_process/2
 send_req_direct(Conn_pid, Url, Headers, Method, Body, Options, Timeout) ->
     case catch parse_url(Url) of
-	#url{} = Parsed_url ->
-	    case do_send_req(Conn_pid, Parsed_url, Headers, Method, Body, Options, Timeout) of
+	#url{host = Host,
+	     port = Port} = Parsed_url ->
+	    Options_1 = merge_options(Host, Port, Options),
+	    case do_send_req(Conn_pid, Parsed_url, Headers, Method, Body, Options_1, Timeout) of
 		{error, {'EXIT', {noproc, _}}} ->
 		    {error, worker_is_dead};
 		Ret ->

Index: ibrowse_http_client.erl
===================================================================
RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_http_client.erl,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- ibrowse_http_client.erl	27 Mar 2008 01:35:50 -0000	1.17
+++ ibrowse_http_client.erl	21 May 2008 15:28:11 -0000	1.18
@@ -1294,7 +1294,7 @@
     ok;
 shutting_down(#state{lb_ets_tid = Tid,
 		     cur_pipeline_size = Sz}) ->
-    ets:delete(Tid, {Sz, self()}).
+    catch ets:delete(Tid, {Sz, self()}).
 
 inc_pipeline_counter(#state{is_closing = true} = State) ->
     State;

Index: ibrowse_test.erl
===================================================================
RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_test.erl,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- ibrowse_test.erl	27 Mar 2008 01:35:50 -0000	1.2
+++ ibrowse_test.erl	21 May 2008 15:28:11 -0000	1.3
@@ -32,6 +32,7 @@
     Start_time = now(),
     ets:new(pid_table, [named_table, public]),
     ets:new(ibrowse_test_results, [named_table, public]),
+    ets:new(ibrowse_errors, [named_table, public, ordered_set]),
     init_results(),
     process_flag(trap_exit, true),
     log_msg("Starting spawning of workers...~n", []),
@@ -45,15 +46,20 @@
     log_msg("End time  : ~1000.p~n", [calendar:now_to_local_time(End_time)]),
     Elapsed_time_secs = trunc(timer:now_diff(End_time, Start_time) / 1000000),
     log_msg("Elapsed   : ~p~n", [Elapsed_time_secs]),
-    log_msg("Reqs/sec  : ~p~n", [(NumWorkers*NumReqsPerWorker) / Elapsed_time_secs]).
+    log_msg("Reqs/sec  : ~p~n", [(NumWorkers*NumReqsPerWorker) / Elapsed_time_secs]),
+    dump_errors().
 
 init_results() ->
     ets:insert(ibrowse_test_results, {crash, 0}),
     ets:insert(ibrowse_test_results, {send_failed, 0}),
     ets:insert(ibrowse_test_results, {other_error, 0}),
     ets:insert(ibrowse_test_results, {success, 0}),
+    ets:insert(ibrowse_test_results, {retry_later, 0}),
+    ets:insert(ibrowse_test_results, {trid_mismatch, 0}),
+    ets:insert(ibrowse_test_results, {success_no_trid, 0}),
     ets:insert(ibrowse_test_results, {failed, 0}),
-    ets:insert(ibrowse_test_results, {timeout, 0}).
+    ets:insert(ibrowse_test_results, {timeout, 0}),
+    ets:insert(ibrowse_test_results, {req_id, 0}).
 
 spawn_workers(_Url, 0, _) ->
     ok;
@@ -89,19 +95,54 @@
 do_send_req_1(_Url, 0) ->
     ets:delete(pid_table, self());
 do_send_req_1(Url, NumReqs) ->
-    case ibrowse:send_req(Url, [], get, [], [], 10000) of
-	{ok, _Status, _Headers, _Body} ->
-	    ets:update_counter(ibrowse_test_results, success, 1);
+    Counter = integer_to_list(ets:update_counter(ibrowse_test_results, req_id, 1)),
+    case ibrowse:send_req(Url, [{"ib_req_id", Counter}], get, [], [], 10000) of
+	{ok, _Status, Headers, _Body} ->
+	    case lists:keysearch("ib_req_id", 1, Headers) of
+		{value, {_, Counter}} ->
+		    ets:update_counter(ibrowse_test_results, success, 1);
+		{value, _} ->
+		    ets:update_counter(ibrowse_test_results, trid_mismatch, 1);
+		false ->
+		    ets:update_counter(ibrowse_test_results, success_no_trid, 1)
+	    end;
 	{error, req_timedout} ->
 	    ets:update_counter(ibrowse_test_results, timeout, 1);
 	{error, send_failed} ->
 	    ets:update_counter(ibrowse_test_results, send_failed, 1);
-	_Err ->
+	{error, retry_later} ->
+	    ets:update_counter(ibrowse_test_results, retry_later, 1);
+	Err ->
+	    ets:insert(ibrowse_errors, {now(), Err}),
 	    ets:update_counter(ibrowse_test_results, other_error, 1),
 	    ok
     end,
     do_send_req_1(Url, NumReqs-1).
 
+dump_errors() ->
+    case ets:info(ibrowse_errors, size) of
+	0 ->
+	    ok;
+	_ ->
+	    {A, B, C} = now(),
+	    Filename = lists:flatten(
+			 io_lib:format("ibrowse_errors_~p_~p_~p.txt" , [A, B, C])),
+	    case file:open(Filename, [write, delayed_write, raw]) of
+		{ok, Iod} ->
+		    dump_errors(ets:first(ibrowse_errors), Iod);
+		Err ->
+		    io:format("failed to create file ~s. Reason: ~p~n", [Filename, Err]),
+		    ok
+	    end
+    end.
+
+dump_errors('$end_of_table', Iod) ->
+    file:close(Iod);
+dump_errors(Key, Iod) ->
+    [{_, Term}] = ets:lookup(ibrowse_errors, Key),
+    file:write(Iod, io_lib:format("~p~n", [Term])),
+    dump_errors(ets:next(ibrowse_errors, Key), Iod).
+
 %%------------------------------------------------------------------------------
 %% Unit Tests
 %%------------------------------------------------------------------------------

jungerl/lib/ibrowse/src ibrowse.erl, 1.6, 1.7 ibrowse_http_client.erl, 1.17, 1.18 ibrowse_test.erl, 1.2, 1.3

[Chandru <chandrusf@us...>]

Update of /cvsroot/jungerl/jungerl/lib/ibrowse/src
In directory sc8-pr-cvs17.sourceforge.net:/tmp/cvs-serv16700/src

Modified Files:
	ibrowse.erl ibrowse_http_client.erl ibrowse_test.erl 
Log Message:
ibrowse was not reading all the options from its config file.

Index: ibrowse.erl
===================================================================
RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse.erl,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- ibrowse.erl	27 Mar 2008 01:35:50 -0000	1.6
+++ ibrowse.erl	21 May 2008 15:28:11 -0000	1.7
@@ -239,10 +239,11 @@
 		     end,
 	    Max_sessions = get_max_sessions(Host, Port, Options),
 	    Max_pipeline_size = get_max_pipeline_size(Host, Port, Options),
+	    Options_1 = merge_options(Host, Port, Options),
 	    {SSLOptions, IsSSL} =
-		case get_value(is_ssl, Options, false) of
+		case get_value(is_ssl, Options_1, false) of
 		    false -> {[], false};
-		    true -> {get_value(ssl_options, Options), true}
+		    true -> {get_value(ssl_options, Options_1), true}
 		end,
 	    case ibrowse_lb:spawn_connection(Lb_pid, Parsed_url,
 					     Max_sessions, 
@@ -250,7 +251,7 @@
 					     {SSLOptions, IsSSL}) of
 		{ok, Conn_Pid} ->
 		    do_send_req(Conn_Pid, Parsed_url, Headers,
-				Method, Body, Options, Timeout);
+				Method, Body, Options_1, Timeout);
 		Err ->
 		    Err
 	    end;
@@ -258,6 +259,18 @@
 	    {error, {url_parsing_failed, Err}}
     end.
 
+merge_options(Host, Port, Options) ->
+    Config_options = get_config_value({options, Host, Port}, []),
+    lists:foldl(
+      fun({Key, Val}, Acc) ->
+			case lists:keysearch(Key, 1, Options) of
+			    false ->
+				[{Key, Val} | Acc];
+			    _ ->
+				Acc
+			end
+      end, Options, Config_options).
+
 get_lb_pid(Url) ->
     gen_server:call(?MODULE, {get_lb_pid, Url}).
 
@@ -354,8 +367,10 @@
 %% returned by spawn_worker_process/2 or spawn_link_worker_process/2
 send_req_direct(Conn_pid, Url, Headers, Method, Body, Options, Timeout) ->
     case catch parse_url(Url) of
-	#url{} = Parsed_url ->
-	    case do_send_req(Conn_pid, Parsed_url, Headers, Method, Body, Options, Timeout) of
+	#url{host = Host,
+	     port = Port} = Parsed_url ->
+	    Options_1 = merge_options(Host, Port, Options),
+	    case do_send_req(Conn_pid, Parsed_url, Headers, Method, Body, Options_1, Timeout) of
 		{error, {'EXIT', {noproc, _}}} ->
 		    {error, worker_is_dead};
 		Ret ->

Index: ibrowse_http_client.erl
===================================================================
RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_http_client.erl,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- ibrowse_http_client.erl	27 Mar 2008 01:35:50 -0000	1.17
+++ ibrowse_http_client.erl	21 May 2008 15:28:11 -0000	1.18
@@ -1294,7 +1294,7 @@
     ok;
 shutting_down(#state{lb_ets_tid = Tid,
 		     cur_pipeline_size = Sz}) ->
-    ets:delete(Tid, {Sz, self()}).
+    catch ets:delete(Tid, {Sz, self()}).
 
 inc_pipeline_counter(#state{is_closing = true} = State) ->
     State;

Index: ibrowse_test.erl
===================================================================
RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_test.erl,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- ibrowse_test.erl	27 Mar 2008 01:35:50 -0000	1.2
+++ ibrowse_test.erl	21 May 2008 15:28:11 -0000	1.3
@@ -32,6 +32,7 @@
     Start_time = now(),
     ets:new(pid_table, [named_table, public]),
     ets:new(ibrowse_test_results, [named_table, public]),
+    ets:new(ibrowse_errors, [named_table, public, ordered_set]),
     init_results(),
     process_flag(trap_exit, true),
     log_msg("Starting spawning of workers...~n", []),
@@ -45,15 +46,20 @@
     log_msg("End time  : ~1000.p~n", [calendar:now_to_local_time(End_time)]),
     Elapsed_time_secs = trunc(timer:now_diff(End_time, Start_time) / 1000000),
     log_msg("Elapsed   : ~p~n", [Elapsed_time_secs]),
-    log_msg("Reqs/sec  : ~p~n", [(NumWorkers*NumReqsPerWorker) / Elapsed_time_secs]).
+    log_msg("Reqs/sec  : ~p~n", [(NumWorkers*NumReqsPerWorker) / Elapsed_time_secs]),
+    dump_errors().
 
 init_results() ->
     ets:insert(ibrowse_test_results, {crash, 0}),
     ets:insert(ibrowse_test_results, {send_failed, 0}),
     ets:insert(ibrowse_test_results, {other_error, 0}),
     ets:insert(ibrowse_test_results, {success, 0}),
+    ets:insert(ibrowse_test_results, {retry_later, 0}),
+    ets:insert(ibrowse_test_results, {trid_mismatch, 0}),
+    ets:insert(ibrowse_test_results, {success_no_trid, 0}),
     ets:insert(ibrowse_test_results, {failed, 0}),
-    ets:insert(ibrowse_test_results, {timeout, 0}).
+    ets:insert(ibrowse_test_results, {timeout, 0}),
+    ets:insert(ibrowse_test_results, {req_id, 0}).
 
 spawn_workers(_Url, 0, _) ->
     ok;
@@ -89,19 +95,54 @@
 do_send_req_1(_Url, 0) ->
     ets:delete(pid_table, self());
 do_send_req_1(Url, NumReqs) ->
-    case ibrowse:send_req(Url, [], get, [], [], 10000) of
-	{ok, _Status, _Headers, _Body} ->
-	    ets:update_counter(ibrowse_test_results, success, 1);
+    Counter = integer_to_list(ets:update_counter(ibrowse_test_results, req_id, 1)),
+    case ibrowse:send_req(Url, [{"ib_req_id", Counter}], get, [], [], 10000) of
+	{ok, _Status, Headers, _Body} ->
+	    case lists:keysearch("ib_req_id", 1, Headers) of
+		{value, {_, Counter}} ->
+		    ets:update_counter(ibrowse_test_results, success, 1);
+		{value, _} ->
+		    ets:update_counter(ibrowse_test_results, trid_mismatch, 1);
+		false ->
+		    ets:update_counter(ibrowse_test_results, success_no_trid, 1)
+	    end;
 	{error, req_timedout} ->
 	    ets:update_counter(ibrowse_test_results, timeout, 1);
 	{error, send_failed} ->
 	    ets:update_counter(ibrowse_test_results, send_failed, 1);
-	_Err ->
+	{error, retry_later} ->
+	    ets:update_counter(ibrowse_test_results, retry_later, 1);
+	Err ->
+	    ets:insert(ibrowse_errors, {now(), Err}),
 	    ets:update_counter(ibrowse_test_results, other_error, 1),
 	    ok
     end,
     do_send_req_1(Url, NumReqs-1).
 
+dump_errors() ->
+    case ets:info(ibrowse_errors, size) of
+	0 ->
+	    ok;
+	_ ->
+	    {A, B, C} = now(),
+	    Filename = lists:flatten(
+			 io_lib:format("ibrowse_errors_~p_~p_~p.txt" , [A, B, C])),
+	    case file:open(Filename, [write, delayed_write, raw]) of
+		{ok, Iod} ->
+		    dump_errors(ets:first(ibrowse_errors), Iod);
+		Err ->
+		    io:format("failed to create file ~s. Reason: ~p~n", [Filename, Err]),
+		    ok
+	    end
+    end.
+
+dump_errors('$end_of_table', Iod) ->
+    file:close(Iod);
+dump_errors(Key, Iod) ->
+    [{_, Term}] = ets:lookup(ibrowse_errors, Key),
+    file:write(Iod, io_lib:format("~p~n", [Term])),
+    dump_errors(ets:next(ibrowse_errors, Key), Iod).
+
 %%------------------------------------------------------------------------------
 %% Unit Tests
 %%------------------------------------------------------------------------------