You can subscribe to this list here.
| 2003 |
Jan
|
Feb
(27) |
Mar
(132) |
Apr
(63) |
May
(100) |
Jun
(22) |
Jul
(7) |
Aug
(3) |
Sep
(14) |
Oct
(24) |
Nov
(49) |
Dec
(17) |
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 2004 |
Jan
(13) |
Feb
(43) |
Mar
(67) |
Apr
(21) |
May
(24) |
Jun
(3) |
Jul
(15) |
Aug
|
Sep
(1) |
Oct
(40) |
Nov
(2) |
Dec
(94) |
| 2005 |
Jan
(5) |
Feb
(36) |
Mar
(10) |
Apr
(8) |
May
(35) |
Jun
(13) |
Jul
(55) |
Aug
(82) |
Sep
(59) |
Oct
(20) |
Nov
(55) |
Dec
(15) |
| 2006 |
Jan
(40) |
Feb
(25) |
Mar
(42) |
Apr
(1) |
May
(12) |
Jun
(3) |
Jul
(2) |
Aug
(23) |
Sep
(33) |
Oct
(4) |
Nov
(5) |
Dec
(22) |
| 2007 |
Jan
(10) |
Feb
|
Mar
(4) |
Apr
(3) |
May
(8) |
Jun
(4) |
Jul
|
Aug
(3) |
Sep
(12) |
Oct
(9) |
Nov
(1) |
Dec
|
| 2008 |
Jan
|
Feb
(5) |
Mar
(4) |
Apr
|
May
(4) |
Jun
|
Jul
|
Aug
|
Sep
|
Oct
(7) |
Nov
|
Dec
|
| 2009 |
Jan
(8) |
Feb
(1) |
Mar
|
Apr
|
May
|
Jun
|
Jul
(11) |
Aug
|
Sep
(3) |
Oct
|
Nov
|
Dec
|
| 2011 |
Jan
(7) |
Feb
|
Mar
|
Apr
|
May
|
Jun
|
Jul
|
Aug
|
Sep
|
Oct
|
Nov
|
Dec
|
|
From: Serge A. <as...@us...> - 2011-01-18 20:55:23
|
Update of /cvsroot/jungerl/jungerl/lib/otp.net/OtpTest In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv15262 Removed Files: OtpTest.csproj Test.cs Log Message: Removed obsolete project replaced by OtpTest1 --- Test.cs DELETED --- --- OtpTest.csproj DELETED --- |
|
From: Serge A. <as...@us...> - 2011-01-18 20:55:23
|
Update of /cvsroot/jungerl/jungerl/lib/otp.net/OtpTest/Properties In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv15262/Properties Removed Files: AssemblyInfo.cs Log Message: Removed obsolete project replaced by OtpTest1 --- AssemblyInfo.cs DELETED --- |
|
From: Serge A. <as...@us...> - 2011-01-18 18:25:25
|
Update of /cvsroot/jungerl/jungerl/lib/otp.net/OtpTest/Properties In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv16790/Properties Added Files: AssemblyInfo.cs Log Message: Version 0.6. Added RPC cast. Bug fixes. --- NEW FILE: AssemblyInfo.cs --- (This appears to be a binary file; contents omitted.) |
|
From: Serge A. <as...@us...> - 2011-01-18 18:24:04
|
Update of /cvsroot/jungerl/jungerl/lib/otp.net/OtpTest/Properties In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv16082/Properties Removed Files: AssemblyInfo.cs Log Message: --- AssemblyInfo.cs DELETED --- |
|
From: Serge A. <as...@us...> - 2011-01-18 17:56:33
|
Update of /cvsroot/jungerl/jungerl/lib/otp.net/OtpTest/Properties In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv9798/otp.net/OtpTest/Properties Modified Files: AssemblyInfo.cs Log Message: Version 0.6. Added RPC cast. Bug fixes. Index: AssemblyInfo.cs =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/otp.net/OtpTest/Properties/AssemblyInfo.cs,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- AssemblyInfo.cs 31 Jan 2009 00:21:24 -0000 1.1 +++ AssemblyInfo.cs 18 Jan 2011 17:56:22 -0000 1.2 @@ -8,9 +8,9 @@ [assembly: AssemblyTitle("ConsoleApplication1")] [assembly: AssemblyDescription("")] [assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("Goldman Sachs")] +[assembly: AssemblyCompany("")] [assembly: AssemblyProduct("ConsoleApplication1")] -[assembly: AssemblyCopyright("Copyright © Goldman Sachs 2008")] +[assembly: AssemblyCopyright("")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] |
|
From: Serge A. <as...@us...> - 2011-01-18 17:56:33
|
Update of /cvsroot/jungerl/jungerl/lib/otp.net/Otp/Erlang In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv9798/otp.net/Otp/Erlang Modified Files: Atom.cs Log Message: Version 0.6. Added RPC cast. Bug fixes. Index: Atom.cs =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/otp.net/Otp/Erlang/Atom.cs,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- Atom.cs 8 May 2004 20:35:59 -0000 1.1 +++ Atom.cs 18 Jan 2011 17:56:22 -0000 1.2 @@ -75,7 +75,7 @@ **/ public Atom(bool t) { - this.atom = t.ToString(); + this.atom = t ? "true" : "false"; } /* |
Update of /cvsroot/jungerl/jungerl/lib/otp.net/Otp In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv9798/otp.net/Otp Modified Files: AbstractConnection.cs AbstractNode.cs Otp.2008.sln Otp.csproj OtpCookedConnection.cs OtpMbox.cs OtpNode.cs release_notes.txt Log Message: Version 0.6. Added RPC cast. Bug fixes. Index: Otp.csproj =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/otp.net/Otp/Otp.csproj,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- Otp.csproj 31 Jan 2009 00:24:09 -0000 1.2 +++ Otp.csproj 18 Jan 2011 17:56:21 -0000 1.3 @@ -1,4 +1,4 @@ -<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> +<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="3.5"> <PropertyGroup> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> @@ -9,6 +9,11 @@ <RootNamespace>otp</RootNamespace> <SignAssembly>true</SignAssembly> <AssemblyOriginatorKeyFile>dotnet_keyfile.snk</AssemblyOriginatorKeyFile> + <FileUpgradeFlags> + </FileUpgradeFlags> + <OldToolsVersion>2.0</OldToolsVersion> + <UpgradeBackupLocation> + </UpgradeBackupLocation> </PropertyGroup> <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "> <DebugSymbols>true</DebugSymbols> Index: AbstractConnection.cs =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/otp.net/Otp/AbstractConnection.cs,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- AbstractConnection.cs 31 Jan 2009 00:24:09 -0000 1.3 +++ AbstractConnection.cs 18 Jan 2011 17:56:21 -0000 1.4 @@ -72,6 +72,7 @@ protected internal OtpPeer peer; // who are we connected to protected internal OtpLocalNode self; // this nodes id internal System.String name; // local name of this connection + protected string auth_cookie; protected internal bool cookieOk = false; // already checked the cookie for this connection protected internal bool sendCookie = true; // Send cookies in messages? @@ -110,6 +111,7 @@ this.self = self; this.peer = new OtpPeer(); this.socket = s; + this.auth_cookie = self.cookie(); this.socket.NoDelay = true; // Use keepalive timer @@ -153,16 +155,27 @@ * @exception OtpAuthException if handshake resulted in an authentication error. */ protected internal AbstractConnection(OtpLocalNode self, OtpPeer other) + : this(self, other, self.cookie()) + { + } + + /* + * Intiate and open a connection to a remote node. + * + * @exception C#.io.IOException if it was not possible to connect to the peer. + * @exception OtpAuthException if handshake resulted in an authentication error. + */ + protected internal AbstractConnection(OtpLocalNode self, OtpPeer other, string cookie) { this.peer = other; this.self = self; this.socket = null; - int port; + this.auth_cookie = cookie; //this.IsBackground = true; // now get a connection between the two... - port = OtpEpmd.lookupPort(peer); + int port = OtpEpmd.lookupPort(peer); // now find highest common dist value if ((peer._proto != self._proto) || (self._distHigh < peer._distLow) || (self._distLow > peer._distHigh)) @@ -216,7 +229,7 @@ header.write_long((long)OtpMsg.Tag.regSendTag); header.write_any(from); if (sendCookie) - header.write_atom(self.cookie()); + header.write_atom(auth_cookie); else header.write_atom(""); header.write_atom(dest); @@ -256,7 +269,7 @@ header.write_tuple_head(3); header.write_long((long)OtpMsg.Tag.sendTag); if (sendCookie) - header.write_atom(self.cookie()); + header.write_atom(auth_cookie); else header.write_atom(""); header.write_any(dest); @@ -548,7 +561,7 @@ cookie = (Erlang.Atom) head.elementAt(1); if (sendCookie) { - if (!cookie.atomValue().Equals(self.cookie())) + if (!cookie.atomValue().Equals(auth_cookie)) { cookieError(self, cookie); } @@ -596,7 +609,7 @@ cookie = (Erlang.Atom) head.elementAt(2); if (sendCookie) { - if (!cookie.atomValue().Equals(self.cookie())) + if (!cookie.atomValue().Equals(auth_cookie)) { cookieError(self, cookie); } @@ -835,22 +848,63 @@ sendBuf(from, "rex", new OtpOutputStream(rpc)); } + /* + * Send an RPC cast request to the remote Erlang node. This convenience + * function creates the following message and sends it to 'rex' on + * the remote node: + * + * <pre> + * { self, { cast, Mod, Fun, Args, user }} + * </pre> + * + * No reply is delivered back + * <p> Note that this method has unpredicatble results if the remote + * node is not an Erlang node. </p> + * + * @param mod the name of the Erlang module containing the function to be called. + * @param fun the name of the function to call. + * @param args a list of Erlang terms, to be used as arguments to the function. + * + * @exception C#.io.IOException if the connection is not active + * or a communication error occurs. + **/ + public virtual void sendRPCcast(Erlang.Pid from, string mod, string fun, Erlang.List args) + { + Erlang.Object rpc = encodeRPCcast(from, mod, fun, args, new Erlang.Atom("user")); + sendBuf(from, "rex", new OtpOutputStream(rpc)); + } + internal static Erlang.Tuple encodeRPC( Erlang.Pid from, string mod, string fun, Erlang.List args, Erlang.Object gleader) { return encodeRPC(from, new Erlang.Atom(mod), new Erlang.Atom(fun), args, gleader); } + internal static Erlang.Tuple encodeRPCcast( + Erlang.Pid from, string mod, string fun, Erlang.List args, Erlang.Object gleader) + { + return encodeRPCcast(from, new Erlang.Atom(mod), new Erlang.Atom(fun), args, gleader); + } + internal static Erlang.Tuple encodeRPC( Erlang.Pid from, Erlang.Atom mod, Erlang.Atom fun, Erlang.List args, Erlang.Object gleader) { - /*{self, { call, Mod, Fun, Args, user}} */ + /*{self, { cast, Mod, Fun, Args, user}} */ return new Erlang.Tuple( from, new Erlang.Tuple(new Erlang.Atom("call"), mod, fun, args, gleader) ); } + internal static Erlang.Tuple encodeRPCcast( + Erlang.Pid from, Erlang.Atom mod, Erlang.Atom fun, Erlang.List args, Erlang.Object gleader) + { + /*{'$gen_cast', { cast, Mod, Fun, Args, user}} */ + return new Erlang.Tuple( + new Erlang.Atom("$gen_cast"), + new Erlang.Tuple(new Erlang.Atom("cast"), mod, fun, args, gleader)); + } + internal static Erlang.Object decodeRPC(Erlang.Object msg) { if (msg is Erlang.Tuple) @@ -1076,7 +1130,7 @@ int our_challenge = genChallenge(); sendChallenge(peer.distChoose, self.flags, our_challenge); int her_challenge = recvChallengeReply(our_challenge); - byte[] our_digest = genDigest(her_challenge, self.cookie()); + byte[] our_digest = genDigest(her_challenge, auth_cookie); sendChallengeAck(our_digest); connected = true; cookieOk = true; @@ -1116,7 +1170,7 @@ sendName(peer.distChoose, self.flags); recvStatus(); int her_challenge = recvChallenge(); - byte[] our_digest = genDigest(her_challenge, self.cookie()); + byte[] our_digest = genDigest(her_challenge, auth_cookie); int our_challenge = genChallenge(); sendChallengeReply(our_challenge, our_digest); recvChallengeAck(our_challenge); @@ -1428,7 +1482,7 @@ } challenge = ibuf.read4BE(); ibuf.readN(her_digest); - byte[] our_digest = genDigest(our_challenge, self.cookie()); + byte[] our_digest = genDigest(our_challenge, auth_cookie); if (!digests_equals(her_digest, our_digest)) { throw new OtpAuthException("Peer authentication error."); @@ -1477,7 +1531,7 @@ throw new System.IO.IOException("Handshake protocol error"); } ibuf.readN(her_digest); - byte[] our_digest = genDigest(our_challenge, self.cookie()); + byte[] our_digest = genDigest(our_challenge, auth_cookie); if (!digests_equals(her_digest, our_digest)) { throw new OtpAuthException("Peer authentication error."); Index: OtpNode.cs =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/otp.net/Otp/OtpNode.cs,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- OtpNode.cs 31 Jan 2009 00:24:09 -0000 1.2 +++ OtpNode.cs 18 Jan 2011 17:56:21 -0000 1.3 @@ -479,11 +479,18 @@ remoteStatus(conn.name, false, e); } + /* + * find or create a connection to the given node using specified cookie + */ + public virtual OtpCookedConnection connection(System.String node) + { + return connection(node, null); + } /* * find or create a connection to the given node */ - public virtual OtpCookedConnection connection(System.String node) + public virtual OtpCookedConnection connection(System.String node, string cookie) { OtpPeer peer = null; OtpCookedConnection conn = null; @@ -503,7 +510,7 @@ { try { - conn = new OtpCookedConnection(this, peer); + conn = new OtpCookedConnection(this, peer, cookie); addConnection(conn); } catch (System.Exception e) Index: release_notes.txt =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/otp.net/Otp/release_notes.txt,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- release_notes.txt 31 Jan 2009 00:24:09 -0000 1.3 +++ release_notes.txt 18 Jan 2011 17:56:21 -0000 1.4 @@ -1,6 +1,11 @@ Release notes ------------- +V 0.6, January 2011 / Serge Aleynikov +------------------------------------- +- Added support for RPC cast +- Implemented some bug fixes + V 0.5, January 2009 / Serge Aleynikov ------------------------------------- - More robust error handling Index: OtpMbox.cs =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/otp.net/Otp/OtpMbox.cs,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- OtpMbox.cs 31 Jan 2009 00:24:09 -0000 1.2 +++ OtpMbox.cs 18 Jan 2011 17:56:21 -0000 1.3 @@ -399,7 +399,7 @@ public void sendRPC(string node, Erlang.Atom mod, Erlang.Atom fun, Erlang.List args) { - sendRPC(node, mod, fun, args, _self /* new Erlang.Atom("user") */); + sendRPC(node, mod, fun, args, new Erlang.Atom("user")); } public void sendRPC(string node, string mod, string fun, Erlang.List args, Erlang.Pid ioServer) @@ -407,12 +407,18 @@ sendRPC(node, new Erlang.Atom(mod), new Erlang.Atom(fun), args, ioServer); } - public void sendRPC(string node, Erlang.Atom mod, Erlang.Atom fun, Erlang.List args, Erlang.Pid ioServer) + /// <summary> + /// Send RPC call to a given node. + /// </summary> + /// <param name="node"></param> + /// <param name="mod"></param> + /// <param name="fun"></param> + /// <param name="args"></param> + /// <param name="ioServer">Either a PID or an Atom containing registered I/O server's name.</param> + public void sendRPC(string node, Erlang.Atom mod, Erlang.Atom fun, Erlang.List args, Erlang.Object ioServer) { if (node.Equals(home.node())) - { throw new System.ArgumentException("Cannot make rpc calls on local node!"); - } else { Erlang.Object msg = AbstractConnection.encodeRPC(_self, mod, fun, args, ioServer ); @@ -424,6 +430,25 @@ } } + public void sendRPCcast(string node, Erlang.Atom mod, Erlang.Atom fun, Erlang.List args) + { + sendRPCcast(node, mod, fun, args, new Erlang.Atom("user")); + } + + public void sendRPCcast(string node, Erlang.Atom mod, Erlang.Atom fun, Erlang.List args, Erlang.Object ioServer) + { + if (node.Equals(home.node())) + throw new System.ArgumentException("Cannot make rpc cast on local node!"); + else + { + Erlang.Object msg = AbstractConnection.encodeRPCcast(_self, mod, fun, args, ioServer); + OtpCookedConnection conn = home.connection(node); + if (conn == null) + throw new System.Exception("Cannot establish connection to node " + node); + conn.send(_self, "rex", msg); + } + } + public Erlang.Object receiveRPC(int timeout) { Erlang.Object result = receive(timeout); Index: AbstractNode.cs =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/otp.net/Otp/AbstractNode.cs,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- AbstractNode.cs 31 Jan 2009 00:24:09 -0000 1.3 +++ AbstractNode.cs 18 Jan 2011 17:56:21 -0000 1.4 @@ -60,6 +60,13 @@ //ntype = NTYPE_R6; //flags = dFlagExtendedReferences | dFlagExtendedPidsPorts; } + + public static string CookieFileName() { + return System.Environment.GetEnvironmentVariable("HOME") + + System.IO.Path.DirectorySeparatorChar + + ".erlang.cookie"; + } + static AbstractNode() { { @@ -83,9 +90,7 @@ if (defaultCookie == null) { - System.String dotCookieFilename = System.Environment.GetEnvironmentVariable("HOME") - + System.IO.Path.DirectorySeparatorChar - + ".erlang.cookie"; + System.String dotCookieFilename = CookieFileName(); System.IO.StreamReader br = null; try { Index: OtpCookedConnection.cs =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/otp.net/Otp/OtpCookedConnection.cs,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- OtpCookedConnection.cs 31 Jan 2009 00:24:09 -0000 1.2 +++ OtpCookedConnection.cs 18 Jan 2011 17:56:21 -0000 1.3 @@ -86,7 +86,21 @@ * @exception OtpAuthException if handshake resulted in an authentication error. */ // package scope - internal OtpCookedConnection(OtpNode self, OtpPeer other):base(self, other) + internal OtpCookedConnection(OtpNode self, OtpPeer other) + : this(self, other, self.cookie()) + { + } + + /* + * Intiate and open a connection to a remote node. + * + * @exception C#.io.IOException if it was not possible to connect to the peer. + * + * @exception OtpAuthException if handshake resulted in an authentication error. + */ + // package scope + internal OtpCookedConnection(OtpNode self, OtpPeer other, string cookie) + : base(self, other, cookie) { this.self = self; this.links = new Links(25); Index: Otp.2008.sln =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/otp.net/Otp/Otp.2008.sln,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- Otp.2008.sln 31 Jan 2009 00:24:09 -0000 1.1 +++ Otp.2008.sln 18 Jan 2011 17:56:21 -0000 1.2 @@ -1,9 +1,9 @@  Microsoft Visual Studio Solution File, Format Version 10.00 # Visual Studio 2008 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Otp", "Otp.csproj", "{8155F56B-4925-406C-8619-7D0E57DD7FE1}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Otp", "Otp.csproj", "{9A16CEEB-FC26-4D35-8F9E-42DA30212F69}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OtpTest1", "..\OtpTest1\OtpTest1.csproj", "{E1AC898B-1B0C-4DFC-82A8-0D547EBDFA8A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OtpTest1", "..\OtpTest1\OtpTest1.csproj", "{51C10949-3327-4CE9-A4D0-58F1336284F4}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -11,14 +11,14 @@ Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {8155F56B-4925-406C-8619-7D0E57DD7FE1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8155F56B-4925-406C-8619-7D0E57DD7FE1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8155F56B-4925-406C-8619-7D0E57DD7FE1}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8155F56B-4925-406C-8619-7D0E57DD7FE1}.Release|Any CPU.Build.0 = Release|Any CPU - {E1AC898B-1B0C-4DFC-82A8-0D547EBDFA8A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E1AC898B-1B0C-4DFC-82A8-0D547EBDFA8A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E1AC898B-1B0C-4DFC-82A8-0D547EBDFA8A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E1AC898B-1B0C-4DFC-82A8-0D547EBDFA8A}.Release|Any CPU.Build.0 = Release|Any CPU + {9A16CEEB-FC26-4D35-8F9E-42DA30212F69}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9A16CEEB-FC26-4D35-8F9E-42DA30212F69}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9A16CEEB-FC26-4D35-8F9E-42DA30212F69}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9A16CEEB-FC26-4D35-8F9E-42DA30212F69}.Release|Any CPU.Build.0 = Release|Any CPU + {51C10949-3327-4CE9-A4D0-58F1336284F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {51C10949-3327-4CE9-A4D0-58F1336284F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {51C10949-3327-4CE9-A4D0-58F1336284F4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {51C10949-3327-4CE9-A4D0-58F1336284F4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE |
|
From: Chandru <cha...@us...> - 2009-09-06 20:04:14
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv6216 Modified Files: README vsn.mk Log Message: Added option to allow caller to specify socket options Index: README =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/README,v retrieving revision 1.19 retrieving revision 1.20 diff -u -d -r1.19 -r1.20 --- README 29 Jul 2009 18:03:21 -0000 1.19 +++ README 6 Sep 2009 20:04:02 -0000 1.20 @@ -18,12 +18,14 @@ Comments to : Cha...@gm... -Version : 1.5.2 +Version : 1.5.3 Latest version : git://github.com/cmullaparthi/ibrowse.git CONTRIBUTIONS & CHANGE HISTORY ============================== +05-09-2009 - * Introduced option to allow caller to set socket options. + 29-07-2009 - * The ETS table created for load balancing of requests was not being deleted which led to the node not being able to create any more ETS tables if queries were made to many number of Index: vsn.mk =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/vsn.mk,v retrieving revision 1.17 retrieving revision 1.18 diff -u -d -r1.17 -r1.18 --- vsn.mk 29 Jul 2009 18:03:21 -0000 1.17 +++ vsn.mk 6 Sep 2009 20:04:02 -0000 1.18 @@ -1,2 +1,2 @@ -IBROWSE_VSN = 1.5.2 +IBROWSE_VSN = 1.5.3 |
|
From: Chandru <cha...@us...> - 2009-09-06 20:04:14
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse/doc In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv6216/doc Modified Files: ibrowse.html Log Message: Added option to allow caller to specify socket options Index: ibrowse.html =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/doc/ibrowse.html,v retrieving revision 1.10 retrieving revision 1.11 diff -u -d -r1.10 -r1.11 --- ibrowse.html 29 Jul 2009 18:03:21 -0000 1.10 +++ ibrowse.html 6 Sep 2009 20:04:02 -0000 1.11 @@ -203,12 +203,14 @@ <div class="spec"> <p><tt>send_req(Url::string(), Headers::<a href="#type-headerList">headerList()</a>, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>, Options::<a href="#type-optionList">optionList()</a>) -> <a href="#type-response">response()</a></tt> <ul class="definitions"><li><tt><a name="type-optionList">optionList()</a> = [<a href="#type-option">option()</a>]</tt></li> -<li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {response_format, <a href="#type-response_format">response_format()</a>} | {stream_chunk_size, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>} | {is_ssl, <a href="#type-boolean">boolean()</a>} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, <a href="#type-boolean">boolean()</a>} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-srtf">srtf()</a>} | {stream_to, <a href="#type-stream_to">stream_to()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {inactivity_timeout, integer()} | {connect_timeout, integer()} | {transfer_encoding, {chunked, ChunkSize}}</tt></li> +<li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {response_format, <a href="#type-response_format">response_format()</a>} | {stream_chunk_size, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>} | {is_ssl, <a href="#type-boolean">boolean()</a>} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, <a href="#type-boolean">boolean()</a>} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-srtf">srtf()</a>} | {stream_to, <a href="#type-stream_to">stream_to()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {inactivity_timeout, integer()} | {connect_timeout, integer()} | {socket_options, Sock_opts} | {transfer_encoding, {chunked, ChunkSize}}</tt></li> <li><tt><a name="type-stream_to">stream_to()</a> = <a href="#type-process">process()</a> | {<a href="#type-process">process()</a>, once}</tt></li> <li><tt><a name="type-process">process()</a> = pid() | atom()</tt></li> <li><tt><a name="type-username">username()</a> = string()</tt></li> <li><tt><a name="type-password">password()</a> = string()</tt></li> <li><tt>SSLOpt = term()</tt></li> +<li><tt>Sock_opts = [Sock_opt]</tt></li> +<li><tt>Sock_opt = term()</tt></li> <li><tt>ChunkSize = integer()</tt></li> <li><tt><a name="type-srtf">srtf()</a> = <a href="#type-boolean">boolean()</a> | <a href="#type-filename">filename()</a></tt></li> <li><tt><a name="type-filename">filename()</a> = string()</tt></li> @@ -271,6 +273,10 @@ for connection setup. </li> </ul> + + <li> The <code>socket_options</code> option can be used to set + specific options on the socket. The <code>{active, true | false | once}</code> + and <code>{packet_type, Packet_type}</code> will be filtered out by ibrowse. </li> </p> <h3 class="function"><a name="send_req-6">send_req/6</a></h3> @@ -417,6 +423,6 @@ <hr> <div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div> -<p><i>Generated by EDoc, Jul 29 2009, 18:43:30.</i></p> +<p><i>Generated by EDoc, Sep 5 2009, 23:59:48.</i></p> </body> </html> |
|
From: Chandru <cha...@us...> - 2009-09-06 20:04:11
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse/src In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv6216/src Modified Files: ibrowse.erl ibrowse_http_client.erl Log Message: Added option to allow caller to specify socket options Index: ibrowse.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse.erl,v retrieving revision 1.10 retrieving revision 1.11 diff -u -d -r1.10 -r1.11 --- ibrowse.erl 8 Jul 2009 11:05:45 -0000 1.10 +++ ibrowse.erl 6 Sep 2009 20:04:02 -0000 1.11 @@ -7,7 +7,7 @@ %%%------------------------------------------------------------------- %% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com> %% @copyright 2005-2009 Chandrashekhar Mullaparthi -%% @version 1.5.1 +%% @version 1.5.2 %% @doc The ibrowse application implements an HTTP 1.1 client. This %% module implements the API of the HTTP client. There is one named %% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is @@ -98,6 +98,7 @@ trace_on/2, trace_off/2, all_trace_off/0, + show_dest_status/0, show_dest_status/2 ]). @@ -227,6 +228,10 @@ %% </li> %% </ul> %% +%% <li> The <code>socket_options</code> option can be used to set +%% specific options on the socket. The <code>{active, true | false | once}</code> +%% and <code>{packet_type, Packet_type}</code> will be filtered out by ibrowse. </li> +%% %% @spec send_req(Url::string(), Headers::headerList(), Method::method(), Body::body(), Options::optionList()) -> response() %% optionList() = [option()] %% option() = {max_sessions, integer()} | @@ -252,6 +257,7 @@ %% {host_header, string()} | %% {inactivity_timeout, integer()} | %% {connect_timeout, integer()} | +%% {socket_options, Sock_opts} | %% {transfer_encoding, {chunked, ChunkSize}} %% %% stream_to() = process() | {process(), once} @@ -259,6 +265,8 @@ %% username() = string() %% password() = string() %% SSLOpt = term() +%% Sock_opts = [Sock_opt] +%% Sock_opt = term() %% ChunkSize = integer() %% srtf() = boolean() | filename() %% filename() = string() @@ -480,6 +488,44 @@ ibrowse ! all_trace_off, ok. +show_dest_status() -> + Dests = lists:filter(fun({lb_pid, {Host, Port}, _}) when is_list(Host), + is_integer(Port) -> + true; + (_) -> + false + end, ets:tab2list(ibrowse_lb)), + All_ets = ets:all(), + io:format("~-40.40s | ~-5.5s | ~-10.10s | ~s~n", + ["Server:port", "ETS", "Num conns", "LB Pid"]), + io:format("~80.80.=s~n", [""]), + lists:foreach(fun({lb_pid, {Host, Port}, Lb_pid}) -> + case lists:dropwhile( + fun(Tid) -> + ets:info(Tid, owner) /= Lb_pid + end, All_ets) of + [] -> + io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n", + [Host ++ ":" ++ integer_to_list(Port), + "", + "", + io_lib:format("~p", [Lb_pid])] + ); + [Tid | _] -> + catch ( + begin + Size = ets:info(Tid, size), + io:format("~40.40s | ~-5.5s | ~-5.5s | ~s~n", + [Host ++ ":" ++ integer_to_list(Port), + integer_to_list(Tid), + integer_to_list(Size), + io_lib:format("~p", [Lb_pid])] + ) + end + ) + end + end, Dests). + %% @doc Shows some internal information about load balancing to a %% specified Host:Port. Info about workers spawned using %% spawn_worker_process/2 or spawn_link_worker_process/2 is not Index: ibrowse_http_client.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_http_client.erl,v retrieving revision 1.21 retrieving revision 1.22 diff -u -d -r1.21 -r1.22 --- ibrowse_http_client.erl 29 Jul 2009 18:03:21 -0000 1.21 +++ ibrowse_http_client.erl 6 Sep 2009 20:04:02 -0000 1.22 @@ -52,6 +52,7 @@ -record(request, {url, method, options, from, stream_to, caller_controls_socket = false, + caller_socket_options = [], req_id, stream_chunk_size, save_response_to_file = false, @@ -412,15 +413,31 @@ State end. -do_connect(Host, Port, _Options, #state{is_ssl=true, ssl_options=SSLOptions}, Timeout) -> +do_connect(Host, Port, Options, #state{is_ssl=true, ssl_options=SSLOptions}, Timeout) -> + Caller_socket_options = get_value(socket_options, Options, []), + Other_sock_options = filter_sock_options(SSLOptions ++ Caller_socket_options), ssl:connect(Host, Port, - [binary, {nodelay, true}, {active, false} | SSLOptions], + [binary, {nodelay, true}, {active, false} | Other_sock_options], Timeout); -do_connect(Host, Port, _Options, _State, Timeout) -> +do_connect(Host, Port, Options, _State, Timeout) -> + Caller_socket_options = get_value(socket_options, Options, []), + Other_sock_options = filter_sock_options(Caller_socket_options), gen_tcp:connect(Host, Port, - [binary, {nodelay, true}, {active, false}], + [binary, {nodelay, true}, {active, false} | Other_sock_options], Timeout). +%% We don't want the caller to specify certain options +filter_sock_options(Opts) -> + lists:filter(fun({active, _}) -> + false; + ({packet, _}) -> + false; + (list) -> + false; + (_) -> + true + end, Opts). + do_send(Req, #state{socket = Sock, is_ssl = true}) -> ssl:send(Sock, Req); do_send(Req, #state{socket = Sock, is_ssl = false}) -> gen_tcp:send(Sock, Req). @@ -461,6 +478,7 @@ active_once(#state{socket = Socket, is_ssl = Is_ssl}) -> do_setopts(Socket, [{active, once}], Is_ssl). +do_setopts(_Sock, [], _) -> ok; do_setopts(Sock, Opts, true) -> ssl:setopts(Sock, Opts); do_setopts(Sock, Opts, false) -> inet:setopts(Sock, Opts). @@ -517,9 +535,12 @@ port = Port, path = RelPath} = Url, Headers, Method, Body, Options, Timeout, - #state{status = Status} = State) -> + #state{status = Status, + socket = Socket, + is_ssl = Is_ssl} = State) -> ReqId = make_req_id(), Resp_format = get_value(response_format, Options, list), + Caller_socket_options = get_value(socket_options, Options, []), {StreamTo, Caller_controls_socket} = case get_value(stream_to, Options, undefined) of {Caller, once} when is_pid(Caller) or @@ -540,6 +561,7 @@ method = Method, stream_to = StreamTo, caller_controls_socket = Caller_controls_socket, + caller_socket_options = Caller_socket_options, options = Options, req_id = ReqId, save_response_to_file = SaveResponseToFile, @@ -547,7 +569,7 @@ response_format = Resp_format, from = From}, State_1 = State#state{reqs=queue:in(NewReq, State#state.reqs)}, - Headers_1 = add_auth_headers(Url, Options, Headers, State), + Headers_1 = add_auth_headers(Url, Options, Headers, State_1), HostHeaderValue = case lists:keysearch(host_header, 1, Options) of false -> case Port of @@ -559,7 +581,7 @@ end, {Req, Body_1} = make_request(Method, [{"Host", HostHeaderValue} | Headers_1], - AbsPath, RelPath, Body, Options, State#state.use_proxy), + AbsPath, RelPath, Body, Options, State_1#state.use_proxy), case get(my_trace_flag) of true -> %%Avoid the binary operations if trace is not on... @@ -569,12 +591,13 @@ "--- Request End ---~n", [NReq]); _ -> ok end, - case do_send(Req, State) of + do_setopts(Socket, Caller_socket_options, Is_ssl), + case do_send(Req, State_1) of ok -> - case do_send_body(Body_1, State) of + case do_send_body(Body_1, State_1) of ok -> State_2 = inc_pipeline_counter(State_1), - active_once(State_1), + active_once(State_2), Ref = case Timeout of infinity -> undefined; |
|
From: Chandru <cha...@us...> - 2009-07-29 18:03:32
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv3680 Modified Files: README vsn.mk Log Message: 29-07-2009 - * The ETS table created for load balancing of requests was not being deleted which led to the node not being able to create any more ETS tables if queries were made to many number of webservers. ibrowse now deletes the ETS table it creates once the last connection to a webserver is dropped. Reported by Seth Falcon. * Spurious data being returned at end of body in certain cases of chunked encoded responses from the server. Reported by Chris Newcombe. Index: README =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/README,v retrieving revision 1.18 retrieving revision 1.19 diff -u -d -r1.18 -r1.19 --- README 7 Jul 2009 22:30:57 -0000 1.18 +++ README 29 Jul 2009 18:03:21 -0000 1.19 @@ -18,12 +18,22 @@ Comments to : Cha...@gm... -Version : 1.5.1 +Version : 1.5.2 Latest version : git://github.com/cmullaparthi/ibrowse.git CONTRIBUTIONS & CHANGE HISTORY ============================== +29-07-2009 - * The ETS table created for load balancing of requests was not + being deleted which led to the node not being able to create + any more ETS tables if queries were made to many number of + webservers. ibrowse now deletes the ETS table it creates once the + last connection to a webserver is dropped. + Reported by Seth Falcon. + * Spurious data being returned at end of body in certain cases of + chunked encoded responses from the server. + Reported by Chris Newcombe. + 03-07-2009 - Added option {stream_to, {Pid, once}} which allows the caller to control when it wants to receive more data. If this option is used, the call ibrowse:stream_next(Req_id) should be used Index: vsn.mk =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/vsn.mk,v retrieving revision 1.16 retrieving revision 1.17 diff -u -d -r1.16 -r1.17 --- vsn.mk 7 Jul 2009 22:30:57 -0000 1.16 +++ vsn.mk 29 Jul 2009 18:03:21 -0000 1.17 @@ -1,2 +1,2 @@ -IBROWSE_VSN = 1.5.1 +IBROWSE_VSN = 1.5.2 |
|
From: Chandru <cha...@us...> - 2009-07-29 18:03:30
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse/doc In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv3680/doc Modified Files: ibrowse.html Log Message: 29-07-2009 - * The ETS table created for load balancing of requests was not being deleted which led to the node not being able to create any more ETS tables if queries were made to many number of webservers. ibrowse now deletes the ETS table it creates once the last connection to a webserver is dropped. Reported by Seth Falcon. * Spurious data being returned at end of body in certain cases of chunked encoded responses from the server. Reported by Chris Newcombe. Index: ibrowse.html =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/doc/ibrowse.html,v retrieving revision 1.9 retrieving revision 1.10 diff -u -d -r1.9 -r1.10 --- ibrowse.html 7 Jul 2009 22:30:58 -0000 1.9 +++ ibrowse.html 29 Jul 2009 18:03:21 -0000 1.10 @@ -12,7 +12,7 @@ <ul class="index"><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>The ibrowse application implements an HTTP 1.1 client. <p>Copyright © 2005-2009 Chandrashekhar Mullaparthi</p> -<p><b>Version:</b> 1.5.1</p> +<p><b>Version:</b> 1.5.2</p> <p><b>Behaviours:</b> <a href="gen_server.html"><tt>gen_server</tt></a>.</p> <p><b>Authors:</b> Chandrashekhar Mullaparthi (<a href="mailto:chandrashekhar dot mullaparthi at gmail dot com"><tt>chandrashekhar dot mullaparthi at gmail dot com</tt></a>).</p> @@ -90,6 +90,7 @@ <tr><td valign="top"><a href="#set_dest-3">set_dest/3</a></td><td>Deprecated.</td></tr> <tr><td valign="top"><a href="#set_max_pipeline_size-3">set_max_pipeline_size/3</a></td><td>Set the maximum pipeline size for each connection to a specific Host:Port.</td></tr> <tr><td valign="top"><a href="#set_max_sessions-3">set_max_sessions/3</a></td><td>Set the maximum number of connections allowed to a specific Host:Port.</td></tr> +<tr><td valign="top"><a href="#show_dest_status-0">show_dest_status/0</a></td><td></td></tr> <tr><td valign="top"><a href="#show_dest_status-2">show_dest_status/2</a></td><td>Shows some internal information about load balancing to a specified Host:Port.</td></tr> <tr><td valign="top"><a href="#spawn_link_worker_process-2">spawn_link_worker_process/2</a></td><td>Same as spawn_worker_process/2 except the the calling process @@ -320,6 +321,11 @@ <p><tt>set_max_sessions(Host::string(), Port::integer(), Max::integer()) -> ok</tt></p> </div><p>Set the maximum number of connections allowed to a specific Host:Port.</p> +<h3 class="function"><a name="show_dest_status-0">show_dest_status/0</a></h3> +<div class="spec"> +<p><tt>show_dest_status() -> any()</tt></p> +</div> + <h3 class="function"><a name="show_dest_status-2">show_dest_status/2</a></h3> <div class="spec"> <p><tt>show_dest_status(Host, Port) -> any()</tt></p> @@ -411,6 +417,6 @@ <hr> <div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div> -<p><i>Generated by EDoc, Jul 7 2009, 23:13:24.</i></p> +<p><i>Generated by EDoc, Jul 29 2009, 18:43:30.</i></p> </body> </html> |
|
From: Chandru <cha...@us...> - 2009-07-29 18:03:29
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse/src In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv3680/src Modified Files: ibrowse_http_client.erl ibrowse_lb.erl ibrowse_test.erl Log Message: 29-07-2009 - * The ETS table created for load balancing of requests was not being deleted which led to the node not being able to create any more ETS tables if queries were made to many number of webservers. ibrowse now deletes the ETS table it creates once the last connection to a webserver is dropped. Reported by Seth Falcon. * Spurious data being returned at end of body in certain cases of chunked encoded responses from the server. Reported by Chris Newcombe. Index: ibrowse_lb.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_lb.erl,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- ibrowse_lb.erl 7 Jul 2009 22:30:58 -0000 1.3 +++ ibrowse_lb.erl 29 Jul 2009 18:03:21 -0000 1.4 @@ -108,18 +108,19 @@ %% Update max_sessions in #state with supplied value handle_call({spawn_connection, _Url, Max_sess, Max_pipe, _}, _From, - #state{ets_tid = Tid, - num_cur_sessions = Num} = State) + #state{num_cur_sessions = Num} = State) when Num >= Max_sess -> - Reply = find_best_connection(Tid, Max_pipe), - {reply, Reply, State#state{max_sessions = Max_sess}}; + State_1 = maybe_create_ets(State), + Reply = find_best_connection(State_1#state.ets_tid, Max_pipe), + {reply, Reply, State_1#state{max_sessions = Max_sess}}; handle_call({spawn_connection, Url, _Max_sess, _Max_pipe, SSL_options}, _From, - #state{num_cur_sessions = Cur, - ets_tid = Tid} = State) -> + #state{num_cur_sessions = Cur} = State) -> + State_1 = maybe_create_ets(State), + Tid = State_1#state.ets_tid, {ok, Pid} = ibrowse_http_client:start_link({Tid, Url, SSL_options}), ets:insert(Tid, {{1, Pid}, []}), - {reply, {ok, Pid}, State#state{num_cur_sessions = Cur + 1}}; + {reply, {ok, Pid}, State_1#state{num_cur_sessions = Cur + 1}}; handle_call(Request, _From, State) -> Reply = {unknown_request, Request}, @@ -145,11 +146,26 @@ handle_info({'EXIT', Parent, _Reason}, #state{parent_pid = Parent} = State) -> {stop, normal, State}; +handle_info({'EXIT', _Pid, _Reason}, #state{ets_tid = undefined} = State) -> + {noreply, State}; + handle_info({'EXIT', Pid, _Reason}, #state{num_cur_sessions = Cur, ets_tid = Tid} = State) -> ets:match_delete(Tid, {{'_', Pid}, '_'}), - {noreply, State#state{num_cur_sessions = Cur - 1}}; + Cur_1 = Cur - 1, + State_1 = case Cur_1 of + 0 -> + ets:delete(Tid), + State#state{ets_tid = undefined}; + _ -> + State + end, + {noreply, State_1#state{num_cur_sessions = Cur_1}}; + +handle_info({trace, Bool}, #state{ets_tid = undefined} = State) -> + put(my_trace_flag, Bool), + {noreply, State}; handle_info({trace, Bool}, #state{ets_tid = Tid} = State) -> ets:foldl(fun({{_, Pid}, _}, Acc) when is_pid(Pid) -> @@ -192,3 +208,9 @@ _ -> {error, retry_later} end. + +maybe_create_ets(#state{ets_tid = undefined} = State) -> + Tid = ets:new(ibrowse_lb, [public, ordered_set]), + State#state{ets_tid = Tid}; +maybe_create_ets(State) -> + State. Index: ibrowse_http_client.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_http_client.erl,v retrieving revision 1.20 retrieving revision 1.21 diff -u -d -r1.20 -r1.21 --- ibrowse_http_client.erl 7 Jul 2009 22:30:58 -0000 1.20 +++ ibrowse_http_client.erl 29 Jul 2009 18:03:21 -0000 1.21 @@ -137,7 +137,7 @@ handle_call(stop, _From, State) -> do_close(State), do_error_reply(State, closing_on_request), - {stop, normal, State}; + {stop, normal, ok, State}; handle_call(Request, _From, State) -> Reply = {unknown_request, Request}, @@ -184,6 +184,15 @@ handle_sock_closed(State), {stop, normal, State}; +handle_info({tcp_error, _Sock}, State) -> + io:format("Error on connection to ~1000.p:~1000.p~n", [State#state.host, State#state.port]), + handle_sock_closed(State), + {stop, normal, State}; +handle_info({ssl_error, _Sock}, State) -> + io:format("Error on SSL connection to ~1000.p:~1000.p~n", [State#state.host, State#state.port]), + handle_sock_closed(State), + {stop, normal, State}; + handle_info({req_timedout, From}, State) -> case lists:keysearch(From, #request.from, queue:to_list(State#state.reqs)) of false -> @@ -204,6 +213,8 @@ {noreply, State}; handle_info(Info, State) -> + io:format("Unknown message recvd for ~1000.p:~1000.p -> ~p~n", + [State#state.host, State#state.port, Info]), io:format("Recvd unknown message ~p when in state: ~p~n", [Info, State]), {noreply, State}. @@ -388,7 +399,7 @@ case TmpFilename of undefined -> do_reply(State, From, StreamTo, ReqId, Resp_format, - {ok, SC, Headers, lists:reverse(Buf)}); + {ok, SC, Headers, Buf}); _ -> file:close(Fd), do_reply(State, From, StreamTo, ReqId, Resp_format, @@ -869,8 +880,8 @@ %% This clause determines the chunk size when given data from the beginning of the chunk parse_11_response(DataRecvd, - #state{transfer_encoding=chunked, - chunk_size=chunk_start, + #state{transfer_encoding = chunked, + chunk_size = chunk_start, chunk_size_buffer = Chunk_sz_buf } = State) -> case scan_crlf(Chunk_sz_buf, DataRecvd) of @@ -906,20 +917,20 @@ {yes, _, NextChunk} -> State_1 = State#state{chunk_size = chunk_start, chunk_size_buffer = <<>>, -%% reply_buffer = Buf_1, deleted_crlf = true}, parse_11_response(NextChunk, State_1); {no, Data_1} -> -%% State#state{reply_buffer = Data_1, rep_buf_size = size(Data_1)} State#state{chunk_size_buffer = Data_1} end; -%% This clause deals with the end of a chunked transfer +%% This clause deals with the end of a chunked transfer. ibrowse does +%% not support Trailers in the Chunked Transfer encoding. Any trailer +%% received is silently discarded. parse_11_response(DataRecvd, #state{transfer_encoding = chunked, chunk_size = 0, cur_req = CurReq, deleted_crlf = DelCrlf, - reply_buffer = Trailer, reqs = Reqs}=State) -> + chunk_size_buffer = Trailer, reqs = Reqs}=State) -> do_trace("Detected end of chunked transfer...~n", []), DataRecvd_1 = case DelCrlf of false -> @@ -933,7 +944,7 @@ State_1 = handle_response(CurReq, State#state{reqs = Reqs_1}), parse_response(Rem, reset_state(State_1)); {no, Rem} -> - State#state{reply_buffer = Rem, rep_buf_size = size(Rem), deleted_crlf = false} + State#state{chunk_size_buffer = Rem, deleted_crlf = false} end; %% This clause extracts a chunk, given the size. Index: ibrowse_test.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_test.erl,v retrieving revision 1.5 retrieving revision 1.6 diff -u -d -r1.5 -r1.6 --- ibrowse_test.erl 7 Jul 2009 22:30:58 -0000 1.5 +++ ibrowse_test.erl 29 Jul 2009 18:03:21 -0000 1.6 @@ -231,6 +231,7 @@ {'DOWN', Ref, _, _, Info} -> io:format("Test process crashed: ~p~n", [Info]) after 60000 -> + exit(Pid, kill), io:format("Timed out waiting for tests to complete~n", []) end. @@ -301,6 +302,9 @@ receive {async_result, Pid, Res} -> Res; + {async_result, Other_pid, _} -> + io:format("~p: Waiting for result from ~p: got from ~p~n", [self(), Pid, Other_pid]), + wait_for_resp(Pid); {'DOWN', _, _, Pid, Reason} -> {'EXIT', Reason}; {'DOWN', _, _, _, _} -> |
|
From: Chandru <cha...@us...> - 2009-07-08 11:05:52
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse/src In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv9808 Modified Files: ibrowse.erl Log Message: Fix to allow functions to be passed as body. Thanks to http://github.com/benoitc Index: ibrowse.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse.erl,v retrieving revision 1.9 retrieving revision 1.10 diff -u -d -r1.9 -r1.10 --- ibrowse.erl 7 Jul 2009 22:30:58 -0000 1.9 +++ ibrowse.erl 8 Jul 2009 11:05:45 -0000 1.10 @@ -372,10 +372,11 @@ Ret end. -ensure_bin(L) when is_list(L) -> - list_to_binary(L); -ensure_bin(B) when is_binary(B) -> - B. +ensure_bin(L) when is_list(L) -> list_to_binary(L); +ensure_bin(B) when is_binary(B) -> B; +ensure_bin(Fun) when is_function(Fun) -> Fun; +ensure_bin({Fun}) when is_function(Fun) -> Fun; +ensure_bin({Fun, _} = Body) when is_function(Fun) -> Body. %% @doc Creates a HTTP client process to the specified Host:Port which %% is not part of the load balancing pool. This is useful in cases |
|
From: Chandru <cha...@us...> - 2009-07-07 22:31:04
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse/src In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv7720/src Modified Files: ibrowse.erl ibrowse_http_client.erl ibrowse_lb.erl ibrowse_lib.erl ibrowse_test.erl Log Message: Added option {stream_to, {process(), once}} to allow calling process to control data rate on socket Index: ibrowse.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse.erl,v retrieving revision 1.8 retrieving revision 1.9 diff -u -d -r1.8 -r1.9 --- ibrowse.erl 1 Jul 2009 22:43:19 -0000 1.8 +++ ibrowse.erl 7 Jul 2009 22:30:58 -0000 1.9 @@ -7,7 +7,7 @@ %%%------------------------------------------------------------------- %% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com> %% @copyright 2005-2009 Chandrashekhar Mullaparthi -%% @version 1.5.0 +%% @version 1.5.1 %% @doc The ibrowse application implements an HTTP 1.1 client. This %% module implements the API of the HTTP client. There is one named %% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is @@ -89,6 +89,7 @@ send_req_direct/5, send_req_direct/6, send_req_direct/7, + stream_next/1, set_max_sessions/3, set_max_pipeline_size/3, set_dest/3, @@ -150,7 +151,8 @@ %% respHeader() = {headerName(), headerValue()} %% headerName() = string() %% headerValue() = string() -%% response() = {ok, Status, ResponseHeaders, ResponseBody} | {error, Reason} +%% response() = {ok, Status, ResponseHeaders, ResponseBody} | {ibrowse_req_id, req_id() } | {error, Reason} +%% req_id() = term() %% ResponseBody = string() | {file, Filename} %% Reason = term() send_req(Url, Headers, Method) -> @@ -167,7 +169,7 @@ send_req(Url, Headers, Method, Body, []). %% @doc Same as send_req/4. -%% For a description of SSL Options, look in the ssl manpage. If the +%% For a description of SSL Options, look in the <a href="http://www.erlang.org/doc/apps/ssl/index.html">ssl</a> manpage. If the %% HTTP Version to use is not specified, the default is 1.1. %% <br/> %% <p>The <code>host_header</code> option is useful in the case where ibrowse is @@ -179,6 +181,14 @@ %% used to specify what should go in the <code>Host</code> header in %% the request.</p> %% <ul> +%% <li>The <code>stream_to</code> option can be used to have the HTTP +%% response streamed to a process as messages as data arrives on the +%% socket. If the calling process wishes to control the rate at which +%% data is received from the server, the option <code>{stream_to, +%% {process(), once}}</code> can be specified. The calling process +%% will have to invoke <code>ibrowse:stream_next(Request_id)</code> to +%% receive the next packet.</li> +%% %% <li>When both the options <code>save_response_to_file</code> and <code>stream_to</code> %% are specified, the former takes precedence.</li> %% @@ -237,13 +247,14 @@ %% {content_length, integer()} | %% {content_type, string()} | %% {save_response_to_file, srtf()} | -%% {stream_to, process()} | +%% {stream_to, stream_to()} | %% {http_vsn, {MajorVsn, MinorVsn}} | %% {host_header, string()} | %% {inactivity_timeout, integer()} | %% {connect_timeout, integer()} | %% {transfer_encoding, {chunked, ChunkSize}} %% +%% stream_to() = process() | {process(), once} %% process() = pid() | atom() %% username() = string() %% password() = string() @@ -425,7 +436,20 @@ Err -> {error, {url_parsing_failed, Err}} end. - + +%% @doc Tell ibrowse to stream the next chunk of data to the +%% caller. Should be used in conjunction with the +%% <code>stream_to</code> option +%% @spec stream_next(Req_id :: req_id()) -> ok | {error, unknown_req_id} +stream_next(Req_id) -> + case ets:lookup(ibrowse_stream, {req_id_pid, Req_id}) of + [] -> + {error, unknown_req_id}; + [{_, Pid}] -> + catch Pid ! {stream_next, Req_id}, + ok + end. + %% @doc Turn tracing on for the ibrowse process trace_on() -> ibrowse ! {trace, true}. @@ -522,6 +546,7 @@ put(ibrowse_trace_token, "ibrowse"), ets:new(ibrowse_lb, [named_table, public, {keypos, 2}]), ets:new(ibrowse_conf, [named_table, protected, {keypos, 2}]), + ets:new(ibrowse_stream, [named_table, public]), import_config(), {ok, #state{}}. @@ -539,9 +564,9 @@ {ok, Terms} -> ets:delete_all_objects(ibrowse_conf), Fun = fun({dest, Host, Port, MaxSess, MaxPipe, Options}) - when list(Host), integer(Port), - integer(MaxSess), MaxSess > 0, - integer(MaxPipe), MaxPipe > 0, list(Options) -> + when is_list(Host), is_integer(Port), + is_integer(MaxSess), MaxSess > 0, + is_integer(MaxPipe), MaxPipe > 0, is_list(Options) -> I = [{{max_sessions, Host, Port}, MaxSess}, {{max_pipeline_size, Host, Port}, MaxPipe}, {{options, Host, Port}, Options}], @@ -641,13 +666,6 @@ true -> catch Pid ! {trace, false} end; - (#client_conn{key = {H, P, Pid}}, _) -> - case lists:member({H, P}, Trace_on_dests) of - false -> - ok; - true -> - catch Pid ! {trace, false} - end; (_, Acc) -> Acc end, @@ -664,10 +682,6 @@ when H == Host, P == Port -> catch Pid ! {trace, Bool}; - (#client_conn{key = {H, P, Pid}}, _) - when H == Host, - P == Port -> - catch Pid ! {trace, Bool}; (_, Acc) -> Acc end, Index: ibrowse_lb.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_lb.erl,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- ibrowse_lb.erl 1 Jul 2009 22:43:19 -0000 1.2 +++ ibrowse_lb.erl 7 Jul 2009 22:30:58 -0000 1.3 @@ -151,7 +151,13 @@ ets:match_delete(Tid, {{'_', Pid}, '_'}), {noreply, State#state{num_cur_sessions = Cur - 1}}; -handle_info({trace, Bool}, State) -> +handle_info({trace, Bool}, #state{ets_tid = Tid} = State) -> + ets:foldl(fun({{_, Pid}, _}, Acc) when is_pid(Pid) -> + catch Pid ! {trace, Bool}, + Acc; + (_, Acc) -> + Acc + end, undefined, Tid), put(my_trace_flag, Bool), {noreply, State}; Index: ibrowse_lib.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_lib.erl,v retrieving revision 1.6 retrieving revision 1.7 diff -u -d -r1.6 -r1.7 --- ibrowse_lib.erl 27 Mar 2008 01:35:50 -0000 1.6 +++ ibrowse_lib.erl 7 Jul 2009 22:30:58 -0000 1.7 @@ -49,7 +49,7 @@ %% @spec url_encode(Str) -> UrlEncodedStr %% Str = string() %% UrlEncodedStr = string() -url_encode(Str) when list(Str) -> +url_encode(Str) when is_list(Str) -> url_encode_char(lists:reverse(Str), []). url_encode_char([X | T], Acc) when X >= $0, X =< $9 -> @@ -70,7 +70,7 @@ d2h(N) when N<10 -> N+$0; d2h(N) -> N+$a-10. -decode_rfc822_date(String) when list(String) -> +decode_rfc822_date(String) when is_list(String) -> case catch decode_rfc822_date_1(string:tokens(String, ", \t\r\n")) of {'EXIT', _} -> {error, invalid_date}; @@ -177,9 +177,9 @@ %% @spec encode_base64(In) -> Out %% In = string() | binary() %% Out = string() | binary() -encode_base64(List) when list(List) -> +encode_base64(List) when is_list(List) -> encode_base64_1(list_to_binary(List)); -encode_base64(Bin) when binary(Bin) -> +encode_base64(Bin) when is_binary(Bin) -> List = encode_base64_1(Bin), list_to_binary(List). @@ -197,9 +197,9 @@ %% @spec decode_base64(In) -> Out | exit({error, invalid_input}) %% In = string() | binary() %% Out = string() | binary() -decode_base64(List) when list(List) -> +decode_base64(List) when is_list(List) -> decode_base64_1(List, []); -decode_base64(Bin) when binary(Bin) -> +decode_base64(Bin) when is_binary(Bin) -> List = decode_base64_1(binary_to_list(Bin), []), list_to_binary(List). Index: ibrowse_http_client.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_http_client.erl,v retrieving revision 1.19 retrieving revision 1.20 diff -u -d -r1.19 -r1.20 --- ibrowse_http_client.erl 1 Jul 2009 22:43:19 -0000 1.19 +++ ibrowse_http_client.erl 7 Jul 2009 22:30:58 -0000 1.20 @@ -47,11 +47,12 @@ is_closing, send_timer, content_length, deleted_crlf = false, transfer_encoding, chunk_size, chunk_size_buffer = <<>>, recvd_chunk_size, - lb_ets_tid, cur_pipeline_size = 0 + lb_ets_tid, cur_pipeline_size = 0, prev_req_id }). -record(request, {url, method, options, from, - stream_to, req_id, + stream_to, caller_controls_socket = false, + req_id, stream_chunk_size, save_response_to_file = false, tmp_file_name, tmp_file_fd, @@ -126,144 +127,15 @@ %%-------------------------------------------------------------------- %% Received a request when the remote server has already sent us a %% Connection: Close header -handle_call({send_req, _}, - _From, - #state{is_closing=true}=State) -> +handle_call({send_req, _}, _From, #state{is_closing = true} = State) -> {reply, {error, connection_closing}, State}; handle_call({send_req, {Url, Headers, Method, Body, Options, Timeout}}, - From, - #state{socket=undefined, - host=Host, port=Port}=State) -> - Resp_format = get_value(response_format, Options, list), - {Host_1, Port_1, State_1} = - case get_value(proxy_host, Options, false) of - false -> - {Host, Port, State}; - PHost -> - ProxyUser = get_value(proxy_user, Options, []), - ProxyPassword = get_value(proxy_password, Options, []), - Digest = http_auth_digest(ProxyUser, ProxyPassword), - {PHost, get_value(proxy_port, Options, 80), - State#state{use_proxy = true, - proxy_auth_digest = Digest}} - end, - StreamTo = get_value(stream_to, Options, undefined), - ReqId = make_req_id(), - SaveResponseToFile = get_value(save_response_to_file, Options, false), - NewReq = #request{url=Url, - method=Method, - stream_to=StreamTo, - options=Options, - req_id=ReqId, - save_response_to_file = SaveResponseToFile, - stream_chunk_size = get_stream_chunk_size(Options), - response_format = Resp_format, - from=From}, - Reqs = queue:in(NewReq, State#state.reqs), - State_2 = check_ssl_options(Options, State_1#state{reqs = Reqs}), - do_trace("Connecting...~n", []), - Start_ts = now(), - Conn_timeout = get_value(connect_timeout, Options, Timeout), - case do_connect(Host_1, Port_1, Options, State_2, Conn_timeout) of - {ok, Sock} -> - do_trace("Connected!~n", []), - End_ts = now(), - Ref = case Timeout of - infinity -> - undefined; - _ -> - Rem_time = Timeout - trunc(round(timer:now_diff(End_ts, Start_ts) / 1000)), - case Rem_time > 0 of - true -> - erlang:send_after(Rem_time, self(), {req_timedout, From}); - false -> - shutting_down(State_2), - do_error_reply(State_2, req_timedout), - exit(normal) - end - end, - case send_req_1(Url, Headers, Method, Body, Options, Sock, State_2) of - ok -> - do_setopts(Sock, [{active, once}], State_2#state.is_ssl), - case StreamTo of - undefined -> - ok; - _ -> - gen_server:reply(From, {ibrowse_req_id, ReqId}) - end, - State_3 = inc_pipeline_counter(State_2#state{socket = Sock, - send_timer = Ref, - cur_req = NewReq, - status = get_header}), - {noreply, State_3, get_inac_timeout(State_3)}; - Err -> - shutting_down(State_2), - do_trace("Send failed... Reason: ~p~n", [Err]), - gen_server:reply(From, {error, send_failed}), - {stop, normal, State_2} - end; - Err -> - shutting_down(State_2), - do_trace("Error connecting. Reason: ~1000.p~n", [Err]), - gen_server:reply(From, {error, conn_failed}), - {stop, normal, State_2} - end; - -%% Request which is to be pipelined -handle_call({send_req, {Url, Headers, Method, - Body, Options, Timeout}}, - From, - #state{socket=Sock, status=Status, reqs=Reqs}=State) -> - do_trace("Recvd request in connected state. Status -> ~p NumPending: ~p~n", [Status, length(queue:to_list(Reqs))]), - Resp_format = get_value(response_format, Options, list), - StreamTo = get_value(stream_to, Options, undefined), - SaveResponseToFile = get_value(save_response_to_file, Options, false), - ReqId = make_req_id(), - NewReq = #request{url=Url, - stream_to=StreamTo, - method=Method, - options=Options, - req_id=ReqId, - save_response_to_file = SaveResponseToFile, - stream_chunk_size = get_stream_chunk_size(Options), - response_format = Resp_format, - from=From}, - State_1 = State#state{reqs=queue:in(NewReq, State#state.reqs)}, - case send_req_1(Url, Headers, Method, Body, Options, Sock, State_1) of - ok -> - State_2 = inc_pipeline_counter(State_1), - do_setopts(Sock, [{active, once}], State#state.is_ssl), - case Timeout of - infinity -> - ok; - _ -> - erlang:send_after(Timeout, self(), {req_timedout, From}) - end, - State_3 = case Status of - idle -> - State_2#state{status = get_header, - cur_req = NewReq}; - _ -> - State_2 - end, - case StreamTo of - undefined -> - ok; - _ -> - gen_server:reply(From, {ibrowse_req_id, ReqId}) - end, - {noreply, State_3, get_inac_timeout(State_3)}; - Err -> - shutting_down(State_1), - do_trace("Send request failed: Reason: ~p~n", [Err]), - gen_server:reply(From, {error, send_failed}), - do_error_reply(State, send_failed), - {stop, normal, State_1} - end; + From, State) -> + send_req_1(From, Url, Headers, Method, Body, Options, Timeout, State); -handle_call(stop, _From, #state{socket = Socket, is_ssl = Is_ssl} = State) -> - do_close(Socket, Is_ssl), +handle_call(stop, _From, State) -> + do_close(State), do_error_reply(State, closing_on_request), {stop, normal, State}; @@ -294,6 +166,15 @@ handle_info({ssl, _Sock, Data}, State) -> handle_sock_data(Data, State); +handle_info({stream_next, Req_id}, #state{socket = Socket, + is_ssl = Is_ssl, + cur_req = #request{req_id = Req_id}} = State) -> + do_setopts(Socket, [{active, once}], Is_ssl), + {noreply, State}; + +handle_info({stream_next, _Req_id}, State) -> + {noreply, State}; + handle_info({tcp_closed, _Sock}, State) -> do_trace("TCP connection closed by peer!~n", []), handle_sock_closed(State), @@ -332,12 +213,7 @@ %% Returns: any (ignored by gen_server) %%-------------------------------------------------------------------- terminate(_Reason, State) -> - case State#state.socket of - undefined -> - ok; - Sock -> - do_close(Sock, State#state.is_ssl) - end. + do_close(State). %%-------------------------------------------------------------------- %% Func: code_change/3 @@ -358,10 +234,10 @@ do_trace("Data recvd on socket in state idle!. ~1000.p~n", [Data]), shutting_down(State), do_error_reply(State, data_in_status_idle), - do_close(State#state.socket, State#state.is_ssl), + do_close(State), {stop, normal, State}; -handle_sock_data(Data, #state{status=get_header, socket=Sock}=State) -> +handle_sock_data(Data, #state{status = get_header}=State) -> case parse_response(Data, State) of {error, _Reason} -> shutting_down(State), @@ -370,14 +246,15 @@ shutting_down(State), {stop, normal, State}; State_1 -> - do_setopts(Sock, [{active, once}], State#state.is_ssl), + active_once(State_1), {noreply, State_1, get_inac_timeout(State_1)} end; -handle_sock_data(Data, #state{status=get_body, content_length=CL, +handle_sock_data(Data, #state{status = get_body, + content_length = CL, http_status_code = StatCode, - recvd_headers=Headers, - chunk_size=CSz, socket=Sock}=State) -> + recvd_headers = Headers, + chunk_size = CSz} = State) -> case (CL == undefined) and (CSz == undefined) of true -> case accumulate_response(Data, State) of @@ -387,7 +264,7 @@ {error, {Reason, {stat_code, StatCode}, Headers}}), {stop, normal, State}; State_1 -> - do_setopts(Sock, [{active, once}], State#state.is_ssl), + active_once(State_1), {noreply, State_1, get_inac_timeout(State_1)} end; _ -> @@ -401,7 +278,7 @@ shutting_down(State), {stop, normal, State}; State_1 -> - do_setopts(Sock, [{active, once}], State#state.is_ssl), + active_once(State_1), {noreply, State_1, get_inac_timeout(State_1)} end end. @@ -452,22 +329,27 @@ cur_req = CurReq}=State) -> #request{stream_to=StreamTo, req_id=ReqId, stream_chunk_size = Stream_chunk_size, - response_format = Response_format} = CurReq, + response_format = Response_format, + caller_controls_socket = Caller_controls_socket} = CurReq, RepBuf_1 = concat_binary([RepBuf, Data]), New_data_size = RepBufSize - Streamed_size, case StreamTo of undefined -> State#state{reply_buffer = RepBuf_1}; - _ when New_data_size < Stream_chunk_size -> - State#state{reply_buffer = RepBuf_1}; - _ -> + _ when Caller_controls_socket == true -> + do_interim_reply(StreamTo, Response_format, ReqId, RepBuf_1), + State#state{reply_buffer = <<>>, + streamed_size = Streamed_size + size(RepBuf_1)}; + _ when New_data_size >= Stream_chunk_size -> {Stream_chunk, Rem_data} = split_binary(RepBuf_1, Stream_chunk_size), do_interim_reply(StreamTo, Response_format, ReqId, Stream_chunk), accumulate_response( Rem_data, State#state{ reply_buffer = <<>>, - streamed_size = Streamed_size + Stream_chunk_size}) + streamed_size = Streamed_size + Stream_chunk_size}); + _ -> + State#state{reply_buffer = RepBuf_1} end. make_tmp_filename() -> @@ -528,37 +410,45 @@ [binary, {nodelay, true}, {active, false}], Timeout). -do_send(Sock, Req, true) -> ssl:send(Sock, Req); -do_send(Sock, Req, false) -> gen_tcp:send(Sock, Req). +do_send(Req, #state{socket = Sock, is_ssl = true}) -> ssl:send(Sock, Req); +do_send(Req, #state{socket = Sock, is_ssl = false}) -> gen_tcp:send(Sock, Req). %% @spec do_send_body(Sock::socket_descriptor(), Source::source_descriptor(), IsSSL::boolean()) -> ok | error() %% source_descriptor() = fun_arity_0 | %% {fun_arity_0} | %% {fun_arity_1, term()} %% error() = term() -do_send_body(Sock, Source, IsSSL) when is_function(Source) -> - do_send_body(Sock, {Source}, IsSSL); -do_send_body(Sock, {Source}, IsSSL) when is_function(Source) -> - do_send_body1(Sock, Source, IsSSL, Source()); -do_send_body(Sock, {Source, State}, IsSSL) when is_function(Source) -> - do_send_body1(Sock, Source, IsSSL, Source(State)); -do_send_body(Sock, Body, IsSSL) -> - do_send(Sock, Body, IsSSL). +do_send_body(Source, State) when is_function(Source) -> + do_send_body({Source}, State); +do_send_body({Source}, State) when is_function(Source) -> + do_send_body1(Source, Source(), State); +do_send_body({Source, Source_state}, State) when is_function(Source) -> + do_send_body1(Source, Source(Source_state), State); +do_send_body(Body, State) -> + do_send(Body, State). -do_send_body1(Sock, Source, IsSSL, Resp) -> +do_send_body1(Source, Resp, State) -> case Resp of {ok, Data} -> - do_send(Sock, Data, IsSSL), - do_send_body(Sock, {Source}, IsSSL); - {ok, Data, NewState} -> - do_send(Sock, Data, IsSSL), - do_send_body(Sock, {Source, NewState}, IsSSL); - eof -> ok; - Err -> Err + do_send(Data, State), + do_send_body({Source}, State); + {ok, Data, New_source_state} -> + do_send(Data, State), + do_send_body({Source, New_source_state}, State); + eof -> + ok; + Err -> + Err end. -do_close(Sock, true) -> ssl:close(Sock); -do_close(Sock, false) -> gen_tcp:close(Sock). +do_close(#state{socket = undefined}) -> ok; +do_close(#state{socket = Sock, is_ssl = true}) -> ssl:close(Sock); +do_close(#state{socket = Sock, is_ssl = false}) -> gen_tcp:close(Sock). + +active_once(#state{cur_req = #request{caller_controls_socket = true}}) -> + ok; +active_once(#state{socket = Socket, is_ssl = Is_ssl}) -> + do_setopts(Socket, [{active, once}], Is_ssl). do_setopts(Sock, Opts, true) -> ssl:setopts(Sock, Opts); do_setopts(Sock, Opts, false) -> inet:setopts(Sock, Opts). @@ -571,11 +461,81 @@ State#state{is_ssl=true, ssl_options=get_value(ssl_options, Options)} end. -send_req_1(#url{abspath = AbsPath, - host = Host, - port = Port, - path = RelPath} = Url, - Headers, Method, Body, Options, Sock, State) -> +send_req_1(From, + #url{host = Host, + port = Port} = Url, + Headers, Method, Body, Options, Timeout, + #state{socket = undefined} = State) -> + {Host_1, Port_1, State_1} = + case get_value(proxy_host, Options, false) of + false -> + {Host, Port, State}; + PHost -> + ProxyUser = get_value(proxy_user, Options, []), + ProxyPassword = get_value(proxy_password, Options, []), + Digest = http_auth_digest(ProxyUser, ProxyPassword), + {PHost, get_value(proxy_port, Options, 80), + State#state{use_proxy = true, + proxy_auth_digest = Digest}} + end, + State_2 = check_ssl_options(Options, State_1), + do_trace("Connecting...~n", []), + Start_ts = now(), + Conn_timeout = get_value(connect_timeout, Options, Timeout), + case do_connect(Host_1, Port_1, Options, State_2, Conn_timeout) of + {ok, Sock} -> + do_trace("Connected!~n", []), + End_ts = now(), + Timeout_1 = case Timeout of + infinity -> + infinity; + _ -> + Timeout - trunc(round(timer:now_diff(End_ts, Start_ts) / 1000)) + end, + State_3 = State_2#state{socket = Sock}, + send_req_1(From, Url, Headers, Method, Body, Options, Timeout_1, State_3); + Err -> + shutting_down(State_2), + do_trace("Error connecting. Reason: ~1000.p~n", [Err]), + gen_server:reply(From, {error, conn_failed}), + {stop, normal, State_2} + end; +send_req_1(From, + #url{abspath = AbsPath, + host = Host, + port = Port, + path = RelPath} = Url, + Headers, Method, Body, Options, Timeout, + #state{status = Status} = State) -> + ReqId = make_req_id(), + Resp_format = get_value(response_format, Options, list), + {StreamTo, Caller_controls_socket} = + case get_value(stream_to, Options, undefined) of + {Caller, once} when is_pid(Caller) or + is_atom(Caller) -> + Async_pid_rec = {{req_id_pid, ReqId}, self()}, + true = ets:insert(ibrowse_stream, Async_pid_rec), + {Caller, true}; + undefined -> + {undefined, false}; + Caller when is_pid(Caller) or + is_atom(Caller) -> + {Caller, false}; + Stream_to_inv -> + exit({invalid_option, {stream_to, Stream_to_inv}}) + end, + SaveResponseToFile = get_value(save_response_to_file, Options, false), + NewReq = #request{url = Url, + method = Method, + stream_to = StreamTo, + caller_controls_socket = Caller_controls_socket, + options = Options, + req_id = ReqId, + save_response_to_file = SaveResponseToFile, + stream_chunk_size = get_stream_chunk_size(Options), + response_format = Resp_format, + from = From}, + State_1 = State#state{reqs=queue:in(NewReq, State#state.reqs)}, Headers_1 = add_auth_headers(Url, Options, Headers, State), HostHeaderValue = case lists:keysearch(host_header, 1, Options) of false -> @@ -598,14 +558,45 @@ "--- Request End ---~n", [NReq]); _ -> ok end, - SndRes = case do_send(Sock, Req, State#state.is_ssl) of - ok -> do_send_body(Sock, Body_1, State#state.is_ssl); - Err -> - io:format("Err: ~p~n", [Err]), - Err - end, - do_setopts(Sock, [{active, once}], State#state.is_ssl), - SndRes. + case do_send(Req, State) of + ok -> + case do_send_body(Body_1, State) of + ok -> + State_2 = inc_pipeline_counter(State_1), + active_once(State_1), + Ref = case Timeout of + infinity -> + undefined; + _ -> + erlang:send_after(Timeout, self(), {req_timedout, From}) + end, + State_3 = case Status of + idle -> + State_2#state{status = get_header, + cur_req = NewReq, + send_timer = Ref}; + _ -> + State_2#state{send_timer = Ref} + end, + case StreamTo of + undefined -> + ok; + _ -> + gen_server:reply(From, {ibrowse_req_id, ReqId}) + end, + {noreply, State_3, get_inac_timeout(State_3)}; + Err -> + shutting_down(State_1), + do_trace("Send failed... Reason: ~p~n", [Err]), + gen_server:reply(From, {error, send_failed}), + {stop, normal, State_1} + end; + Err -> + shutting_down(State_1), + do_trace("Send failed... Reason: ~p~n", [Err]), + gen_server:reply(From, {error, send_failed}), + {stop, normal, State_1} + end. add_auth_headers(#url{username = User, password = UPw}, @@ -719,9 +710,9 @@ encode_headers(L, []). encode_headers([{http_vsn, _Val} | T], Acc) -> encode_headers(T, Acc); -encode_headers([{Name,Val} | T], Acc) when list(Name) -> +encode_headers([{Name,Val} | T], Acc) when is_list(Name) -> encode_headers(T, [[Name, ": ", fmt_val(Val), crnl()] | Acc]); -encode_headers([{Name,Val} | T], Acc) when atom(Name) -> +encode_headers([{Name,Val} | T], Acc) when is_atom(Name) -> encode_headers(T, [[atom_to_list(Name), ": ", fmt_val(Val), crnl()] | Acc]); encode_headers([], Acc) -> lists:reverse(Acc). @@ -732,25 +723,25 @@ chunk_request_body(Body, _ChunkSize, Acc) when Body == <<>>; Body == [] -> LastChunk = "0\r\n", lists:reverse(["\r\n", LastChunk | Acc]); -chunk_request_body(Body, ChunkSize, Acc) when binary(Body), +chunk_request_body(Body, ChunkSize, Acc) when is_binary(Body), size(Body) >= ChunkSize -> <<ChunkBody:ChunkSize/binary, Rest/binary>> = Body, Chunk = [ibrowse_lib:dec2hex(4, ChunkSize),"\r\n", ChunkBody, "\r\n"], chunk_request_body(Rest, ChunkSize, [Chunk | Acc]); -chunk_request_body(Body, _ChunkSize, Acc) when binary(Body) -> +chunk_request_body(Body, _ChunkSize, Acc) when is_binary(Body) -> BodySize = size(Body), Chunk = [ibrowse_lib:dec2hex(4, BodySize),"\r\n", Body, "\r\n"], LastChunk = "0\r\n", lists:reverse(["\r\n", LastChunk, Chunk | Acc]); -chunk_request_body(Body, ChunkSize, Acc) when list(Body), +chunk_request_body(Body, ChunkSize, Acc) when is_list(Body), length(Body) >= ChunkSize -> {ChunkBody, Rest} = split_list_at(Body, ChunkSize), Chunk = [ibrowse_lib:dec2hex(4, ChunkSize),"\r\n", ChunkBody, "\r\n"], chunk_request_body(Rest, ChunkSize, [Chunk | Acc]); -chunk_request_body(Body, _ChunkSize, Acc) when list(Body) -> +chunk_request_body(Body, _ChunkSize, Acc) when is_list(Body) -> BodySize = length(Body), Chunk = [ibrowse_lib:dec2hex(4, BodySize),"\r\n", Body, "\r\n"], @@ -840,7 +831,7 @@ {error, content_length_undefined}; V -> case catch list_to_integer(V) of - V_1 when integer(V_1), V_1 >= 0 -> + V_1 when is_integer(V_1), V_1 >= 0 -> send_async_headers(ReqId, StreamTo, StatCode, Headers_1), do_trace("Recvd Content-Length of ~p~n", [V_1]), State_2 = State_1#state{rep_buf_size=0, @@ -1058,17 +1049,20 @@ parse_headers(Headers) -> case scan_crlf(Headers) of {yes, StatusLine, T} -> - Headers_1 = parse_headers_1(T), - case parse_status_line(StatusLine) of - {ok, HttpVsn, StatCode, _Msg} -> - put(http_prot_vsn, HttpVsn), - {HttpVsn, StatCode, Headers_1}; - _ -> %% A HTTP 0.9 response? - put(http_prot_vsn, "HTTP/0.9"), - {"HTTP/0.9", undefined, Headers} - end; - _ -> - {error, no_status_line} + parse_headers(StatusLine, T); + {no, StatusLine} -> + parse_headers(StatusLine, <<>>) + end. + +parse_headers(StatusLine, Headers) -> + Headers_1 = parse_headers_1(Headers), + case parse_status_line(StatusLine) of + {ok, HttpVsn, StatCode, _Msg} -> + put(http_prot_vsn, HttpVsn), + {HttpVsn, StatCode, Headers_1}; + _ -> %% A HTTP 0.9 response? + put(http_prot_vsn, "HTTP/0.9"), + {"HTTP/0.9", undefined, Headers} end. % From RFC 2616 @@ -1079,10 +1073,10 @@ % SP. A recipient MAY replace any linear white space with a single % SP before interpreting the field value or forwarding the message % downstream. -parse_headers_1(B) when is_binary(B) -> - parse_headers_1(binary_to_list(B)); -parse_headers_1(String) -> - parse_headers_1(String, [], []). + parse_headers_1(B) when is_binary(B) -> + parse_headers_1(binary_to_list(B)); + parse_headers_1(String) -> + parse_headers_1(String, [], []). parse_headers_1([$\n, H |T], [$\r | L], Acc) when H == 32; H == $\t -> @@ -1205,10 +1199,10 @@ %% scan_crlf([H|T], L) -> scan_crlf(T, [H|L]); %% scan_crlf([], L) -> {no, L}. -fmt_val(L) when list(L) -> L; -fmt_val(I) when integer(I) -> integer_to_list(I); -fmt_val(A) when atom(A) -> atom_to_list(A); -fmt_val(Term) -> io_lib:format("~p", [Term]). +fmt_val(L) when is_list(L) -> L; +fmt_val(I) when is_integer(I) -> integer_to_list(I); +fmt_val(A) when is_atom(A) -> atom_to_list(A); +fmt_val(Term) -> io_lib:format("~p", [Term]). crnl() -> "\r\n". @@ -1306,7 +1300,8 @@ do_reply(State, From, undefined, _, _, Msg) -> gen_server:reply(From, Msg), dec_pipeline_counter(State); -do_reply(State, _From, StreamTo, ReqId, Resp_format, {ok, _, _, Body}) -> +do_reply(#state{prev_req_id = Prev_req_id} = State, + _From, StreamTo, ReqId, Resp_format, {ok, _, _, Body}) -> State_1 = dec_pipeline_counter(State), case Body of [] -> @@ -1316,7 +1311,18 @@ catch StreamTo ! {ibrowse_async_response, ReqId, Body_1} end, catch StreamTo ! {ibrowse_async_response_end, ReqId}, - State_1; + %% We don't want to delete the Req-id to Pid mapping straightaway + %% as the client may send a stream_next message just while we are + %% sending back this ibrowse_async_response_end message. If we + %% deleted this mapping straightaway, the caller will see a + %% {error, unknown_req_id} when it calls ibrowse:stream_next/1. To + %% get around this, we store the req id, and clear it after the + %% next request. If there are wierd combinations of stream, + %% stream_once and sync requests on the same connection, it will + %% take a while for the req_id-pid mapping to get cleared, but it + %% should do no harm. + ets:delete(ibrowse_stream, {req_id_pid, Prev_req_id}), + State_1#state{prev_req_id = ReqId}; do_reply(State, _From, StreamTo, ReqId, Resp_format, Msg) -> State_1 = dec_pipeline_counter(State), Msg_1 = format_response_data(Resp_format, Msg), @@ -1333,6 +1339,7 @@ ReqList = queue:to_list(Reqs), lists:foreach(fun(#request{from=From, stream_to=StreamTo, req_id=ReqId, response_format = Resp_format}) -> + ets:delete(ibrowse_stream, {req_id_pid, ReqId}), do_reply(State, From, StreamTo, ReqId, Resp_format, {error, Err}) end, ReqList). Index: ibrowse_test.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_test.erl,v retrieving revision 1.4 retrieving revision 1.5 diff -u -d -r1.4 -r1.5 --- ibrowse_test.erl 1 Jul 2009 22:43:19 -0000 1.4 +++ ibrowse_test.erl 7 Jul 2009 22:30:58 -0000 1.5 @@ -18,9 +18,50 @@ ue_test/1, verify_chunked_streaming/0, verify_chunked_streaming/1, - i_do_async_req_list/4 + i_do_async_req_list/4, + test_stream_once/3, + test_stream_once/4 ]). +test_stream_once(Url, Method, Options) -> + test_stream_once(Url, Method, Options, 5000). + +test_stream_once(Url, Method, Options, Timeout) -> + case ibrowse:send_req(Url, [], Method, [], [{stream_to, {self(), once}} | Options], Timeout) of + {ibrowse_req_id, Req_id} -> + case ibrowse:stream_next(Req_id) of + ok -> + test_stream_once(Req_id); + Err -> + Err + end; + Err -> + Err + end. + +test_stream_once(Req_id) -> + receive + {ibrowse_async_headers, Req_id, StatCode, Headers} -> + io:format("Recvd headers~n~p~n", [{ibrowse_async_headers, Req_id, StatCode, Headers}]), + case ibrowse:stream_next(Req_id) of + ok -> + test_stream_once(Req_id); + Err -> + Err + end; + {ibrowse_async_response, Req_id, {error, Err}} -> + io:format("Recvd error: ~p~n", [Err]); + {ibrowse_async_response, Req_id, Body_1} -> + io:format("Recvd body part: ~n~p~n", [{ibrowse_async_response, Req_id, Body_1}]), + case ibrowse:stream_next(Req_id) of + ok -> + test_stream_once(Req_id); + Err -> + Err + end; + {ibrowse_async_response_end, Req_id} -> + ok + end. %% Use ibrowse:set_max_sessions/3 and ibrowse:set_max_pipeline_size/3 to %% tweak settings before running the load test. The defaults are 10 and 10. load_test(Url, NumWorkers, NumReqsPerWorker) when is_list(Url), @@ -182,7 +223,8 @@ unit_tests([]). unit_tests(Options) -> - {Pid, Ref} = erlang:spawn_monitor(?MODULE, unit_tests_1, [self(), Options]), + Options_1 = Options ++ [{connect_timeout, 5000}], + {Pid, Ref} = erlang:spawn_monitor(?MODULE, unit_tests_1, [self(), Options_1]), receive {done, Pid} -> ok; |
|
From: Chandru <cha...@us...> - 2009-07-07 22:31:04
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv7720 Modified Files: LICENSE README vsn.mk Log Message: Added option {stream_to, {process(), once}} to allow calling process to control data rate on socket Index: README =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/README,v retrieving revision 1.17 retrieving revision 1.18 diff -u -d -r1.17 -r1.18 --- README 1 Jul 2009 22:43:19 -0000 1.17 +++ README 7 Jul 2009 22:30:57 -0000 1.18 @@ -1,47 +1,54 @@ ibrowse is a HTTP client. The following are a list of features. - - RFC2616 compliant (AFAIK) - - supports GET, POST, OPTIONS, HEAD, PUT, DELETE, TRACE, + - RFC2616 compliant (AFAIK) + - supports GET, POST, OPTIONS, HEAD, PUT, DELETE, TRACE, MKCOL, PROPFIND, PROPPATCH, LOCK, UNLOCK, MOVE and COPY - - Understands HTTP/0.9, HTTP/1.0 and HTTP/1.1 - - Understands chunked encoding + - Understands HTTP/0.9, HTTP/1.0 and HTTP/1.1 + - Understands chunked encoding - Can generate requests using Chunked Transfer-Encoding - - Pools of connections to each webserver - - Pipelining support - - Download to file - - Asynchronous requests. Responses are streamed to a process - - Basic authentication - - Supports proxy authentication - - Can talk to Secure webservers using SSL - - any other features in the code not listed here :) + - Pools of connections to each webserver + - Pipelining support + - Download to file + - Asynchronous requests. Responses are streamed to a process + - Basic authentication + - Supports proxy authentication + - Can talk to Secure webservers using SSL + - any other features in the code not listed here :) ibrowse is available under two different licenses. LGPL and the BSD license. Comments to : Cha...@gm... -Version : 1.5.0 +Version : 1.5.1 Latest version : git://github.com/cmullaparthi/ibrowse.git CONTRIBUTIONS & CHANGE HISTORY ============================== -29-06-2009 - * Fixed following issues reported by Oscar Hellström +03-07-2009 - Added option {stream_to, {Pid, once}} which allows the caller + to control when it wants to receive more data. If this option + is used, the call ibrowse:stream_next(Req_id) should be used + to get more data. + - Patch submitted by Steve Vinoski to remove compiler warnings + about the use of obsolete guards + +29-06-2009 - * Fixed following issues reported by Oscar Hellström - Use {active, once} instead of {active, true} - - Fix 'dodgy' timeout handling - - Use binaries internally instead of lists to reduce memory + - Fix 'dodgy' timeout handling + - Use binaries internally instead of lists to reduce memory consumption on 64 bit platforms. The default response format is still 'list' to maintain backwards compatibility. Use the option {response_format, binary} to get responses as binaries. - * Fixed chunking bug (reported by Adam Kocoloski) - * Added new option {inactivity_timeout, Milliseconds} to timeout + * Fixed chunking bug (reported by Adam Kocoloski) + * Added new option {inactivity_timeout, Milliseconds} to timeout requests if no data is received on the link for the specified interval. Useful when responses are large and links are flaky. - * Added ibrowse:all_trace_off/0 to turn off all tracing - * Change to the way responses to asynchronous requests are + * Added ibrowse:all_trace_off/0 to turn off all tracing + * Change to the way responses to asynchronous requests are returned. The following messages have been removed. * {ibrowse_async_response, Req_id, {chunk_start, Chunk_size}} - * {ibrowse_async_response, Req_id, chunk_end} - * Fixed Makefiles as part of Debian packaging - (thanks to Thomas Lindgren) + * {ibrowse_async_response, Req_id, chunk_end} + * Fixed Makefiles as part of Debian packaging + (thanks to Thomas Lindgren) * Moved repository from Sourceforge to Github 11-06-2009 - * Added option to control size of streamed chunks. Also added @@ -83,7 +90,7 @@ 17-10-2007 - Matthew Reilly (matthew dot reilly _at_ sipphone dot com) sent a bug report and a fix. If the chunk trailer spans two TCP packets, then ibrowse fails to recognise that the chunked transfer - has ended. + has ended. 29-08-2007 - Bug report by Peter Kristensen(ptx _at_ daimi dot au dot dk). ibrowse crashes when the webserver returns just the Status line @@ -104,7 +111,7 @@ 12-01-2007 - Derek Upham sent in a bug fix. The reset_state function was not behaving correctly when the transfer encoding was not chunked. -13-11-2006 - YounÂs Hafri reported a bug where ibrowse was not returning the +13-11-2006 - Youns Hafri reported a bug where ibrowse was not returning the temporary filename when the server was closing the connection after sending the data (as in HTTP/1.0). Released ibrowse under the BSD license @@ -123,8 +130,8 @@ 22-Nov-2005 - Added ability to generate requests using the Chunked Transfer-Encoding. -08-May-2005 - YounÂs Hafri made a CRUX LINUX port of ibrowse. - http://yhafri.club.fr/crux/index.html +08-May-2005 - Youns Hafri made a CRUX LINUX port of ibrowse. + http://yhafri.club.fr/crux/index.html Here are some usage examples. Enjoy! @@ -147,10 +154,10 @@ %% ============================================================================= %% A GET using a proxy 7> ibrowse:send_req("http://www.google.com/", [], get, [], - [{proxy_user, "XXXXX"}, - {proxy_password, "XXXXX"}, - {proxy_host, "proxy"}, - {proxy_port, 8080}], 1000). + [{proxy_user, "XXXXX"}, + {proxy_password, "XXXXX"}, + {proxy_host, "proxy"}, + {proxy_port, 8080}], 1000). {ok,"302", [{"Date","Fri, 17 Dec 2004 15:22:56 GMT"}, {"Content-Length","217"}, @@ -170,20 +177,20 @@ %% be set using the application env var 'download_dir' - the default %% is the current working directory. 8> ibrowse:send_req("http://www.erlang.se/", [], get, [], - [{proxy_user, "XXXXX"}, - {proxy_password, "XXXXX"}, - {proxy_host, "proxy"}, - {proxy_port, 8080}, - {save_response_to_file, true}], 1000). + [{proxy_user, "XXXXX"}, + {proxy_password, "XXXXX"}, + {proxy_host, "proxy"}, + {proxy_port, 8080}, + {save_response_to_file, true}], 1000). {error,req_timedout} %% ============================================================================= 9> ibrowse:send_req("http://www.erlang.se/", [], get, [], - [{proxy_user, "XXXXX"}, - {proxy_password, "XXXXX"}, - {proxy_host, "proxy"}, - {proxy_port, 8080}, - {save_response_to_file, true}], 5000). + [{proxy_user, "XXXXX"}, + {proxy_password, "XXXXX"}, + {proxy_host, "proxy"}, + {proxy_port, 8080}, + {save_response_to_file, true}], 5000). {ok,"200", [{"Transfer-Encoding","chunked"}, {"Date","Fri, 17 Dec 2004 15:24:36 GMT"}, @@ -197,7 +204,7 @@ %% number of maximum connections to this server to 10 and the pipeline %% size to 1. Connections are setup a required. 11> ibrowse:set_dest("www.hotmail.com", 80, [{max_sessions, 10}, - {max_pipeline_size, 1}]). + {max_pipeline_size, 1}]). ok %% ============================================================================= @@ -231,51 +238,51 @@ %% ============================================================================= %% Example of using Asynchronous requests 18> ibrowse:send_req("http://www.google.com", [], get, [], - [{proxy_user, "XXXXX"}, - {proxy_password, "XXXXX"}, - {proxy_host, "proxy"}, - {proxy_port, 8080}, - {stream_to, self()}]). + [{proxy_user, "XXXXX"}, + {proxy_password, "XXXXX"}, + {proxy_host, "proxy"}, + {proxy_port, 8080}, + {stream_to, self()}]). {ibrowse_req_id,{1115,327256,389608}} 19> flush(). Shell got {ibrowse_async_headers,{1115,327256,389608}, - "302", - [{"Date","Thu, 05 May 2005 21:06:41 GMT"}, - {"Content-Length","217"}, - {"Content-Type","text/html"}, - {"Set-Cookie", - "PREF=ID=b601f16bfa32f071:CR=1:TM=1115327201:LM=1115327201:S=OX5hSB525AMjUUu7; expires=Sun, 17-Jan-2038 19:14:07 GMT; path=/; domain=.google.com"}, - {"Server","GWS/2.1"}, - {"Location", - "http://www.google.co.uk/cxfer?c=PREF%3D:TM%3D1115327201:S%3DDS9pDJ4IHcAuZ_AS&prev=/"}, - {"Via", - "1.1 hatproxy01 (NetCache NetApp/5.6.2)"}]} + "302", + [{"Date","Thu, 05 May 2005 21:06:41 GMT"}, + {"Content-Length","217"}, + {"Content-Type","text/html"}, + {"Set-Cookie", + "PREF=ID=b601f16bfa32f071:CR=1:TM=1115327201:LM=1115327201:S=OX5hSB525AMjUUu7; expires=Sun, 17-Jan-2038 19:14:07 GMT; path=/; domain=.google.com"}, + {"Server","GWS/2.1"}, + {"Location", + "http://www.google.co.uk/cxfer?c=PREF%3D:TM%3D1115327201:S%3DDS9pDJ4IHcAuZ_AS&prev=/"}, + {"Via", + "1.1 hatproxy01 (NetCache NetApp/5.6.2)"}]} Shell got {ibrowse_async_response,{1115,327256,389608}, - "<HTML><HEAD><TITLE>302 Moved</TITLE></HEAD><BODY>\n<H1>302 Moved</H1>\nThe document has moved\n<A HREF=\"http://www.google.co.uk/cxfer?c=PREF%3D:TM%3D1115327201:S%3DDS9pDJ4IHcAuZ_AS&prev=/\">here</A>.\r\n</BODY></HTML>\r\n"} + "<HTML><HEAD><TITLE>302 Moved</TITLE></HEAD><BODY>\n<H1>302 Moved</H1>\nThe document has moved\n<A HREF=\"http://www.google.co.uk/cxfer?c=PREF%3D:TM%3D1115327201:S%3DDS9pDJ4IHcAuZ_AS&prev=/\">here</A>.\r\n</BODY></HTML>\r\n"} Shell got {ibrowse_async_response_end,{1115,327256,389608}} ok %% ============================================================================= %% Another example of using async requests 24> ibrowse:send_req("http://yaws.hyber.org/simple_ex2.yaws", [], get, [], - [{proxy_user, "XXXXX"}, - {proxy_password, "XXXXX"}, - {proxy_host, "proxy"}, - {proxy_port, 8080}, - {stream_to, self()}]). + [{proxy_user, "XXXXX"}, + {proxy_password, "XXXXX"}, + {proxy_host, "proxy"}, + {proxy_port, 8080}, + {stream_to, self()}]). {ibrowse_req_id,{1115,327430,512314}} 25> flush(). Shell got {ibrowse_async_headers,{1115,327430,512314}, - "200", - [{"Date","Thu, 05 May 2005 20:58:08 GMT"}, - {"Content-Length","64"}, - {"Content-Type","text/html;charset="}, - {"Server", - "Yaws/1.54 Yet Another Web Server"}, - {"Via", - "1.1 hatproxy01 (NetCache NetApp/5.6.2)"}]} + "200", + [{"Date","Thu, 05 May 2005 20:58:08 GMT"}, + {"Content-Length","64"}, + {"Content-Type","text/html;charset="}, + {"Server", + "Yaws/1.54 Yet Another Web Server"}, + {"Via", + "1.1 hatproxy01 (NetCache NetApp/5.6.2)"}]} Shell got {ibrowse_async_response,{1115,327430,512314}, - "<html>\n\n\n<h1> Yesssssss </h1>\n\n<h2> Hello again </h2>\n\n\n</html>\n"} + "<html>\n\n\n<h1> Yesssssss </h1>\n\n<h2> Hello again </h2>\n\n\n</html>\n"} Shell got {ibrowse_async_response_end,{1115,327430,512314}} %% ============================================================================= @@ -287,12 +294,12 @@ %% Example of request using both Proxy-Authorization and authorization by the final webserver. 17> ibrowse:send_req("http://www.erlang.se/lic_area/protected/patches/erl_756_otp_beam.README", - [], get, [], - [{proxy_user, "XXXXX"}, - {proxy_password, "XXXXX"}, - {proxy_host, "proxy"}, - {proxy_port, 8080}, - {basic_auth, {"XXXXX", "XXXXXX"}}]). + [], get, [], + [{proxy_user, "XXXXX"}, + {proxy_password, "XXXXX"}, + {proxy_host, "proxy"}, + {proxy_port, 8080}, + {basic_auth, {"XXXXX", "XXXXXX"}}]). {ok,"200", [{"Accept-Ranges","bytes"}, {"Date","Thu, 05 May 2005 21:02:09 GMT"}, @@ -309,10 +316,10 @@ %% support this. Nor did www.google.com. But good old BBC supports %% this. 35> 37> ibrowse:send_req("http://www.bbc.co.uk/", [], trace, [], - [{proxy_user, "XXXXX"}, - {proxy_password, "XXXXX"}, - {proxy_host, "proxy"}, - {proxy_port, 8080}]). + [{proxy_user, "XXXXX"}, + {proxy_password, "XXXXX"}, + {proxy_host, "proxy"}, + {proxy_port, 8080}]). {ok,"200", [{"Transfer-Encoding","chunked"}, {"Date","Thu, 05 May 2005 21:40:27 GMT"}, Index: LICENSE =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/LICENSE,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- LICENSE 22 Feb 2006 23:59:58 -0000 1.1 +++ LICENSE 7 Jul 2009 22:30:57 -0000 1.2 @@ -1,5 +1,5 @@ ibrowse - a HTTP client written in erlang -Copyright (C) 2005 Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com> +Copyright (C) 2005-2009 Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. Index: vsn.mk =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/vsn.mk,v retrieving revision 1.15 retrieving revision 1.16 diff -u -d -r1.15 -r1.16 --- vsn.mk 1 Jul 2009 22:43:19 -0000 1.15 +++ vsn.mk 7 Jul 2009 22:30:57 -0000 1.16 @@ -1,2 +1,2 @@ -IBROWSE_VSN = 1.5.0 +IBROWSE_VSN = 1.5.1 |
|
From: Chandru <cha...@us...> - 2009-07-07 22:31:01
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse/doc In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv7720/doc Modified Files: ibrowse.html Log Message: Added option {stream_to, {process(), once}} to allow calling process to control data rate on socket Index: ibrowse.html =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/doc/ibrowse.html,v retrieving revision 1.8 retrieving revision 1.9 diff -u -d -r1.8 -r1.9 --- ibrowse.html 1 Jul 2009 22:43:19 -0000 1.8 +++ ibrowse.html 7 Jul 2009 22:30:58 -0000 1.9 @@ -12,7 +12,7 @@ <ul class="index"><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>The ibrowse application implements an HTTP 1.1 client. <p>Copyright © 2005-2009 Chandrashekhar Mullaparthi</p> -<p><b>Version:</b> 1.5.0</p> +<p><b>Version:</b> 1.5.1</p> <p><b>Behaviours:</b> <a href="gen_server.html"><tt>gen_server</tt></a>.</p> <p><b>Authors:</b> Chandrashekhar Mullaparthi (<a href="mailto:chandrashekhar dot mullaparthi at gmail dot com"><tt>chandrashekhar dot mullaparthi at gmail dot com</tt></a>).</p> @@ -101,6 +101,8 @@ <tr><td valign="top"><a href="#stop-0">stop/0</a></td><td>Stop the ibrowse process.</td></tr> <tr><td valign="top"><a href="#stop_worker_process-1">stop_worker_process/1</a></td><td>Terminate a worker process spawned using spawn_worker_process/2 or spawn_link_worker_process/2.</td></tr> +<tr><td valign="top"><a href="#stream_next-1">stream_next/1</a></td><td>Tell ibrowse to stream the next chunk of data to the + caller.</td></tr> <tr><td valign="top"><a href="#terminate-2">terminate/2</a></td><td></td></tr> <tr><td valign="top"><a href="#trace_off-0">trace_off/0</a></td><td>Turn tracing off for the ibrowse process.</td></tr> <tr><td valign="top"><a href="#trace_off-2">trace_off/2</a></td><td>Turn tracing OFF for all connections to the specified HTTP @@ -177,7 +179,8 @@ <li><tt><a name="type-respHeader">respHeader()</a> = {<a href="#type-headerName">headerName()</a>, <a href="#type-headerValue">headerValue()</a>}</tt></li> <li><tt><a name="type-headerName">headerName()</a> = string()</tt></li> <li><tt><a name="type-headerValue">headerValue()</a> = string()</tt></li> -<li><tt><a name="type-response">response()</a> = {ok, Status, ResponseHeaders, ResponseBody} | {error, Reason}</tt></li> +<li><tt><a name="type-response">response()</a> = {ok, Status, ResponseHeaders, ResponseBody} | {ibrowse_req_id, <a href="#type-req_id">req_id()</a>} | {error, Reason}</tt></li> +<li><tt><a name="type-req_id">req_id()</a> = term()</tt></li> <li><tt>ResponseBody = string() | {file, Filename}</tt></li> <li><tt>Reason = term()</tt></li> </ul></p> @@ -199,7 +202,8 @@ <div class="spec"> <p><tt>send_req(Url::string(), Headers::<a href="#type-headerList">headerList()</a>, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>, Options::<a href="#type-optionList">optionList()</a>) -> <a href="#type-response">response()</a></tt> <ul class="definitions"><li><tt><a name="type-optionList">optionList()</a> = [<a href="#type-option">option()</a>]</tt></li> -<li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {response_format, <a href="#type-response_format">response_format()</a>} | {stream_chunk_size, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>} | {is_ssl, <a href="#type-boolean">boolean()</a>} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, <a href="#type-boolean">boolean()</a>} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-srtf">srtf()</a>} | {stream_to, <a href="#type-process">process()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {inactivity_timeout, integer()} | {connect_timeout, integer()} | {transfer_encoding, {chunked, ChunkSize}}</tt></li> +<li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {response_format, <a href="#type-response_format">response_format()</a>} | {stream_chunk_size, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>} | {is_ssl, <a href="#type-boolean">boolean()</a>} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, <a href="#type-boolean">boolean()</a>} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-srtf">srtf()</a>} | {stream_to, <a href="#type-stream_to">stream_to()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {inactivity_timeout, integer()} | {connect_timeout, integer()} | {transfer_encoding, {chunked, ChunkSize}}</tt></li> +<li><tt><a name="type-stream_to">stream_to()</a> = <a href="#type-process">process()</a> | {<a href="#type-process">process()</a>, once}</tt></li> <li><tt><a name="type-process">process()</a> = pid() | atom()</tt></li> <li><tt><a name="type-username">username()</a> = string()</tt></li> <li><tt><a name="type-password">password()</a> = string()</tt></li> @@ -210,7 +214,7 @@ <li><tt><a name="type-response_format">response_format()</a> = list | binary</tt></li> </ul></p> </div><p>Same as send_req/4. - For a description of SSL Options, look in the ssl manpage. If the + For a description of SSL Options, look in the <a href="http://www.erlang.org/doc/apps/ssl/index.html">ssl</a> manpage. If the HTTP Version to use is not specified, the default is 1.1. <br> <p>The <code>host_header</code> option is useful in the case where ibrowse is @@ -221,6 +225,14 @@ used to specify what should go in the <code>Host</code> header in the request.</p> <ul> + <li>The <code>stream_to</code> option can be used to have the HTTP + response streamed to a process as messages as data arrives on the + socket. If the calling process wishes to control the rate at which + data is received from the server, the option <code>{stream_to, + {process(), once}}</code> can be specified. The calling process + will have to invoke <code>ibrowse:stream_next(Request_id)</code> to + receive the next packet.</li> + <li>When both the options <code>save_response_to_file</code> and <code>stream_to</code> are specified, the former takes precedence.</li> @@ -360,6 +372,13 @@ spawn_worker_process/2 or spawn_link_worker_process/2. Requests in progress will get the error response <pre>{error, closing_on_request}</pre></p> +<h3 class="function"><a name="stream_next-1">stream_next/1</a></h3> +<div class="spec"> +<p><tt>stream_next(Req_id::<a href="#type-req_id">req_id()</a>) -> ok | {error, unknown_req_id}</tt></p> +</div><p>Tell ibrowse to stream the next chunk of data to the + caller. Should be used in conjunction with the + <code>stream_to</code> option</p> + <h3 class="function"><a name="terminate-2">terminate/2</a></h3> <div class="spec"> <p><tt>terminate(Reason, State) -> any()</tt></p> @@ -392,6 +411,6 @@ <hr> <div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div> -<p><i>Generated by EDoc, Jun 30 2009, 23:44:01.</i></p> +<p><i>Generated by EDoc, Jul 7 2009, 23:13:24.</i></p> </body> </html> |
|
From: Chandru <cha...@us...> - 2009-07-01 22:46:15
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse/doc In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv26761/doc Modified Files: ibrowse.html Log Message: ibrowse-1.5.0 - Use binaries internally. Lots of other changes. See README Index: ibrowse.html =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/doc/ibrowse.html,v retrieving revision 1.7 retrieving revision 1.8 diff -u -d -r1.7 -r1.8 --- ibrowse.html 21 May 2008 15:28:11 -0000 1.7 +++ ibrowse.html 1 Jul 2009 22:43:19 -0000 1.8 @@ -10,9 +10,9 @@ <h1>Module ibrowse</h1> <ul class="index"><li><a href="#description">Description</a></li><li><a href="#index">Function Index</a></li><li><a href="#functions">Function Details</a></li></ul>The ibrowse application implements an HTTP 1.1 client. -<p>Copyright © 2005-2008 Chandrashekhar Mullaparthi</p> +<p>Copyright © 2005-2009 Chandrashekhar Mullaparthi</p> -<p><b>Version:</b> 1.4</p> +<p><b>Version:</b> 1.5.0</p> <p><b>Behaviours:</b> <a href="gen_server.html"><tt>gen_server</tt></a>.</p> <p><b>Authors:</b> Chandrashekhar Mullaparthi (<a href="mailto:chandrashekhar dot mullaparthi at gmail dot com"><tt>chandrashekhar dot mullaparthi at gmail dot com</tt></a>).</p> @@ -64,7 +64,8 @@ speed achieved using only erlang has been good enough, so the driver isn't actually used.</p> <h2><a name="index">Function Index</a></h2> -<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#code_change-3">code_change/3</a></td><td></td></tr> +<table width="100%" border="1" cellspacing="0" cellpadding="2" summary="function index"><tr><td valign="top"><a href="#all_trace_off-0">all_trace_off/0</a></td><td>Turn Off ALL tracing.</td></tr> +<tr><td valign="top"><a href="#code_change-3">code_change/3</a></td><td></td></tr> <tr><td valign="top"><a href="#get_config_value-1">get_config_value/1</a></td><td>Internal export.</td></tr> <tr><td valign="top"><a href="#get_config_value-2">get_config_value/2</a></td><td>Internal export.</td></tr> <tr><td valign="top"><a href="#handle_call-3">handle_call/3</a></td><td></td></tr> @@ -111,44 +112,49 @@ <h2><a name="functions">Function Details</a></h2> +<h3 class="function"><a name="all_trace_off-0">all_trace_off/0</a></h3> +<div class="spec"> +<p><tt>all_trace_off() -> ok</tt></p> +</div><p>Turn Off ALL tracing</p> + <h3 class="function"><a name="code_change-3">code_change/3</a></h3> <div class="spec"> -<p><tt>code_change() -> term()</tt></p> +<p><tt>code_change(OldVsn, State, Extra) -> any()</tt></p> </div> <h3 class="function"><a name="get_config_value-1">get_config_value/1</a></h3> <div class="spec"> -<p><tt>get_config_value() -> term()</tt></p> +<p><tt>get_config_value(Key) -> any()</tt></p> </div><p>Internal export</p> <h3 class="function"><a name="get_config_value-2">get_config_value/2</a></h3> <div class="spec"> -<p><tt>get_config_value() -> term()</tt></p> +<p><tt>get_config_value(Key, DefVal) -> any()</tt></p> </div><p>Internal export</p> <h3 class="function"><a name="handle_call-3">handle_call/3</a></h3> <div class="spec"> -<p><tt>handle_call() -> term()</tt></p> +<p><tt>handle_call(Request, From, State) -> any()</tt></p> </div> <h3 class="function"><a name="handle_cast-2">handle_cast/2</a></h3> <div class="spec"> -<p><tt>handle_cast() -> term()</tt></p> +<p><tt>handle_cast(Msg, State) -> any()</tt></p> </div> <h3 class="function"><a name="handle_info-2">handle_info/2</a></h3> <div class="spec"> -<p><tt>handle_info() -> term()</tt></p> +<p><tt>handle_info(Info, State) -> any()</tt></p> </div> <h3 class="function"><a name="init-1">init/1</a></h3> <div class="spec"> -<p><tt>init() -> term()</tt></p> +<p><tt>init(X1) -> any()</tt></p> </div> <h3 class="function"><a name="rescan_config-0">rescan_config/0</a></h3> <div class="spec"> -<p><tt>rescan_config() -> term()</tt></p> +<p><tt>rescan_config() -> any()</tt></p> </div><p>Clear current configuration for ibrowse and load from the file ibrowse.conf in the IBROWSE_EBIN/../priv directory. Current configuration is cleared only if the ibrowse.conf file is readable @@ -156,7 +162,7 @@ <h3 class="function"><a name="rescan_config-1">rescan_config/1</a></h3> <div class="spec"> -<p><tt>rescan_config() -> term()</tt></p> +<p><tt>rescan_config(File) -> any()</tt></p> </div> <h3 class="function"><a name="send_req-3">send_req/3</a></h3> @@ -193,7 +199,7 @@ <div class="spec"> <p><tt>send_req(Url::string(), Headers::<a href="#type-headerList">headerList()</a>, Method::<a href="#type-method">method()</a>, Body::<a href="#type-body">body()</a>, Options::<a href="#type-optionList">optionList()</a>) -> <a href="#type-response">response()</a></tt> <ul class="definitions"><li><tt><a name="type-optionList">optionList()</a> = [<a href="#type-option">option()</a>]</tt></li> -<li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>} | {is_ssl, <a href="#type-boolean">boolean()</a>} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, <a href="#type-boolean">boolean()</a>} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-srtf">srtf()</a>} | {stream_to, <a href="#type-process">process()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {transfer_encoding, {chunked, ChunkSize}}</tt></li> +<li><tt><a name="type-option">option()</a> = {max_sessions, integer()} | {response_format, <a href="#type-response_format">response_format()</a>} | {stream_chunk_size, integer()} | {max_pipeline_size, integer()} | {trace, <a href="#type-boolean">boolean()</a>} | {is_ssl, <a href="#type-boolean">boolean()</a>} | {ssl_options, [SSLOpt]} | {pool_name, atom()} | {proxy_host, string()} | {proxy_port, integer()} | {proxy_user, string()} | {proxy_password, string()} | {use_absolute_uri, <a href="#type-boolean">boolean()</a>} | {basic_auth, {<a href="#type-username">username()</a>, <a href="#type-password">password()</a>}} | {cookie, string()} | {content_length, integer()} | {content_type, string()} | {save_response_to_file, <a href="#type-srtf">srtf()</a>} | {stream_to, <a href="#type-process">process()</a>} | {http_vsn, {MajorVsn, MinorVsn}} | {host_header, string()} | {inactivity_timeout, integer()} | {connect_timeout, integer()} | {transfer_encoding, {chunked, ChunkSize}}</tt></li> <li><tt><a name="type-process">process()</a> = pid() | atom()</tt></li> <li><tt><a name="type-username">username()</a> = string()</tt></li> <li><tt><a name="type-password">password()</a> = string()</tt></li> @@ -201,12 +207,13 @@ <li><tt>ChunkSize = integer()</tt></li> <li><tt><a name="type-srtf">srtf()</a> = <a href="#type-boolean">boolean()</a> | <a href="#type-filename">filename()</a></tt></li> <li><tt><a name="type-filename">filename()</a> = string()</tt></li> +<li><tt><a name="type-response_format">response_format()</a> = list | binary</tt></li> </ul></p> </div><p>Same as send_req/4. For a description of SSL Options, look in the ssl manpage. If the HTTP Version to use is not specified, the default is 1.1. <br> - <p>The <code>host_header</code> is useful in the case where ibrowse is + <p>The <code>host_header</code> option is useful in the case where ibrowse is connecting to a component such as <a href="http://www.stunnel.org">stunnel</a> which then sets up a secure connection to a webserver. In this case, the URL supplied to ibrowse must have the stunnel host/port details, but that won't @@ -223,7 +230,35 @@ <li>Whenever an error occurs in the processing of a request, ibrowse will return as much information as it has, such as HTTP Status Code and HTTP Headers. When this happens, the response is of the form <code>{error, {Reason, {stat_code, StatusCode}, HTTP_headers}}</code></li> - </ul></p> + + <li>The <code>inactivity_timeout</code> option is useful when + dealing with large response bodies and/or slow links. In these + cases, it might be hard to estimate how long a request will take to + complete. In such cases, the client might want to timeout if no + data has been received on the link for a certain time interval.</li> + + <li> + The <code>connect_timeout</code> option is to specify how long the + client process should wait for connection establishment. This is + useful in scenarios where connections to servers are usually setup + very fast, but responses might take much longer compared to + connection setup. In such cases, it is better for the calling + process to timeout faster if there is a problem (DNS lookup + delays/failures, network routing issues, etc). The total timeout + value specified for the request will enforced. To illustrate using + an example: + <code> + ibrowse:send_req("http://www.example.com/cgi-bin/request", [], get, [], [{connect_timeout, 100}], 1000). + </code> + In the above invocation, if the connection isn't established within + 100 milliseconds, the request will fail with + <code>{error, conn_failed}</code>.<br> + If connection setup succeeds, the total time allowed for the + request to complete will be 1000 milliseconds minus the time taken + for connection setup. + </li> + </ul> + </p> <h3 class="function"><a name="send_req-6">send_req/6</a></h3> <div class="spec"> @@ -235,31 +270,31 @@ <h3 class="function"><a name="send_req_direct-4">send_req_direct/4</a></h3> <div class="spec"> -<p><tt>send_req_direct() -> term()</tt></p> +<p><tt>send_req_direct(Conn_pid, Url, Headers, Method) -> any()</tt></p> </div><p>Same as send_req/3 except that the first argument is the PID returned by spawn_worker_process/2 or spawn_link_worker_process/2</p> <h3 class="function"><a name="send_req_direct-5">send_req_direct/5</a></h3> <div class="spec"> -<p><tt>send_req_direct() -> term()</tt></p> +<p><tt>send_req_direct(Conn_pid, Url, Headers, Method, Body) -> any()</tt></p> </div><p>Same as send_req/4 except that the first argument is the PID returned by spawn_worker_process/2 or spawn_link_worker_process/2</p> <h3 class="function"><a name="send_req_direct-6">send_req_direct/6</a></h3> <div class="spec"> -<p><tt>send_req_direct() -> term()</tt></p> +<p><tt>send_req_direct(Conn_pid, Url, Headers, Method, Body, Options) -> any()</tt></p> </div><p>Same as send_req/5 except that the first argument is the PID returned by spawn_worker_process/2 or spawn_link_worker_process/2</p> <h3 class="function"><a name="send_req_direct-7">send_req_direct/7</a></h3> <div class="spec"> -<p><tt>send_req_direct() -> term()</tt></p> +<p><tt>send_req_direct(Conn_pid, Url, Headers, Method, Body, Options, Timeout) -> any()</tt></p> </div><p>Same as send_req/6 except that the first argument is the PID returned by spawn_worker_process/2 or spawn_link_worker_process/2</p> <h3 class="function"><a name="set_dest-3">set_dest/3</a></h3> <div class="spec"> -<p><tt>set_dest() -> term()</tt></p> +<p><tt>set_dest(Host, Port, T) -> any()</tt></p> </div><p>Deprecated. Use set_max_sessions/3 and set_max_pipeline_size/3 for achieving the same effect.</p> @@ -275,7 +310,7 @@ <h3 class="function"><a name="show_dest_status-2">show_dest_status/2</a></h3> <div class="spec"> -<p><tt>show_dest_status() -> term()</tt></p> +<p><tt>show_dest_status(Host, Port) -> any()</tt></p> </div><p>Shows some internal information about load balancing to a specified Host:Port. Info about workers spawned using spawn_worker_process/2 or spawn_link_worker_process/2 is not @@ -283,7 +318,7 @@ <h3 class="function"><a name="spawn_link_worker_process-2">spawn_link_worker_process/2</a></h3> <div class="spec"> -<p><tt>spawn_link_worker_process() -> term()</tt></p> +<p><tt>spawn_link_worker_process(Host, Port) -> any()</tt></p> </div><p>Same as spawn_worker_process/2 except the the calling process is linked to the worker process which is spawned.</p> @@ -305,7 +340,7 @@ <h3 class="function"><a name="start-0">start/0</a></h3> <div class="spec"> -<p><tt>start() -> term()</tt></p> +<p><tt>start() -> any()</tt></p> </div><p>Starts the ibrowse process without linking. Useful when testing using the shell</p> <h3 class="function"><a name="start_link-0">start_link/0</a></h3> @@ -315,7 +350,7 @@ <h3 class="function"><a name="stop-0">stop/0</a></h3> <div class="spec"> -<p><tt>stop() -> term()</tt></p> +<p><tt>stop() -> any()</tt></p> </div><p>Stop the ibrowse process. Useful when testing using the shell.</p> <h3 class="function"><a name="stop_worker_process-1">stop_worker_process/1</a></h3> @@ -327,28 +362,28 @@ <h3 class="function"><a name="terminate-2">terminate/2</a></h3> <div class="spec"> -<p><tt>terminate() -> term()</tt></p> +<p><tt>terminate(Reason, State) -> any()</tt></p> </div> <h3 class="function"><a name="trace_off-0">trace_off/0</a></h3> <div class="spec"> -<p><tt>trace_off() -> term()</tt></p> +<p><tt>trace_off() -> any()</tt></p> </div><p>Turn tracing off for the ibrowse process</p> <h3 class="function"><a name="trace_off-2">trace_off/2</a></h3> <div class="spec"> -<p><tt>trace_off(Host, Port) -> term()</tt></p> +<p><tt>trace_off(Host, Port) -> ok</tt></p> </div><p>Turn tracing OFF for all connections to the specified HTTP server.</p> <h3 class="function"><a name="trace_on-0">trace_on/0</a></h3> <div class="spec"> -<p><tt>trace_on() -> term()</tt></p> +<p><tt>trace_on() -> any()</tt></p> </div><p>Turn tracing on for the ibrowse process</p> <h3 class="function"><a name="trace_on-2">trace_on/2</a></h3> <div class="spec"> -<p><tt>trace_on(Host, Port) -> term()</tt> +<p><tt>trace_on(Host, Port) -> ok</tt> <ul class="definitions"><li><tt>Host = string()</tt></li> <li><tt>Port = integer()</tt></li> </ul></p> @@ -357,6 +392,6 @@ <hr> <div class="navbar"><a name="#navbar_bottom"></a><table width="100%" border="0" cellspacing="0" cellpadding="2" summary="navigation bar"><tr><td><a href="overview-summary.html" target="overviewFrame">Overview</a></td><td><a href="http://www.erlang.org/"><img src="erlang.png" align="right" border="0" alt="erlang logo"></a></td></tr></table></div> -<p><i>Generated by EDoc, Mar 27 2008, 01:20:55.</i></p> +<p><i>Generated by EDoc, Jun 30 2009, 23:44:01.</i></p> </body> </html> |
|
From: Chandru <cha...@us...> - 2009-07-01 22:44:28
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv26761 Modified Files: README vsn.mk Added Files: Makefile Log Message: ibrowse-1.5.0 - Use binaries internally. Lots of other changes. See README Index: README =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/README,v retrieving revision 1.16 retrieving revision 1.17 diff -u -d -r1.16 -r1.17 --- README 21 May 2008 15:28:11 -0000 1.16 +++ README 1 Jul 2009 22:43:19 -0000 1.17 @@ -1,5 +1,3 @@ -$Id$ - ibrowse is a HTTP client. The following are a list of features. - RFC2616 compliant (AFAIK) - supports GET, POST, OPTIONS, HEAD, PUT, DELETE, TRACE, @@ -18,10 +16,37 @@ ibrowse is available under two different licenses. LGPL and the BSD license. -Comments to : Cha...@t-... +Comments to : Cha...@gm... + +Version : 1.5.0 + +Latest version : git://github.com/cmullaparthi/ibrowse.git CONTRIBUTIONS & CHANGE HISTORY ============================== +29-06-2009 - * Fixed following issues reported by Oscar Hellström + - Use {active, once} instead of {active, true} + - Fix 'dodgy' timeout handling + - Use binaries internally instead of lists to reduce memory + consumption on 64 bit platforms. The default response format + is still 'list' to maintain backwards compatibility. Use the + option {response_format, binary} to get responses as binaries. + * Fixed chunking bug (reported by Adam Kocoloski) + * Added new option {inactivity_timeout, Milliseconds} to timeout + requests if no data is received on the link for the specified + interval. Useful when responses are large and links are flaky. + * Added ibrowse:all_trace_off/0 to turn off all tracing + * Change to the way responses to asynchronous requests are + returned. The following messages have been removed. + * {ibrowse_async_response, Req_id, {chunk_start, Chunk_size}} + * {ibrowse_async_response, Req_id, chunk_end} + * Fixed Makefiles as part of Debian packaging + (thanks to Thomas Lindgren) + * Moved repository from Sourceforge to Github + +11-06-2009 - * Added option to control size of streamed chunks. Also added + option for the client to receive responses in binary format. + 21-05-2008 - * Fixed bug in reading some options from the ibrowse.conf file. Reported by Erik Reitsma on the erlyaws mailing list * Fixed bug when cleaning up closing connections --- NEW FILE: Makefile --- include vsn.mk all: (cd src ; make) clean: (cd src ; make clean) install: all mkdir -p $(DESTDIR)/lib/ibrowse-$(IBROWSE_VSN)/ cp -r ebin $(DESTDIR)/lib/ibrowse-$(IBROWSE_VSN)/ Index: vsn.mk =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/vsn.mk,v retrieving revision 1.14 retrieving revision 1.15 diff -u -d -r1.14 -r1.15 --- vsn.mk 21 May 2008 15:28:11 -0000 1.14 +++ vsn.mk 1 Jul 2009 22:43:19 -0000 1.15 @@ -1,2 +1,2 @@ -IBROWSE_VSN = 1.4.1 +IBROWSE_VSN = 1.5.0 |
|
From: Chandru <cha...@us...> - 2009-07-01 22:44:24
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse/ebin In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv26761/ebin Modified Files: ibrowse.app Log Message: ibrowse-1.5.0 - Use binaries internally. Lots of other changes. See README Index: ibrowse.app =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/ebin/ibrowse.app,v retrieving revision 1.2 retrieving revision 1.3 diff -u -d -r1.2 -r1.3 --- ibrowse.app 27 Mar 2008 01:35:49 -0000 1.2 +++ ibrowse.app 1 Jul 2009 22:43:19 -0000 1.3 @@ -1,6 +1,6 @@ {application, ibrowse, [{description, "HTTP client application"}, - {vsn, "1.4"}, + {vsn, "1.5.0"}, {modules, [ ibrowse, ibrowse_http_client, ibrowse_app, |
|
From: Chandru <cha...@us...> - 2009-07-01 22:44:14
|
Update of /cvsroot/jungerl/jungerl/lib/ibrowse/src In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv26761/src Modified Files: ibrowse.erl ibrowse_http_client.erl ibrowse_lb.erl ibrowse_test.erl Log Message: ibrowse-1.5.0 - Use binaries internally. Lots of other changes. See README Index: ibrowse.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse.erl,v retrieving revision 1.7 retrieving revision 1.8 diff -u -d -r1.7 -r1.8 --- ibrowse.erl 21 May 2008 15:28:11 -0000 1.7 +++ ibrowse.erl 1 Jul 2009 22:43:19 -0000 1.8 @@ -6,8 +6,8 @@ %%% Created : 11 Oct 2003 by Chandrashekhar Mullaparthi <cha...@t-...> %%%------------------------------------------------------------------- %% @author Chandrashekhar Mullaparthi <chandrashekhar dot mullaparthi at gmail dot com> -%% @copyright 2005-2008 Chandrashekhar Mullaparthi -%% @version 1.4 +%% @copyright 2005-2009 Chandrashekhar Mullaparthi +%% @version 1.5.0 %% @doc The ibrowse application implements an HTTP 1.1 client. This %% module implements the API of the HTTP client. There is one named %% process called 'ibrowse' which assists in load balancing and maintaining configuration. There is one load balancing process per unique webserver. There is @@ -96,6 +96,7 @@ trace_off/0, trace_on/2, trace_off/2, + all_trace_off/0, show_dest_status/2 ]). @@ -105,8 +106,6 @@ -import(ibrowse_lib, [ parse_url/1, - printable_date/0, - get_value/2, get_value/3, do_trace/2 ]). @@ -114,6 +113,7 @@ -record(state, {trace = false}). -include("ibrowse.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). -define(DEF_MAX_SESSIONS,10). -define(DEF_MAX_PIPELINE_SIZE,10). @@ -170,7 +170,7 @@ %% For a description of SSL Options, look in the ssl manpage. If the %% HTTP Version to use is not specified, the default is 1.1. %% <br/> -%% <p>The <code>host_header</code> is useful in the case where ibrowse is +%% <p>The <code>host_header</code> option is useful in the case where ibrowse is %% connecting to a component such as <a %% href="http://www.stunnel.org">stunnel</a> which then sets up a %% secure connection to a webserver. In this case, the URL supplied to @@ -188,10 +188,40 @@ %% <li>Whenever an error occurs in the processing of a request, ibrowse will return as much %% information as it has, such as HTTP Status Code and HTTP Headers. When this happens, the response %% is of the form <code>{error, {Reason, {stat_code, StatusCode}, HTTP_headers}}</code></li> +%% +%% <li>The <code>inactivity_timeout</code> option is useful when +%% dealing with large response bodies and/or slow links. In these +%% cases, it might be hard to estimate how long a request will take to +%% complete. In such cases, the client might want to timeout if no +%% data has been received on the link for a certain time interval.</li> +%% +%% <li> +%% The <code>connect_timeout</code> option is to specify how long the +%% client process should wait for connection establishment. This is +%% useful in scenarios where connections to servers are usually setup +%% very fast, but responses might take much longer compared to +%% connection setup. In such cases, it is better for the calling +%% process to timeout faster if there is a problem (DNS lookup +%% delays/failures, network routing issues, etc). The total timeout +%% value specified for the request will enforced. To illustrate using +%% an example: +%% <code> +%% ibrowse:send_req("http://www.example.com/cgi-bin/request", [], get, [], [{connect_timeout, 100}], 1000). +%% </code> +%% In the above invocation, if the connection isn't established within +%% 100 milliseconds, the request will fail with +%% <code>{error, conn_failed}</code>.<br/> +%% If connection setup succeeds, the total time allowed for the +%% request to complete will be 1000 milliseconds minus the time taken +%% for connection setup. +%% </li> %% </ul> +%% %% @spec send_req(Url::string(), Headers::headerList(), Method::method(), Body::body(), Options::optionList()) -> response() %% optionList() = [option()] %% option() = {max_sessions, integer()} | +%% {response_format,response_format()}| +%% {stream_chunk_size, integer()} | %% {max_pipeline_size, integer()} | %% {trace, boolean()} | %% {is_ssl, boolean()} | @@ -210,8 +240,10 @@ %% {stream_to, process()} | %% {http_vsn, {MajorVsn, MinorVsn}} | %% {host_header, string()} | +%% {inactivity_timeout, integer()} | +%% {connect_timeout, integer()} | %% {transfer_encoding, {chunked, ChunkSize}} -%% +%% %% process() = pid() | atom() %% username() = string() %% password() = string() @@ -219,7 +251,7 @@ %% ChunkSize = integer() %% srtf() = boolean() | filename() %% filename() = string() -%% +%% response_format() = list | binary send_req(Url, Headers, Method, Body, Options) -> send_req(Url, Headers, Method, Body, Options, 30000). @@ -230,7 +262,8 @@ send_req(Url, Headers, Method, Body, Options, Timeout) -> case catch parse_url(Url) of #url{host = Host, - port = Port} = Parsed_url -> + port = Port, + protocol = Protocol} = Parsed_url -> Lb_pid = case ets:lookup(ibrowse_lb, {Host, Port}) of [] -> get_lb_pid(Parsed_url); @@ -241,9 +274,10 @@ Max_pipeline_size = get_max_pipeline_size(Host, Port, Options), Options_1 = merge_options(Host, Port, Options), {SSLOptions, IsSSL} = - case get_value(is_ssl, Options_1, false) of + case (Protocol == https) orelse + get_value(is_ssl, Options_1, false) of false -> {[], false}; - true -> {get_value(ssl_options, Options_1), true} + true -> {get_value(ssl_options, Options_1, []), true} end, case ibrowse_lb:spawn_connection(Lb_pid, Parsed_url, Max_sessions, @@ -310,16 +344,28 @@ do_send_req(Conn_Pid, Parsed_url, Headers, Method, Body, Options, Timeout) -> case catch ibrowse_http_client:send_req(Conn_Pid, Parsed_url, - Headers, Method, Body, + Headers, Method, ensure_bin(Body), Options, Timeout) of {'EXIT', {timeout, _}} -> {error, req_timedout}; {'EXIT', Reason} -> {error, {'EXIT', Reason}}; + {ok, St_code, Headers, Body} = Ret when is_binary(Body) -> + case get_value(response_format, Options, list) of + list -> + {ok, St_code, Headers, binary_to_list(Body)}; + binary -> + Ret + end; Ret -> Ret end. +ensure_bin(L) when is_list(L) -> + list_to_binary(L); +ensure_bin(B) when is_binary(B) -> + B. + %% @doc Creates a HTTP client process to the specified Host:Port which %% is not part of the load balancing pool. This is useful in cases %% where some requests to a webserver might take a long time whereas @@ -389,17 +435,25 @@ %% @doc Turn tracing on for all connections to the specified HTTP %% server. Host is whatever is specified as the domain name in the URL -%% @spec trace_on(Host, Port) -> term() +%% @spec trace_on(Host, Port) -> ok %% Host = string() %% Port = integer() trace_on(Host, Port) -> - ibrowse ! {trace, true, Host, Port}. + ibrowse ! {trace, true, Host, Port}, + ok. %% @doc Turn tracing OFF for all connections to the specified HTTP %% server. -%% @spec trace_off(Host, Port) -> term() +%% @spec trace_off(Host, Port) -> ok trace_off(Host, Port) -> - ibrowse ! {trace, false, Host, Port}. + ibrowse ! {trace, false, Host, Port}, + ok. + +%% @doc Turn Off ALL tracing +%% @spec all_trace_off() -> ok +all_trace_off() -> + ibrowse ! all_trace_off, + ok. %% @doc Shows some internal information about load balancing to a %% specified Host:Port. Info about workers spawned using @@ -577,6 +631,30 @@ %% {noreply, State, Timeout} | %% {stop, Reason, State} (terminate/2 is called) %%-------------------------------------------------------------------- +handle_info(all_trace_off, State) -> + Mspec = [{{ibrowse_conf,{trace,'$1','$2'},true},[],[{{'$1','$2'}}]}], + Trace_on_dests = ets:select(ibrowse_conf, Mspec), + Fun = fun(#lb_pid{host_port = {H, P}, pid = Pid}, _) -> + case lists:member({H, P}, Trace_on_dests) of + false -> + ok; + true -> + catch Pid ! {trace, false} + end; + (#client_conn{key = {H, P, Pid}}, _) -> + case lists:member({H, P}, Trace_on_dests) of + false -> + ok; + true -> + catch Pid ! {trace, false} + end; + (_, Acc) -> + Acc + end, + ets:foldl(Fun, undefined, ibrowse_lb), + ets:select_delete(ibrowse_conf, [{{ibrowse_conf,{trace,'$1','$2'},true},[],['true']}]), + {noreply, State}; + handle_info({trace, Bool}, State) -> put(my_trace_flag, Bool), {noreply, State}; Index: ibrowse_lb.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_lb.erl,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- ibrowse_lb.erl 27 Mar 2008 01:36:21 -0000 1.1 +++ ibrowse_lb.erl 1 Jul 2009 22:43:19 -0000 1.2 @@ -39,13 +39,6 @@ max_pipeline_size, num_cur_sessions = 0}). --import(ibrowse_lib, [ - parse_url/1, - printable_date/0, - get_value/3 - ]). - - -include("ibrowse.hrl"). %%==================================================================== Index: ibrowse_http_client.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_http_client.erl,v retrieving revision 1.18 retrieving revision 1.19 diff -u -d -r1.18 -r1.19 --- ibrowse_http_client.erl 21 May 2008 15:28:11 -0000 1.18 +++ ibrowse_http_client.erl 1 Jul 2009 22:43:19 -0000 1.19 @@ -38,28 +38,33 @@ -include("ibrowse.hrl"). --record(state, {host, port, +-record(state, {host, port, use_proxy = false, proxy_auth_digest, - ssl_options = [], is_ssl = false, socket, - reqs=queue:new(), cur_req, status=idle, http_status_code, - reply_buffer=[], rep_buf_size=0, recvd_headers=[], + ssl_options = [], is_ssl = false, socket, + reqs=queue:new(), cur_req, status=idle, http_status_code, [...1108 lines suppressed...] + +flatten([H | _] = L) when is_integer(H) -> + L; +flatten([H | _] = L) when is_list(H) -> + lists:flatten(L); +flatten([]) -> + []. + +get_stream_chunk_size(Options) -> + case lists:keysearch(stream_chunk_size, 1, Options) of + {value, {_, V}} when V > 0 -> + V; + _ -> + ?DEFAULT_STREAM_CHUNK_SIZE + end. + +get_inac_timeout(#state{cur_req = #request{options = Opts}}) -> + get_value(inactivity_timeout, Opts, infinity); +get_inac_timeout(#state{cur_req = undefined}) -> + infinity. Index: ibrowse_test.erl =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/ibrowse/src/ibrowse_test.erl,v retrieving revision 1.3 retrieving revision 1.4 diff -u -d -r1.3 -r1.4 --- ibrowse_test.erl 21 May 2008 15:28:11 -0000 1.3 +++ ibrowse_test.erl 1 Jul 2009 22:43:19 -0000 1.4 @@ -11,14 +11,16 @@ do_send_req/2, unit_tests/0, unit_tests/1, + unit_tests_1/2, drv_ue_test/0, drv_ue_test/1, ue_test/0, - ue_test/1 + ue_test/1, + verify_chunked_streaming/0, + verify_chunked_streaming/1, + i_do_async_req_list/4 ]). --import(ibrowse_lib, [printable_date/0]). - %% Use ibrowse:set_max_sessions/3 and ibrowse:set_max_pipeline_size/3 to %% tweak settings before running the load test. The defaults are 10 and 10. load_test(Url, NumWorkers, NumReqsPerWorker) when is_list(Url), @@ -46,7 +48,7 @@ log_msg("End time : ~1000.p~n", [calendar:now_to_local_time(End_time)]), Elapsed_time_secs = trunc(timer:now_diff(End_time, Start_time) / 1000000), log_msg("Elapsed : ~p~n", [Elapsed_time_secs]), - log_msg("Reqs/sec : ~p~n", [(NumWorkers*NumReqsPerWorker) / Elapsed_time_secs]), + log_msg("Reqs/sec : ~p~n", [round(trunc((NumWorkers*NumReqsPerWorker) / Elapsed_time_secs))]), dump_errors(). init_results() -> @@ -88,7 +90,7 @@ do_wait() end end. - + do_send_req(Url, NumReqs) -> do_send_req_1(Url, NumReqs). @@ -149,7 +151,7 @@ -define(TEST_LIST, [{"http://intranet/messenger", get}, {"http://www.google.co.uk", get}, {"http://www.google.com", get}, - {"http://www.google.com", options}, + {"http://www.google.com", options}, {"http://www.sun.com", get}, {"http://www.oracle.com", get}, {"http://www.bbc.co.uk", get}, @@ -172,26 +174,129 @@ {"http://jigsaw.w3.org/HTTP/400/toolong/", get}, {"http://jigsaw.w3.org/HTTP/300/", get}, {"http://jigsaw.w3.org/HTTP/Basic/", get, [{basic_auth, {"guest", "guest"}}]}, - {"http://jigsaw.w3.org/HTTP/CL/", get} + {"http://jigsaw.w3.org/HTTP/CL/", get}, + {"http://www.httpwatch.com/httpgallery/chunked/", get} ]). unit_tests() -> unit_tests([]). unit_tests(Options) -> + {Pid, Ref} = erlang:spawn_monitor(?MODULE, unit_tests_1, [self(), Options]), + receive + {done, Pid} -> + ok; + {'DOWN', Ref, _, _, Info} -> + io:format("Test process crashed: ~p~n", [Info]) + after 60000 -> + io:format("Timed out waiting for tests to complete~n", []) + end. + +unit_tests_1(Parent, Options) -> lists:foreach(fun({Url, Method}) -> execute_req(Url, Method, Options); ({Url, Method, X_Opts}) -> execute_req(Url, Method, X_Opts ++ Options) - end, ?TEST_LIST). + end, ?TEST_LIST), + Parent ! {done, self()}. -execute_req(Url, Method) -> - execute_req(Url, Method, []). +verify_chunked_streaming() -> + verify_chunked_streaming([]). + +verify_chunked_streaming(Options) -> + Url = "http://www.httpwatch.com/httpgallery/chunked/", + io:format("URL: ~s~n", [Url]), + io:format("Fetching data without streaming...~n", []), + Result_without_streaming = ibrowse:send_req( + Url, [], get, [], + [{response_format, binary} | Options]), + io:format("Fetching data with streaming as list...~n", []), + Async_response_list = do_async_req_list( + Url, get, [{response_format, list} | Options]), + io:format("Fetching data with streaming as binary...~n", []), + Async_response_bin = do_async_req_list( + Url, get, [{response_format, binary} | Options]), + compare_responses(Result_without_streaming, Async_response_list, Async_response_bin). + +compare_responses({ok, St_code, _, Body}, {ok, St_code, _, Body}, {ok, St_code, _, Body}) -> + success; +compare_responses({ok, St_code, _, Body_1}, {ok, St_code, _, Body_2}, {ok, St_code, _, Body_3}) -> + case Body_1 of + Body_2 -> + io:format("Body_1 and Body_2 match~n", []); + Body_3 -> + io:format("Body_1 and Body_3 match~n", []); + _ when Body_2 == Body_3 -> + io:format("Body_2 and Body_3 match~n", []); + _ -> + io:format("All three bodies are different!~n", []) + end, + io:format("Body_1 -> ~p~n", [Body_1]), + io:format("Body_2 -> ~p~n", [Body_2]), + io:format("Body_3 -> ~p~n", [Body_3]), + fail_bodies_mismatch; +compare_responses(R1, R2, R3) -> + io:format("R1 -> ~p~n", [R1]), + io:format("R2 -> ~p~n", [R2]), + io:format("R3 -> ~p~n", [R3]), + fail. + +%% do_async_req_list(Url) -> +%% do_async_req_list(Url, get). + +%% do_async_req_list(Url, Method) -> +%% do_async_req_list(Url, Method, [{stream_to, self()}, +%% {stream_chunk_size, 1000}]). + +do_async_req_list(Url, Method, Options) -> + {Pid,_} = erlang:spawn_monitor(?MODULE, i_do_async_req_list, + [self(), Url, Method, + Options ++ [{stream_chunk_size, 1000}]]), + io:format("Spawned process ~p~n", [Pid]), + wait_for_resp(Pid). + +wait_for_resp(Pid) -> + receive + {async_result, Pid, Res} -> + Res; + {'DOWN', _, _, Pid, Reason} -> + {'EXIT', Reason}; + {'DOWN', _, _, _, _} -> + wait_for_resp(Pid); + Msg -> + io:format("Recvd unknown message: ~p~n", [Msg]), + wait_for_resp(Pid) + after 10000 -> + {error, timeout} + end. + +i_do_async_req_list(Parent, Url, Method, Options) -> + Res = ibrowse:send_req(Url, [], Method, [], [{stream_to, self()} | Options]), + case Res of + {ibrowse_req_id, Req_id} -> + Result = wait_for_async_resp(Req_id, undefined, undefined, []), + Parent ! {async_result, self(), Result}; + Err -> + Parent ! {async_result, self(), Err} + end. + +wait_for_async_resp(Req_id, Acc_Stat_code, Acc_Headers, Body) -> + receive + {ibrowse_async_headers, Req_id, StatCode, Headers} -> + wait_for_async_resp(Req_id, StatCode, Headers, Body); + {ibrowse_async_response_end, Req_id} -> + Body_1 = list_to_binary(lists:reverse(Body)), + {ok, Acc_Stat_code, Acc_Headers, Body_1}; + {ibrowse_async_response, Req_id, Data} -> + wait_for_async_resp(Req_id, Acc_Stat_code, Acc_Headers, [Data | Body]); + Err -> + {ok, Acc_Stat_code, Acc_Headers, Err} + end. execute_req(Url, Method, Options) -> - io:format("~s, ~p: ", [Url, Method]), + io:format("~7.7w, ~50.50s: ", [Method, Url]), Result = (catch ibrowse:send_req(Url, [], Method, [], Options)), - case Result of + case Result of {ok, SCode, _H, _B} -> io:format("Status code: ~p~n", [SCode]); Err -> |
|
From: Serge A. <as...@us...> - 2009-02-03 02:57:49
|
Update of /cvsroot/jungerl/jungerl/lib/otp.net/Otp In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv24832 Added Files: dotnet_keyfile.snk Log Message: Added missing keyfile --- NEW FILE: dotnet_keyfile.snk --- (This appears to be a binary file; contents omitted.) |
|
From: Serge A. <as...@us...> - 2009-01-31 16:37:41
|
Update of /cvsroot/jungerl/jungerl/lib/otp.net In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv30329 Added Files: LICENSE Log Message: Added LICENSE file --- NEW FILE: LICENSE --- The contents of this project are subject to the Erlang Public License, Version 1.1, (the "License"); you may not use this file except in compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved via the world wide web at http://www.erlang.org/. Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. The Initial Developer of the Original Code is Ericsson Utvecklings AB. Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings AB. All Rights Reserved. Converted from Java to C# by Vlad Dumitrescu (vla...@ho...) Contributors: - Paul Shaffer, - Serge Aleynikov (sa...@gm...) |
|
From: Serge A. <as...@us...> - 2009-01-31 00:25:22
|
Update of /cvsroot/jungerl/jungerl/lib/otp.net/Otp/Erlang In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv9154/Erlang Modified Files: List.cs Tuple.cs Log Message: Version 0.5 Index: Tuple.cs =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/otp.net/Otp/Erlang/Tuple.cs,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- Tuple.cs 8 May 2004 20:35:59 -0000 1.1 +++ Tuple.cs 31 Jan 2009 00:25:15 -0000 1.2 @@ -39,7 +39,7 @@ * * @exception C#.lang.IllegalArgumentException if the array is * empty (null). - **/ + * public Tuple(Object elem) { if (elem == null) @@ -49,7 +49,7 @@ this.elems = new Object[1]; elems[0] = elem; } - + */ /* * Create a tuple from an array of terms. * @@ -57,11 +57,11 @@ * * @exception C#.lang.IllegalArgumentException if the array is * empty (null) or contains null elements. - **/ + */ public Tuple(Object[] elems):this(elems, 0, elems.Length) { } - + /* * Create a tuple from an array of terms. * @@ -71,7 +71,7 @@ * * @exception C#.lang.IllegalArgumentException if the array is * empty (null) or contains null elements. - **/ + */ public Tuple(Object[] elems, int start, int count) { if ((elems == null) || (count < 1)) @@ -94,9 +94,48 @@ } } } - + + /* + * Create a tuple from an array of terms. + * + * @param elems the array of terms to create the tuple from. + * @param start the offset of the first term to insert. + * @param count the number of terms to insert. + * + * @exception C#.lang.IllegalArgumentException if the array is + * empty (null) or contains null elements. + **/ + public Tuple(params System.Object[] elems) + { + if (elems == null) + elems = new Object[] {}; + else + { + this.elems = new Object[elems.Length]; + for (int i = 0; i < elems.Length; i++) + { + if (elems[i] == null) + throw new System.ArgumentException("Tuple element cannot be null (element" + i + ")"); + else + { + System.Object o = elems[i]; + if (o is int) this.elems[i] = new Int((int)o); + else if (o is string) this.elems[i] = new String((string)o); + else if (o is float) this.elems[i] = new Double((float)o); + else if (o is double) this.elems[i] = new Double((double)o); + else if (o is Erlang.Object) this.elems[i] = (Erlang.Object)o; + //else if (o is BigInteger) this.elems[i] = (BigInteger)o; + else if (o is uint) this.elems[i] = new UInt((int)o); + else if (o is short) this.elems[i] = new Short((short)o); + else if (o is ushort) this.elems[i] = new UShort((short)o); + else + throw new System.ArgumentException("Unknown type of element[" + i + "]: " + o.GetType().ToString()); + } + } + } + } /* - * Create a tuple from a stream containing an tuple encoded in Erlang + * Create a tuple from a stream containing an tuple encoded in Erlang * external format. * * @param buf the stream containing the encoded tuple. @@ -157,7 +196,12 @@ return res; } - + public Object this[int index] + { + get { return this.elems[index]; } + set { this.elems[index] = value; } + } + /* * Get the string representation of the tuple. * Index: List.cs =================================================================== RCS file: /cvsroot/jungerl/jungerl/lib/otp.net/Otp/Erlang/List.cs,v retrieving revision 1.1 retrieving revision 1.2 diff -u -d -r1.1 -r1.2 --- List.cs 8 May 2004 20:35:59 -0000 1.1 +++ List.cs 31 Jan 2009 00:25:15 -0000 1.2 @@ -49,70 +49,87 @@ * Create a list of characters. * * @param str the characters from which to create the list. - **/ + * public List(System.String str) { - int len = 0; - - if (str != null) - len = str.Length; - - if (len > 0) - { - this.elems = new Object[len]; - - for (int i = 0; i < len; i++) - { - this.elems[i] = new Char(str[i]); - } - } + this.elems = new Object[] { new Erlang.String(str) }; } - /* * Create a list containing one element. * * @param elem the elememet to make the list from. - **/ + * public List(Object elem) { this.elems = new Object[1]; elems[0] = elem; } - + */ + /* - * Create a list from an array of arbitrary Erlang terms. + * Create a list from an array of arbitrary Erlang terms. * * @param elems the array of terms from which to create the list. - **/ - public List(Object[] elems):this(elems, 0, elems.Length) - { - } - + * @param count the number of terms to insert. + */ + public List(Object[] elems): this(elems, 0, elems.Length) + { + } + /* - * Create a list from an array of arbitrary Erlang terms. + * Create a list from an array of arbitrary Erlang terms. * * @param elems the array of terms from which to create the list. * @param start the offset of the first term to insert. * @param count the number of terms to insert. - **/ - public List(Object[] elems, int start, int count) + */ + public List(Object[] elems, int start, int count) { if ((elems != null) && (count > 0)) { this.elems = new Object[count]; - Array.Copy(elems, start, this.elems, 0, count); - } + Array.Copy(elems, 0, this.elems, start, count); + } } - - /* - * Create a list from a stream containing an list encoded in Erlang - * external format. - * - * @param buf the stream containing the encoded list. - * - * @exception DecodeException if the buffer does not - * contain a valid external representation of an Erlang list. - **/ + + /* + * Create a list from an array of arbitrary Erlang terms. + * + * @param elems the array of terms from which to create the list. + **/ + public List(params System.Object[] elems) + { + if ((elems != null) && (elems.Length > 0)) + { + this.elems = new Object[elems.Length]; + + for (int i=0; i < elems.Length; i++) + { + System.Object o = elems[i]; + if (o is int) this.elems[i] = new Int((int)o); + else if (o is string) this.elems[i] = new String((string)o); + else if (o is float) this.elems[i] = new Double((float)o); + else if (o is double) this.elems[i] = new Double((double)o); + else if (o is Erlang.Object) this.elems[i] = (o as Erlang.Object); + //else if (o is BigInteger) this.elems[i] = (BigInteger)o; + else if (o is uint) this.elems[i] = new UInt((int)o); + else if (o is short) this.elems[i] = new Short((short)o); + else if (o is ushort) this.elems[i] = new UShort((short)o); + else + throw new System.ArgumentException("Unknown type of element[" + i + "]: " + o.GetType().ToString()); + } + } + } + + /* + * Create a list from a stream containing an list encoded in Erlang + * external format. + * + * @param buf the stream containing the encoded list. + * + * @exception DecodeException if the buffer does not + * contain a valid external representation of an Erlang list. + **/ public List(OtpInputStream buf) { this.elems = null; @@ -138,7 +155,7 @@ * * @return the number of elements contained in the list. **/ - public virtual int arity() + public int arity() { if (elems == null) return 0; @@ -155,7 +172,7 @@ * @return the requested element, of null if i is not a valid * element index. **/ - public virtual Object elementAt(int i) + public Object elementAt(int i) { if ((i >= arity()) || (i < 0)) return null; @@ -167,7 +184,7 @@ * * @return an array containing all of the list's elements. **/ - public virtual Object[] elements() + public Object[] elements() { if (arity() == 0) return null; @@ -178,7 +195,18 @@ return res; } } - + + public Object this[int index] + { + get { return elementAt(index); } + set { this.elems[index] = value; } + } + + public int Length + { + get { return this.elems.Length; } + } + /* * Get the string representation of the list. * |
|
From: Serge A. <as...@us...> - 2009-01-31 00:21:31
|
Update of /cvsroot/jungerl/jungerl/lib/otp.net/OtpTest/Properties In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv8739/OtpTest/Properties Added Files: AssemblyInfo.cs Log Message: Version 0.5 --- NEW FILE: AssemblyInfo.cs --- (This appears to be a binary file; contents omitted.) |