Webware for Python

Author: chrisz
Date: Sat Jul 16 05:49:47 2011
New Revision: 8162

Log:
WebUtils.FieldStorage did not work as before with Python 2.6 and newer.

Added:
   Webware/trunk/WebKit/Testing/FieldStorage.py
Modified:
   Webware/trunk/WebKit/Docs/RelNotes-X.Y.phtml
   Webware/trunk/WebKit/HTTPRequest.py
   Webware/trunk/WebKit/Testing/TestCases.data
   Webware/trunk/WebKit/Tests/twill/Testing.twill
   Webware/trunk/WebUtils/FieldStorage.py

Modified: Webware/trunk/WebKit/Docs/RelNotes-X.Y.phtml
==============================================================================
--- Webware/trunk/WebKit/Docs/RelNotes-X.Y.phtml	Sat Jul 16 05:34:03 2011	(r8161)
+++ Webware/trunk/WebKit/Docs/RelNotes-X.Y.phtml	Sat Jul 16 05:49:47 2011	(r8162)
@@ -64,6 +64,14 @@
   isn't necessary here any more).</li>
   <li>For some kinds of bad URLs, the URLParser raised an AssertionError
   instead of HTTPNotFound.</li>
+  <li>The modified <code>FieldStorage</code> class used by WebKit did not work
+  as before with Python 2.6 and newer. These versions already add query string
+  parameters to the fields passed via POST, but differently to how we did this
+  before. In WebKit, POST parameters used to completely override query string
+  parameters, the values were not appended to possibly existing values in the
+  query string. In order to stay compatible and because it seems to be more
+  reasonable, we restored the old behavior even for newer versions of Python
+  (thanks to Fionn Behrens for bringing this up).</li>
 </ul>
 
 <a name="Deprecations"></a><h2>Deprecations</h2>

Modified: Webware/trunk/WebKit/HTTPRequest.py
==============================================================================
--- Webware/trunk/WebKit/HTTPRequest.py	Sat Jul 16 05:34:03 2011	(r8161)
+++ Webware/trunk/WebKit/HTTPRequest.py	Sat Jul 16 05:49:47 2011	(r8162)
@@ -36,7 +36,6 @@
             self._fields = FieldStorage.FieldStorage(
                 self._input, environ=self._environ,
                 keep_blank_values=True, strict_parsing=False)
-            self._fields.parse_qs()
             self._cookies = Cookie()
             if 'HTTP_COOKIE' in self._environ:
                 # Protect the loading of cookies with an exception handler,

Added: Webware/trunk/WebKit/Testing/FieldStorage.py
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ Webware/trunk/WebKit/Testing/FieldStorage.py	Sat Jul 16 05:49:47 2011	(r8162)
@@ -0,0 +1,107 @@
+
+from WebKit.SidebarPage import SidebarPage
+
+
+class FieldStorage(SidebarPage):
+    """Test the FieldStorage class.
+
+    Since WebKit 0.6, WebKit uses a modified FieldStorage class that
+    parses GET parameters even in a POST request. However, the parameter
+    names must be different. A GET parameters with the same name as a
+    POST parameter is ignored, these values are not appended. In other words,
+    POST parameters always override GET parameters with the same name.
+
+    """
+
+    def cornerTitle(self):
+        return 'Testing'
+
+    def writeContent(self):
+        request = self.request()
+        method = request.method()
+        fields = request.fields()
+        writeln = self.writeln
+        if method == 'GET' and not fields:
+            get_fields = [('getfield1', 'getvalue1'),
+                ('getfield2', 'getvalue21'), ('getfield2', 'getvalue22'),
+                ('dualfield1', 'getdual1'),
+                ('dualfield2', 'getdual21'), ('dualfield2', 'getdual22'),
+                ('getempty', '')]
+            post_fields = [('postfield1', 'postvalue1'),
+                ('postfield2', 'postvalue21'), ('postfield2', 'postvalue22'),
+                ('dualfield1', 'postdual1'),
+                ('dualfield2', 'postdual21'), ('dualfield2', 'postdual22'),
+                ('postempty', '')]
+            writeln('<p>The WebKit FieldStorage class can be tested here.</p>')
+            writeln('<form action="FieldStorage?%s" method="POST">'
+                % '&amp;'.join('%s=%s' % field for field in get_fields))
+            writeln('<p>Please press the button to run the test:')
+            for field in post_fields:
+                writeln('<input type="hidden" name="%s" value="%s">' % field)
+            writeln('<input type="submit" name="testbutton" value="Submit">')
+            writeln('</p></form>')
+        else:
+            errors = []
+            error = errors.append
+            if method != 'POST':
+                error('The method is %s instead of POST' % method)
+            if len(fields) != 9:
+                error('We got %d instead of 9 fields' % len(fields))
+            if not request.hasField('testbutton'):
+                error('We did not get the submit button')
+            elif request.field('testbutton') != 'Submit':
+                error('The submit button field got a wrong value')
+            if not request.hasField('getempty'):
+                error('We did not get the empty GET parameter')
+            elif request.field('getempty') != '':
+                error('The empty GET field got a non-empty value')
+            if not request.hasField('postempty'):
+                error('We did not get the empty POST parameter')
+            elif request.field('postempty') != '':
+                error('The empty POST field got a non-empty value')
+            if not request.hasField('getfield1'):
+                error('The first GET parameter has not been passed')
+            elif request.field('getfield1') != 'getvalue1':
+                error('The first GET field got a wrong value')
+            if not request.hasField('postfield1'):
+                error('The first POST parameter has not been passed')
+            elif request.field('postfield1') != 'postvalue1':
+                error('The first POST field got a wrong value')
+            if not request.hasField('getfield2'):
+                error('The second GET parameter has not been passed')
+            elif request.field('getfield2') != ['getvalue21', 'getvalue22']:
+                error('The second GET field got a wrong value')
+            if not request.hasField('postfield2'):
+                error('The second POST parameter has not been passed')
+            elif request.field('postfield2') != ['postvalue21', 'postvalue22']:
+                error('The second POST field got a wrong value')
+            if not request.hasField('dualfield1'):
+                error('The first dual parameter has not been passed')
+            elif request.field('dualfield1') == 'getdual1':
+                error('The first dual field was not overridden via POST')
+            elif request.field('dualfield1') in (
+                    ['getdual1', 'postdual1'], ['postdual1', 'getdual1']):
+                error('The first dual field'
+                    ' was extended instead of overridden via POST')
+            elif request.field('dualfield1') != 'postdual1':
+                error('The first dual field got a wrong value')
+            if not request.hasField('dualfield2'):
+                error('The second dual parameter has not been passed')
+            elif request.field('dualfield2') == ['getdual21', 'getdual22']:
+                error('The second dual field was not overridden via POST')
+            elif request.field('dualfield2') in (
+                    ['getdual21', 'getdual22', 'postdual21', 'postdual22'],
+                    ['postdual21', 'postdual22', 'getdual21', 'getdual22']):
+                error('The second dual field'
+                    ' was extended instead of overridden via POST')
+            elif request.field('dualfield2') != ['postdual21', 'postdual22']:
+                error('The second dual field got a wrong value')
+            if errors:
+                writeln('<p>FieldStorage does <b>not</b> work as expected:</p>')
+                writeln('<ul>')
+                for error in errors:
+                    writeln('<li>%s.</li>' % error)
+                writeln('</ul>')
+            else:
+                writeln('<p>Everything ok, FieldStorage works as expected.</p>')
+            writeln('<p><a href="./">Back to the test cases overview.</a></p>')

Modified: Webware/trunk/WebKit/Testing/TestCases.data
==============================================================================
--- Webware/trunk/WebKit/Testing/TestCases.data	Sat Jul 16 05:34:03 2011	(r8161)
+++ Webware/trunk/WebKit/Testing/TestCases.data	Sat Jul 16 05:49:47 2011	(r8162)
@@ -10,23 +10,23 @@
 /Welcome.py \
 /Examples/Welcome \
 /Examples/ \
-/Examples/Welcome.py   --> Show /Examples/Welcome
+/Examples/Welcome.py --> Show /Examples/Welcome
 /Admin/ \
 /Admin/Main --> Show admin pages
 
 # Redirects
-                       --> Redirect to /
-/Admin                 --> Redirect to /Admin/
-/Examples              --> Redirect to /Examples/
+ --> Redirect to /
+/Admin --> Redirect to /Admin/
+/Examples --> Redirect to /Examples/
 
 # Display files
 /Testing/Dir/File.html --> Display the file
 /Testing/Dir \
-/Testing/Dir/          --> Display the index file
+/Testing/Dir/ --> Display the index file
 
 # Bad: Slashes after files
 /Welcome/ \
-/Examples/Welcome/     --> Error 404: Not Found<br>if ExtraPathInfo is not set
+/Examples/Welcome/ --> Error 404: Not Found<br>if ExtraPathInfo is not set
 
 # Bad: Unknown URLs (wrong name, slashes after files)
 /BadURL \
@@ -52,6 +52,9 @@
 /Testing/Forward2 --> Dir/Forward2Target
 /Testing/Dir/Forward3 --> Forward3Target
 
+# Fields
+/Testing/FieldStorage --> Test of WebUtils.FieldStorage
+
 # Cookies
 /Testing/SetCookie --> Test of HTTPResponse.setCookie
 

Modified: Webware/trunk/WebKit/Tests/twill/Testing.twill
==============================================================================
--- Webware/trunk/WebKit/Tests/twill/Testing.twill	Sat Jul 16 05:34:03 2011	(r8161)
+++ Webware/trunk/WebKit/Tests/twill/Testing.twill	Sat Jul 16 05:49:47 2011	(r8162)
@@ -160,6 +160,19 @@
 notfind "Test cases"
 back
 
+go FieldStorage
+code 200
+title FieldStorage
+find "FieldStorage class can be tested here"
+find "press the button"
+notfind "Test cases"
+submit
+find "Everything ok"
+find "FieldStorage works as expected"
+follow "Back to the test cases overview."
+code 200
+find "Test cases"
+
 go SetCookie
 code 200
 title SetCookie

Modified: Webware/trunk/WebUtils/FieldStorage.py
==============================================================================
--- Webware/trunk/WebUtils/FieldStorage.py	Sat Jul 16 05:34:03 2011	(r8161)
+++ Webware/trunk/WebUtils/FieldStorage.py	Sat Jul 16 05:49:47 2011	(r8162)
@@ -10,22 +10,36 @@
 
 
 class FieldStorage(cgi.FieldStorage):
+    """Modified FieldStorage class for POST requests with query strings.
 
-    def __init__(self, fp=None, headers=None, outerboundary="",
+    Parameters in the query string which have not been sent via POST are
+    appended to the field list. This is different from the behavior of
+    Python versions before 2.6 which completely ignored the query string in
+    POST request, but it's also different from the behavior of the later Python
+    versions which append values from the query string to values sent via POST
+    for parameters with the same name. With other words, our FieldStorage class
+    overrides the query string parameters with the parameters sent via POST.
+
+    """
+
+    def __init__(self, fp=None, headers=None, outerboundary='',
             environ=os.environ, keep_blank_values=False, strict_parsing=False):
-        self._environ = environ
-        cgi.FieldStorage.__init__(self, fp, headers, outerboundary,
-            environ, keep_blank_values, strict_parsing)
-
-    def parse_qs(self):
-        """Explicitly parse the query string, even if it's a POST request."""
-        method = self._environ.get('REQUEST_METHOD', '').upper()
-        if method in ('GET', 'HEAD'):
-            return # bail because cgi.FieldStorage already did this
-        qs = self._environ.get('QUERY_STRING')
-        if not qs:
-            return # bail if no query string
+        method = environ.get('REQUEST_METHOD', 'GET').upper()
+        qs_on_post = method not in ('GET', 'HEAD') and environ.get(
+            'QUERY_STRING', None) or None
+        if qs_on_post:
+            environ['QUERY_STRING'] = ''
+        try:
+            cgi.FieldStorage.__init__(self, fp, headers, outerboundary,
+                environ, keep_blank_values, strict_parsing)
+        finally:
+            if qs_on_post:
+                environ['QUERY_STRING'] = qs_on_post
+        if qs_on_post:
+            self.add_qs(qs_on_post)
 
+    def add_qs(self, qs):
+        """Add all non-existing parameters from the given query string."""
         r = {}
         for name_value in qs.split('&'):
             nv = name_value.split('=', 2)
@@ -40,14 +54,13 @@
                     r[name].append(value)
                 else:
                     r[name] = [value]
-
-        # Only append values that aren't already in the FieldStorage's keys;
-        # this makes POSTed vars override vars on the query string.
-        if not self.list:
+        if self.list is None:
             # This makes sure self.keys() are available, even
             # when valid POST data wasn't encountered.
             self.list = []
         for key in r:
             if key not in self:
+                # Only append values that aren't already the FieldStorage;
+                # this makes POSTed vars override vars on the query string.
                 for value in r[key]:
                     self.list.append(cgi.MiniFieldStorage(key, value))