Thread: [Docutils-checkins] SF.net SVN: docutils:[9403] trunk/docutils (Page 31)

SourceForge Headquarters 1320 Columbia Street Suite 310 San Diego, CA 92101 +1 (858) 422-6466

Revision: 9403
          http://sourceforge.net/p/docutils/code/9403
Author:   milde
Date:     2023-06-24 19:40:12 +0000 (Sat, 24 Jun 2023)
Log Message:
-----------
Input decoding: After detecting a BOM, leave handling it to Python's codecs.

Second part of the announced API changes:

* Don't remove ZWNBSPs in the middle of the source.
* Only remove BOM with `input_encoding`_ values None, '', 'utf-8-sig',
  'utf-16', and 'utf-32'.

Modified Paths:
--------------
    trunk/docutils/docutils/io.py
    trunk/docutils/test/test_io.py

Modified: trunk/docutils/docutils/io.py
===================================================================

--- trunk/docutils/docutils/io.py	2023-06-23 16:55:30 UTC (rev 9402)
+++ trunk/docutils/docutils/io.py	2023-06-24 19:40:12 UTC (rev 9403)
@@ -176,10 +176,7 @@
             try:
                 decoded = str(data, enc, self.error_handler)
                 self.successful_encoding = enc
-                # Return decoded, removing BOM and other ZWNBSPs.
-                # TODO: only remove BOM (ZWNBSP at start of data)
-                #       and only if 'self.encoding' is None. (API change)
-                return decoded.replace('\ufeff', '')
+                return decoded
             except (UnicodeError, LookupError) as err:
                 # keep exception instance for use outside of the "for" loop.
                 error = err
@@ -191,9 +188,12 @@
     coding_slug = re.compile(br"coding[:=]\s*([-\w.]+)")
     """Encoding declaration pattern."""
 
-    byte_order_marks = ((codecs.BOM_UTF8, 'utf-8'),
-                        (codecs.BOM_UTF16_BE, 'utf-16-be'),
-                        (codecs.BOM_UTF16_LE, 'utf-16-le'),)
+    byte_order_marks = ((codecs.BOM_UTF32_BE, 'utf-32'),
+                        (codecs.BOM_UTF32_LE, 'utf-32'),
+                        (codecs.BOM_UTF8, 'utf-8-sig'),
+                        (codecs.BOM_UTF16_BE, 'utf-16'),
+                        (codecs.BOM_UTF16_LE, 'utf-16'),
+                        )
     """Sequence of (start_bytes, encoding) tuples for encoding detection.
     The first bytes of input data are checked against the start_bytes strings.
     A match indicates the given encoding."""
@@ -384,7 +384,7 @@
         :Parameters:
             - `source`: either a file-like object (which is read directly), or
               `None` (which implies `sys.stdin` if no `source_path` given).
-            - `source_path`: a path to a file, which is opened and then read.
+            - `source_path`: a path to a file, which is opened for reading.
             - `encoding`: the expected text encoding of the input file.
             - `error_handler`: the encoding error handler to use.
             - `autoclose`: close automatically after read (except when
@@ -419,7 +419,7 @@
 
     def read(self):
         """
-        Read and decode a single file and return the data (Unicode string).
+        Read and decode a single file, return as `str`.
         """
         if not self.encoding and hasattr(self.source, 'buffer'):
             # read as binary data
@@ -436,7 +436,7 @@
 
     def readlines(self):
         """
-        Return lines of a single file as list of Unicode strings.
+        Return lines of a single file as list of strings.
         """
         return self.read().splitlines(True)
 

Modified: trunk/docutils/test/test_io.py
===================================================================
--- trunk/docutils/test/test_io.py	2023-06-23 16:55:30 UTC (rev 9402)
+++ trunk/docutils/test/test_io.py	2023-06-24 19:40:12 UTC (rev 9403)
@@ -5,7 +5,7 @@
 # Copyright: This module has been placed in the public domain.
 
 """
-Test module for io.py.
+Test module for `docutils.io`.
 """
 import os.path
 
@@ -91,18 +91,22 @@
 
 class InputTests(unittest.TestCase):
 
-    def test_bom(self):
+    def test_bom_handling(self):
         # Provisional:
-        input = du_io.StringInput(source=b'\xef\xbb\xbf foo \xef\xbb\xbf bar')
-        # Assert BOM is gone.
-        # TODO: only remove BOM (ZWNBSP at start of data)
-        self.assertEqual(input.read(), ' foo  bar')
-        # Unicode input is left unchanged:
-        input = du_io.StringInput(source='\ufeff foo \ufeff bar')
-        # Assert ZWNBSPs are still there.
-        self.assertEqual(input.read(), '\ufeff foo \ufeff bar')
+        # default input encoding will change to UTF-8 in Docutils 0.22
+        source = '\ufeffdata\n\ufeff blah\n'
+        expected = 'data\n\ufeff blah\n'  # only leading ZWNBSP removed
+        input = du_io.StringInput(source=source.encode('utf-16-be'))
+        self.assertEqual(input.read(), expected)
+        input = du_io.StringInput(source=source.encode('utf-16-le'))
+        self.assertEqual(input.read(), expected)
+        input = du_io.StringInput(source=source.encode('utf-8'))
+        self.assertEqual(input.read(), expected)
+        # With `str` input all ZWNBSPs are still there.
+        input = du_io.StringInput(source=source)
+        self.assertEqual(input.read(), source)
 
-    def test_coding_slug(self):
+    def test_encoding_declaration(self):
         input = du_io.StringInput(source=b"""\
 .. -*- coding: ascii -*-
 data
@@ -125,16 +129,6 @@
 """)
         self.assertNotEqual(input.successful_encoding, 'ascii')
 
-    def test_bom_detection(self):
-        source = '\ufeffdata\nblah\n'
-        expected = 'data\nblah\n'
-        input = du_io.StringInput(source=source.encode('utf-16-be'))
-        self.assertEqual(input.read(), expected)
-        input = du_io.StringInput(source=source.encode('utf-16-le'))
-        self.assertEqual(input.read(), expected)
-        input = du_io.StringInput(source=source.encode('utf-8'))
-        self.assertEqual(input.read(), expected)
-
     def test_readlines(self):
         input = du_io.FileInput(
             source_path=os.path.join(DATA_ROOT, 'include.txt'))
@@ -161,7 +155,7 @@
         uniinput = du_io.Input(encoding='unicode')
         # keep unicode instances as-is
         self.assertEqual(uniinput.decode('ja'), 'ja')
-        # raise AssertionError if data is not an unicode string
+        # raise AssertionError if data is not a `str` instance
         self.assertRaises(AssertionError, uniinput.decode, b'ja')
 
 

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.





Thread: [Docutils-checkins] SF.net SVN: docutils:[9403] trunk/docutils (Page 31)

docutils-checkins