[Docutils-checkins] SF.net SVN: docutils:[8851] trunk/docutils

[<mi...@us...>]

Revision: 8851
          http://sourceforge.net/p/docutils/code/8851
Author:   milde
Date:     2021-10-12 17:33:24 +0000 (Tue, 12 Oct 2021)
Log Message:
-----------
Fix bug #424 Wrong circular inclusion detection.

The current heuristic for detection of circular includes
fails with nested parsing. Line numbering differs in nested
state-machines, inclusions don't show up in the parent, if
a view is detached (e.g. for table cell content).

The new heuristic uses a "magic" comment instead of line numbers
to keep a log of recursive inclusions.

Add a test case for a false positive and a case for
too late detection in parsed includes.

Adapt tests to (irrelevant) side effects of the patch.

Modified Paths:
--------------
    trunk/docutils/HISTORY.txt
    trunk/docutils/docutils/parsers/rst/directives/misc.py
    trunk/docutils/docutils/parsers/rst/states.py
    trunk/docutils/test/test_parsers/test_rst/test_directives/include10.txt
    trunk/docutils/test/test_parsers/test_rst/test_directives/test_include.py
    trunk/docutils/test/test_parsers/test_rst/test_source_line.py

Modified: trunk/docutils/HISTORY.txt
===================================================================
--- trunk/docutils/HISTORY.txt	2021-10-12 11:42:40 UTC (rev 8850)
+++ trunk/docutils/HISTORY.txt	2021-10-12 17:33:24 UTC (rev 8851)
@@ -21,7 +21,17 @@
   - Don't change a list while looping over it (in 
     document.set_name_id_map()). Thanks to Mickey Endito.
 
+* docutils/parsers/rst/directives/misc.py
 
+  - Fix bug #424 Wrong circular inclusion detection.
+    Use a "magic" comment instead of line numbers
+    to keep a log of recursive inclusions.
+
+* docutils/parsers/rst/states.py
+  
+  -  Use a "magic" comment to update the log of recursive inclusions.
+
+
 Release 0.18b1 (2021-10-05)
 ===========================
 

Modified: trunk/docutils/docutils/parsers/rst/directives/misc.py
===================================================================
--- trunk/docutils/docutils/parsers/rst/directives/misc.py	2021-10-12 11:42:40 UTC (rev 8850)
+++ trunk/docutils/docutils/parsers/rst/directives/misc.py	2021-10-12 17:33:24 UTC (rev 8851)
@@ -52,7 +52,11 @@
                                          'include')
 
     def run(self):
-        """Include a file as part of the content of this reST file."""
+        """Include a file as part of the content of this reST file.
+
+        Depending on the options, the file (or a clipping) is
+        converted to nodes and returned or inserted into the input stream.
+        """
         if not self.state.document.settings.file_insertion_enabled:
             raise self.warning('"%s" directive disabled.' % self.name)
         source = self.state_machine.input_lines.source(
@@ -167,36 +171,33 @@
                                   self.state_machine)
             return codeblock.run()
 
+        # Prevent circular inclusion:
+        clip_options = (startline, endline, before_text, after_text)
+        include_log = self.state.document.include_log
+        # log entries are tuples (<source>, <clip-options>)
+        if not include_log: # new document
+            include_log.append((utils.relative_path(None, source),
+                                (None, None, None, None)))
+        if (path, clip_options) in include_log:
+            raise self.warning('circular inclusion in "%s" directive: %s'
+                % (self.name, ' < '.join([path] + [pth for (pth, opt)
+                                                   in include_log[::-1]])))
+
         if 'parser' in self.options:
+            # parse into a dummy document and return created nodes
             parser = self.options['parser']()
-            # parse into a new (dummy) document
             document = utils.new_document(path, self.state.document.settings)
+            document.include_log = include_log + [(path, clip_options)]
             parser.parse('\n'.join(include_lines), document)
             return document.children
 
-        # include as rST source
+        # Include as rST source:
         #
-        # Prevent circular inclusion:
-        source = utils.relative_path(None, source)
-        clip_options = (startline, endline, before_text, after_text)
-        include_log = self.state.document.include_log
-        if not include_log: # new document:
-            # log entries: (<source>, <clip-options>, <insertion end index>)
-            include_log = [(source, (None,None,None,None), sys.maxsize/2)]
-        # cleanup: we may have passed the last inclusion(s):
-        include_log = [entry for entry in include_log
-                       if entry[2] >= self.lineno]
-        if (path, clip_options) in [(pth, opt)
-                                    for (pth, opt, e) in include_log]:
-            raise self.warning('circular inclusion in "%s" directive: %s'
-                % (self.name, ' < '.join([path] + [pth for (pth, opt, e)
-                                                   in include_log[::-1]])))
-        # include as input
+        # mark end (cf. parsers.rst.states.Body.comment())
+        include_lines += ['', '.. end of inclusion from "%s"' % path]
         self.state_machine.insert_input(include_lines, path)
         # update include-log
-        include_log.append((path, clip_options, self.lineno))
-        self.state.document.include_log = [(pth, opt, e+len(include_lines)+2)
-                                           for (pth, opt, e) in include_log]
+        include_log.append((path, clip_options))
         return []
 
 

Modified: trunk/docutils/docutils/parsers/rst/states.py
===================================================================
--- trunk/docutils/docutils/parsers/rst/states.py	2021-10-12 11:42:40 UTC (rev 8850)
+++ trunk/docutils/docutils/parsers/rst/states.py	2021-10-12 17:33:24 UTC (rev 8851)
@@ -2288,9 +2288,14 @@
         return [error], blank_finish
 
     def comment(self, match):
-        if not match.string[match.end():].strip() \
-              and self.state_machine.is_next_line_blank(): # an empty comment?
-            return [nodes.comment()], 1 # "A tiny but practical wart."
+        if self.state_machine.is_next_line_blank():
+            first_comment_line = match.string[match.end():]
+            if not first_comment_line.strip(): # empty comment
+                return [nodes.comment()], True # "A tiny but practical wart."
+            if first_comment_line.startswith('end of inclusion from "'):
+                # cf. parsers.rst.directives.misc.Include
+                self.document.include_log.pop()
+                return [], True
         indented, indent, offset, blank_finish = \
               self.state_machine.get_first_known_indented(match.end())
         while indented and not indented[-1].strip():
@@ -2704,7 +2709,7 @@
 
     def blank(self, match, context, next_state):
         """End of paragraph."""
-        # NOTE: self.paragraph returns [ node, system_message(s) ], literalnext
+        # NOTE: self.paragraph returns [node, system_message(s)], literalnext
         paragraph, literalnext = self.paragraph(
               context, self.state_machine.abs_line_number() - 1)
         self.parent += paragraph

Modified: trunk/docutils/test/test_parsers/test_rst/test_directives/include10.txt
===================================================================
--- trunk/docutils/test/test_parsers/test_rst/test_directives/include10.txt	2021-10-12 11:42:40 UTC (rev 8850)
+++ trunk/docutils/test/test_parsers/test_rst/test_directives/include10.txt	2021-10-12 17:33:24 UTC (rev 8851)
@@ -71,7 +71,7 @@
 
 :PEP:`-1`
 
-.. unknown:: directive (info still reported with wrong line)
+.. unknown:: directive (TODO: info still reported with wrong line)
 
 ==============  ======
 A simple table  with

Modified: trunk/docutils/test/test_parsers/test_rst/test_directives/test_include.py
===================================================================
--- trunk/docutils/test/test_parsers/test_rst/test_directives/test_include.py	2021-10-12 11:42:40 UTC (rev 8850)
+++ trunk/docutils/test/test_parsers/test_rst/test_directives/test_include.py	2021-10-12 17:33:24 UTC (rev 8851)
@@ -402,8 +402,49 @@
                             includes/sibling/include7.txt
 """],
 ["""\
+Recursive inclusion with specified parser.
+
 In test data
 
+.. include:: %s
+   :parser: rst
+""" % include3,
+"""\
+<document source="test data">
+    <paragraph>
+        Recursive inclusion with specified parser.
+    <paragraph>
+        In test data
+    <paragraph>
+        In include3.txt
+    <paragraph>
+        In includes/include4.txt
+    <paragraph>
+        In includes/include5.txt
+    <paragraph>
+        In includes/more/include6.txt
+    <paragraph>
+        In includes/sibling/include7.txt
+    <literal_block source="test_parsers/test_rst/test_directives/includes/include5.txt" xml:space="preserve">
+        In includes/include5.txt
+        \n\
+        .. include:: more/include6.txt
+    <table>
+        <tgroup cols="2">
+            <colspec colwidth="50">
+            <colspec colwidth="50">
+            <tbody>
+                <row>
+                    <entry>
+                        <paragraph>
+                            In
+                    <entry>
+                        <paragraph>
+                            includes/sibling/include7.txt
+"""],
+["""\
+In test data
+
 Section
 =======
 
@@ -758,7 +799,7 @@
             <paragraph>
                 Unknown directive type "unknown".
             <literal_block xml:space="preserve">
-                .. unknown:: directive (info still reported with wrong line)
+                .. unknown:: directive (TODO: info still reported with wrong line)
         <system_message level="3" line="76" source="%(source)s" type="ERROR">
             <paragraph>
                 Malformed table.
@@ -767,6 +808,8 @@
                 ==============  ======
                 A simple table  with
                 no bottom       border
+                
+                .. end of inclusion from "test_parsers/test_rst/test_directives/include10.txt"
 """ % {'source': reldir(include10), 'nonexistent': reldir(nonexistent),
        'unichr_exception':
        DocutilsTestSupport.exception_data(unichr, int("11111111", 16))[2]
@@ -1223,6 +1266,54 @@
         File "include15.txt": example of rekursive inclusion.
 """ % (reldir(include16), reldir(include15), reldir(include16),
        reldir(include15), reldir(include16))],
+["""\
+Circular inclusion with specified parser.
+
+.. include:: %s
+   :parser: rst
+""" % include15,
+"""\
+<document source="test data">
+    <paragraph>
+        Circular inclusion with specified parser.
+    <paragraph>
+        File "include15.txt": example of rekursive inclusion.
+    <paragraph>
+        File "include16.txt": example of rekursive inclusion.
+    <system_message level="2" line="3" source="%s" type="WARNING">
+        <paragraph>
+            circular inclusion in "include" directive: %s < %s < %s < test data
+        <literal_block xml:space="preserve">
+            .. include:: include15.txt
+    <paragraph>
+        No loop when clipping before the "include" directive:
+    <paragraph>
+        File "include15.txt": example of rekursive inclusion.
+""" % (reldir(include16), reldir(include15),
+       reldir(include16), reldir(include15))],
+["""\
+No circular inclusion.
+
+============================= =============================
+.. include:: data/include.txt .. include:: data/include.txt
+============================= =============================
+""",
+"""\
+<document source="test data">
+    <paragraph>
+        No circular inclusion.
+    <table>
+        <tgroup cols="2">
+            <colspec colwidth="29">
+            <colspec colwidth="29">
+            <tbody>
+                <row>
+                    <entry>
+                        <paragraph>
+                            Some include text.
+                    <entry>
+                        <paragraph>
+                            Some include text."""],
 ]
 
 if __name__ == '__main__':

Modified: trunk/docutils/test/test_parsers/test_rst/test_source_line.py
===================================================================
--- trunk/docutils/test/test_parsers/test_rst/test_source_line.py	2021-10-12 11:42:40 UTC (rev 8850)
+++ trunk/docutils/test/test_parsers/test_rst/test_source_line.py	2021-10-12 17:33:24 UTC (rev 8851)
@@ -174,7 +174,7 @@
             <paragraph internal:line="10" internal:source="test_parsers/test_rst/includes/include14.txt">
                 second item in line 10
     <enumerated_list enumtype="arabic" internal:line="12" internal:source="test_parsers/test_rst/includes/include14.txt" prefix="" suffix=".">
-        <list_item internal:source="internal padding after test_parsers/test_rst/includes/include14.txt">
+        <list_item internal:source="test_parsers/test_rst/includes/include14.txt">
             <paragraph internal:line="12" internal:source="test_parsers/test_rst/includes/include14.txt">
                 enumerated list in line 12
 """],

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.