Menu
▾
▴
[Docutils-checkins] SF.net SVN: docutils:[8670] trunk/docutils
|
From: <mi...@us...> - 2021-04-07 12:04:39
|
Revision: 8670
http://sourceforge.net/p/docutils/code/8670
Author: milde
Date: 2021-04-07 12:04:36 +0000 (Wed, 07 Apr 2021)
Log Message:
-----------
Clean up parser code.
Simplify conditional loading and mock definition for recommonmark.
Do not repeat generic settings in rst parser.
Modified Paths:
--------------
trunk/docutils/docutils/parsers/__init__.py
trunk/docutils/docutils/parsers/recommonmark_wrapper.py
trunk/docutils/docutils/parsers/rst/__init__.py
trunk/docutils/test/test_parsers/test_recommonmark/test_misc.py
trunk/docutils/test/test_parsers/test_rst/test_directives/test_include.py
Modified: trunk/docutils/docutils/parsers/__init__.py
===================================================================
--- trunk/docutils/docutils/parsers/__init__.py 2021-04-06 19:50:57 UTC (rev 8669)
+++ trunk/docutils/docutils/parsers/__init__.py 2021-04-07 12:04:36 UTC (rev 8670)
@@ -15,16 +15,25 @@
class Parser(Component):
-
settings_spec = (
'Generic Parser Options',
None,
- (('Disable the "raw" directives; replaced with a "warning" '
- 'system message.',
+ (('Disable directives that insert the contents of an external file; '
+ 'replaced with a "warning" system message.',
+ ['--no-file-insertion'],
+ {'action': 'store_false', 'default': 1,
+ 'dest': 'file_insertion_enabled',
+ 'validator': frontend.validate_boolean}),
+ ('Enable directives that insert the contents '
+ 'of an external file. (default)',
+ ['--file-insertion-enabled'],
+ {'action': 'store_true'}),
+ ('Disable the "raw" directive; '
+ 'replaced with a "warning" system message.',
['--no-raw'],
{'action': 'store_false', 'default': 1, 'dest': 'raw_enabled',
'validator': frontend.validate_boolean}),
- ('Enable the "raw" directive. Enabled by default.',
+ ('Enable the "raw" directive. (default)',
['--raw-enabled'],
{'action': 'store_true'}),
('Maximal number of characters in an input line. Default 10 000.',
Modified: trunk/docutils/docutils/parsers/recommonmark_wrapper.py
===================================================================
--- trunk/docutils/docutils/parsers/recommonmark_wrapper.py 2021-04-06 19:50:57 UTC (rev 8669)
+++ trunk/docutils/docutils/parsers/recommonmark_wrapper.py 2021-04-07 12:04:36 UTC (rev 8670)
@@ -19,16 +19,14 @@
"""
import docutils.parsers
-from docutils import nodes
+from docutils import nodes, Component
try:
- from recommonmark.parser import CommonMarkParser as _recommonmarkParser
- _recommonmarkParser.supported = ('recommonmark', 'commonmark',
- 'markdown', 'md')
- with_recommonmark = True
+ from recommonmark.parser import CommonMarkParser
+ from recommonmark.transform import AutoStructify
except ImportError as err:
- with_recommonmark = False
- class _recommonmarkParser(docutils.parsers.Parser):
+ CommonMarkParser = None
+ class Parser(docutils.parsers.Parser):
def parse(self, inputstring, document):
error = document.reporter.warning(
'Missing dependency: MarkDown input is processed by a 3rd '
@@ -36,89 +34,101 @@
'"recommonmark" (https://pypi.org/project/recommonmark/).')
document.append(error)
-class Parser(_recommonmarkParser):
- config_section = 'recommonmark parser'
- config_section_dependencies = ('parsers',)
- def parse(self, inputstring, document):
- """Use the upstream parser and clean up afterwards.
- """
- # check for exorbitantly long lines
- for i, line in enumerate(inputstring.split('\n')):
- if len(line) > document.settings.line_length_limit:
- error = document.reporter.error(
- 'Line %d exceeds the line-length-limit.'%(i+1))
+if CommonMarkParser:
+ class Parser(CommonMarkParser):
+ """MarkDown parser based on recommonmark."""
+ # TODO: settings for AutoStructify
+ # settings_spec = docutils.parsers.Parser.settings_spec + (
+ # see https://recommonmark.readthedocs.io/en/latest/#autostructify
+
+ supported = ('recommonmark', 'commonmark',
+ 'markdown', 'md')
+ config_section = 'recommonmark parser'
+ config_section_dependencies = ('parsers',)
+
+ # def get_transforms(self):
+ # return Component.get_transforms(self) + [AutoStructify]
+
+ def parse(self, inputstring, document):
+ """Use the upstream parser and clean up afterwards.
+ """
+ # check for exorbitantly long lines
+ for i, line in enumerate(inputstring.split('\n')):
+ if len(line) > document.settings.line_length_limit:
+ error = document.reporter.error(
+ 'Line %d exceeds the line-length-limit.'%(i+1))
+ document.append(error)
+ return
+
+ # pass to upstream parser
+ try:
+ CommonMarkParser.parse(self, inputstring, document)
+ except Exception as err:
+ error = document.reporter.error('Parsing with "recommonmark" '
+ 'returned the error:\n%s'%err)
document.append(error)
- return
- # pass to upstream parser
- try:
- _recommonmarkParser.parse(self, inputstring, document)
- except Exception as err:
- error = document.reporter.error('Parsing with "recommonmark" '
- 'returned the error:\n%s'%err)
- document.append(error)
+ # Post-Processing
+ # ---------------
- # Post-Processing
- # ---------------
+ # merge adjoining Text nodes:
+ for node in document.traverse(nodes.TextElement):
+ children = node.children
+ i = 0
+ while i+1 < len(children):
+ if (isinstance(children[i], nodes.Text)
+ and isinstance(children[i+1], nodes.Text)):
+ children[i] = nodes.Text(children[i]+children.pop(i+1))
+ children[i].parent = node
+ else:
+ i += 1
- # merge adjoining Text nodes:
- for node in document.traverse(nodes.TextElement):
- children = node.children
- i = 0
- while i+1 < len(children):
- if (isinstance(children[i], nodes.Text)
- and isinstance(children[i+1], nodes.Text)):
- children[i] = nodes.Text(children[i]+children.pop(i+1))
- children[i].parent = node
- else:
- i += 1
+ # add "code" class argument to inline literal (code spans)
+ for node in document.traverse(lambda n: isinstance(n,
+ (nodes.literal, nodes.literal_block))):
+ node['classes'].append('code')
+ # move "language" argument to classes
+ for node in document.traverse(nodes.literal_block):
+ if 'language' in node.attributes:
+ node['classes'].append(node['language'])
+ del node['language']
- # add "code" class argument to inline literal (code spans)
- for node in document.traverse(lambda n: isinstance(n,
- (nodes.literal, nodes.literal_block))):
- node['classes'].append('code')
- # move "language" argument to classes
- for node in document.traverse(nodes.literal_block):
- if 'language' in node.attributes:
- node['classes'].append(node['language'])
- del node['language']
+ # remove empty target nodes
+ for node in document.traverse(nodes.target):
+ # remove empty name
+ node['names'] = [v for v in node['names'] if v]
+ if node.children or [v for v in node.attributes.values() if v]:
+ continue
+ node.parent.remove(node)
- # remove empty target nodes
- for node in document.traverse(nodes.target):
- # remove empty name
- node['names'] = [v for v in node['names'] if v]
- if node.children or [v for v in node.attributes.values() if v]:
- continue
- node.parent.remove(node)
+ # replace raw nodes if raw is not allowed
+ if not document.settings.raw_enabled:
+ for node in document.traverse(nodes.raw):
+ warning = document.reporter.warning('Raw content disabled.')
+ node.parent.replace(node, warning)
- # replace raw nodes if raw is not allowed
- if not document.settings.raw_enabled:
- for node in document.traverse(nodes.raw):
- warning = document.reporter.warning('Raw content disabled.')
- node.parent.replace(node, warning)
+ # fix section nodes
+ for node in document.traverse(nodes.section):
+ # remove spurious IDs (first may be from duplicate name)
+ if len(node['ids']) > 1:
+ node['ids'].pop()
+ # fix section levels
+ section_level = self.get_section_level(node)
+ if node['level'] != section_level:
+ warning = document.reporter.warning(
+ 'Title level inconsistent. Changing from %d to %d.'
+ %(node['level'], section_level),
+ nodes.literal_block('', node[0].astext()))
+ node.insert(1, warning)
+ # remove non-standard attribute "level"
+ del node['level'] # TODO: store the original md level somewhere
- # fix section nodes
- for node in document.traverse(nodes.section):
- # remove spurious IDs (first may be from duplicate name)
- if len(node['ids']) > 1:
- node['ids'].pop()
- # fix section levels
- section_level = self.get_section_level(node)
- if node['level'] != section_level:
- warning = document.reporter.warning(
- 'Title level inconsistent. Changing from %d to %d.'
- %(node['level'], section_level),
- nodes.literal_block('', node[0].astext()))
- node.insert(1, warning)
- # remove non-standard attribute "level"
- del node['level'] # TODO: store the original md level somewhere
-
- def get_section_level(self, node):
- level = 1
- while True:
- node = node.parent
- if isinstance(node, nodes.document):
- return level
- if isinstance(node, nodes.section):
- level += 1
+ def get_section_level(self, node):
+ level = 1
+ while True:
+ node = node.parent
+ if isinstance(node, nodes.document):
+ return level
+ if isinstance(node, nodes.section):
+ level += 1
Modified: trunk/docutils/docutils/parsers/rst/__init__.py
===================================================================
--- trunk/docutils/docutils/parsers/rst/__init__.py 2021-04-06 19:50:57 UTC (rev 8669)
+++ trunk/docutils/docutils/parsers/rst/__init__.py 2021-04-07 12:04:36 UTC (rev 8670)
@@ -84,7 +84,7 @@
supported = ('restructuredtext', 'rst', 'rest', 'restx', 'rtxt', 'rstx')
"""Aliases this parser supports."""
- settings_spec = (
+ settings_spec = docutils.parsers.Parser.settings_spec + (
'reStructuredText Parser Options',
None,
(('Recognize and link to standalone PEP references (like "PEP 258").',
@@ -115,28 +115,6 @@
('Leave spaces before footnote references.',
['--leave-footnote-reference-space'],
{'action': 'store_false', 'dest': 'trim_footnote_reference_space'}),
- ('Disable directives that insert the contents of external file '
- '("include" & "raw"); replaced with a "warning" system message.',
- ['--no-file-insertion'],
- {'action': 'store_false', 'default': 1,
- 'dest': 'file_insertion_enabled',
- 'validator': frontend.validate_boolean}),
- ('Enable directives that insert the contents of external file '
- '("include" & "raw"). Enabled by default.',
- ['--file-insertion-enabled'],
- {'action': 'store_true'}),
- ('Disable the "raw" directives; replaced with a "warning" '
- 'system message.',
- ['--no-raw'],
- {'action': 'store_false', 'default': 1, 'dest': 'raw_enabled',
- 'validator': frontend.validate_boolean}),
- ('Enable the "raw" directive. Enabled by default.',
- ['--raw-enabled'],
- {'action': 'store_true'}),
- ('Maximal number of characters in an input line. Default 10 000.',
- ['--line-length-limit'],
- {'metavar': '<length>', 'type': 'int', 'default': 10000,
- 'validator': frontend.validate_nonnegative_int}),
('Token name set for parsing code with Pygments: one of '
'"long", "short", or "none" (no parsing). Default is "long".',
['--syntax-highlight'],
Modified: trunk/docutils/test/test_parsers/test_recommonmark/test_misc.py
===================================================================
--- trunk/docutils/test/test_parsers/test_recommonmark/test_misc.py 2021-04-06 19:50:57 UTC (rev 8669)
+++ trunk/docutils/test/test_parsers/test_recommonmark/test_misc.py 2021-04-07 12:04:36 UTC (rev 8670)
@@ -50,7 +50,7 @@
class reCommonMarkParserTests(unittest.TestCase):
- @unittest.skipUnless(recommonmark_wrapper.with_recommonmark, skip_msg)
+ @unittest.skipUnless(recommonmark_wrapper.CommonMarkParser, skip_msg)
def test_parsing_error(self):
output = publish_string(sample1, parser_name='recommonmark',
settings_overrides={'warning_stream': ''})
@@ -58,7 +58,7 @@
self.assertIn(b'Parsing with "recommonmark" returned the error:',
output)
- @unittest.skipUnless(recommonmark_wrapper.with_recommonmark, skip_msg)
+ @unittest.skipUnless(recommonmark_wrapper.CommonMarkParser, skip_msg)
def test_raw_disabled(self):
output = publish_string(sample_with_html, parser_name='recommonmark',
settings_overrides={'warning_stream': '',
@@ -67,7 +67,7 @@
self.assertIn(b'<system_message', output)
self.assertIn(b'Raw content disabled.', output)
- @unittest.skipUnless(recommonmark_wrapper.with_recommonmark, skip_msg)
+ @unittest.skipUnless(recommonmark_wrapper.CommonMarkParser, skip_msg)
def test_raw_disabled_inline(self):
output = publish_string('foo <a href="uri">', parser_name='recommonmark',
settings_overrides={'warning_stream': '',
@@ -78,7 +78,7 @@
self.assertIn(b'Raw content disabled.', output)
- @unittest.skipUnless(recommonmark_wrapper.with_recommonmark, skip_msg)
+ @unittest.skipUnless(recommonmark_wrapper.CommonMarkParser, skip_msg)
def test_raw_disabled(self):
output = publish_string(sample_with_html, parser_name='recommonmark',
settings_overrides={'warning_stream': '',
@@ -88,7 +88,7 @@
self.assertNotIn(b'<raw>', output)
self.assertNotIn(b'<system_message', output)
- @unittest.skipIf(recommonmark_wrapper.with_recommonmark,
+ @unittest.skipIf(recommonmark_wrapper.CommonMarkParser,
'recommonmark_wrapper: parser found, fallback not used')
def test_fallback_parser(self):
output = publish_string(sample1, parser_name='recommonmark',
Modified: trunk/docutils/test/test_parsers/test_rst/test_directives/test_include.py
===================================================================
--- trunk/docutils/test/test_parsers/test_rst/test_directives/test_include.py 2021-04-06 19:50:57 UTC (rev 8669)
+++ trunk/docutils/test/test_parsers/test_rst/test_directives/test_include.py 2021-04-07 12:04:36 UTC (rev 8670)
@@ -15,7 +15,7 @@
import __init__
from test_parsers import DocutilsTestSupport
from docutils.parsers.rst import states
-from docutils.parsers.recommonmark_wrapper import with_recommonmark
+from docutils.parsers import recommonmark_wrapper
from docutils.utils.code_analyzer import with_pygments
if sys.version_info >= (3, 0):
@@ -76,7 +76,7 @@
# Parsing with Markdown (recommonmark) is an optional feature depending
# on 3rd-party modules:
-if with_recommonmark:
+if recommonmark_wrapper.CommonMarkParser:
markdown_parsing_result = """\
<section ids="title-1" names="title\\ 1">
<title>
@@ -1048,6 +1048,89 @@
.
""" % reldir(include1)],
["""\
+TAB expansion with included code:
+
+.. include:: %s
+ :code: rst
+""" % include_literal,
+"""\
+<document source="test data">
+ <paragraph>
+ TAB expansion with included code:
+ <literal_block classes="code rst" source="%s" xml:space="preserve">
+ Literal included this should \n\
+ <inline classes="generic strong">
+ **not**
+ be \n\
+ <inline classes="generic emph">
+ *marked*
+ \n\
+ <inline classes="name variable">
+ `up`
+ .
+ <- leading raw tab.
+ \n\
+ Newlines
+ are
+ normalized.
+""" % include_literal],
+["""\
+Custom TAB expansion with included code:
+
+.. include:: %s
+ :code: rst
+ :tab-width: 2
+""" % include_literal,
+"""\
+<document source="test data">
+ <paragraph>
+ Custom TAB expansion with included code:
+ <literal_block classes="code rst" source="%s" xml:space="preserve">
+ Literal included this should \n\
+ <inline classes="generic strong">
+ **not**
+ be \n\
+ <inline classes="generic emph">
+ *marked*
+ \n\
+ <inline classes="name variable">
+ `up`
+ .
+ <- leading raw tab.
+ \n\
+ Newlines
+ are
+ normalized.
+""" % include_literal],
+["""\
+Custom TAB expansion with included code:
+
+.. include:: %s
+ :code: rst
+ :tab-width: -1
+""" % include_literal,
+"""\
+<document source="test data">
+ <paragraph>
+ Custom TAB expansion with included code:
+ <literal_block classes="code rst" source="%s" xml:space="preserve">
+ Literal included this should \n\
+ <inline classes="generic strong">
+ **not**
+ be \n\
+ <inline classes="generic emph">
+ *marked*
+ \n\
+ <inline classes="name variable">
+ `up`
+ .
+ \t<- leading raw tab.
+ \n\
+ Newlines
+ are
+ normalized.
+""" % include_literal],
+["""\
Including includes/include14.txt
.. include:: %s
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
