Menu
▾
▴
[Docutils-checkins] SF.net SVN: docutils:[8565] trunk/docutils
|
From: <mi...@us...> - 2020-09-14 10:26:06
|
Revision: 8565
http://sourceforge.net/p/docutils/code/8565
Author: milde
Date: 2020-09-14 10:26:03 +0000 (Mon, 14 Sep 2020)
Log Message:
-----------
Limit length of input lines and substitution expansion.
Mitigate the danger of DoS attacs using
specially crafted rST input (cf. bug #381).
The default value of 10 000 characters should suffice for
legitimate use cases (e.g. long paragraphs in auto-wrapping editors
or extensive creative use of substitutions).
Applications processing untrusted rST might wish to lower this
limitation (together with other safety measures described in
docs/howto/security.txt).
Unsupervised processing of untrusted rST input should always
be safeguarded with limits on processing time and memory use.
Modified Paths:
--------------
trunk/docutils/docs/howto/security.txt
trunk/docutils/docs/user/config.txt
trunk/docutils/docutils/parsers/rst/__init__.py
trunk/docutils/docutils/parsers/rst/directives/misc.py
trunk/docutils/docutils/statemachine.py
trunk/docutils/docutils/transforms/references.py
Added Paths:
-----------
trunk/docutils/test/test_parsers/test_rst/test_line_length_limit.py
trunk/docutils/test/test_parsers/test_rst/test_line_length_limit_default.py
trunk/docutils/test/test_transforms/test_substitution_expansion_length_limit.py
Modified: trunk/docutils/docs/howto/security.txt
===================================================================
--- trunk/docutils/docs/howto/security.txt 2020-09-13 17:29:38 UTC (rev 8564)
+++ trunk/docutils/docs/howto/security.txt 2020-09-14 10:26:03 UTC (rev 8565)
@@ -76,7 +76,11 @@
It is recommended to enforce limits for the computation time and resource
utilization of the Docutils process when processing untrusted input.
+In addition, the "line_length_limit_" can be adapted.
+.. _line_length_limit: ../user/config.html#line-length-limit
+
+
Securing Docutils
=================
Modified: trunk/docutils/docs/user/config.txt
===================================================================
--- trunk/docutils/docs/user/config.txt 2020-09-13 17:29:38 UTC (rev 8564)
+++ trunk/docutils/docs/user/config.txt 2020-09-14 10:26:03 UTC (rev 8565)
@@ -639,6 +639,21 @@
.. _include: ../ref/rst/directives.html#include
.. _raw: ../ref/rst/directives.html#raw
+line_length_limit
+~~~~~~~~~~~~~~~~~
+
+Maximal number of characters in an input line or `substitution`_
+definition. To prevent extraordinary high processing times or memory
+usage for certain input constructs, a "warning" system message is
+inserted instead.
+
+Default: 10 000.
+Option: ``--line-length-limit``
+
+New in Docutils 0.17.
+
+.. _substitution: ../ref/rst/directives.html#substitution
+
pep_references
~~~~~~~~~~~~~~
Modified: trunk/docutils/docutils/parsers/rst/__init__.py
===================================================================
--- trunk/docutils/docutils/parsers/rst/__init__.py 2020-09-13 17:29:38 UTC (rev 8564)
+++ trunk/docutils/docutils/parsers/rst/__init__.py 2020-09-14 10:26:03 UTC (rev 8565)
@@ -133,6 +133,10 @@
('Enable the "raw" directive. Enabled by default.',
['--raw-enabled'],
{'action': 'store_true'}),
+ ('Maximal number of characters in an input line. Default 10 000.',
+ ['--line-length-limit'],
+ {'metavar': '<length>', 'type': 'int', 'default': 10000,
+ 'validator': frontend.validate_nonnegative_int}),
('Token name set for parsing code with Pygments: one of '
'"long", "short", or "none (no parsing)". Default is "long".',
['--syntax-highlight'],
@@ -188,7 +192,14 @@
inputlines = docutils.statemachine.string2lines(
inputstring, tab_width=document.settings.tab_width,
convert_whitespace=True)
- self.statemachine.run(inputlines, document, inliner=self.inliner)
+ for i, line in enumerate(inputlines):
+ if len(line) > self.document.settings.line_length_limit:
+ error = self.document.reporter.error(
+ 'Line %d exceeds the line-length-limit.'%(i+1))
+ self.document.append(error)
+ break
+ else:
+ self.statemachine.run(inputlines, document, inliner=self.inliner)
# restore the "default" default role after parsing a document
if '' in roles._roles:
del roles._roles['']
Modified: trunk/docutils/docutils/parsers/rst/directives/misc.py
===================================================================
--- trunk/docutils/docutils/parsers/rst/directives/misc.py 2020-09-13 17:29:38 UTC (rev 8564)
+++ trunk/docutils/docutils/parsers/rst/directives/misc.py 2020-09-14 10:26:03 UTC (rev 8565)
@@ -115,8 +115,13 @@
include_lines = statemachine.string2lines(rawtext, tab_width,
convert_whitespace=True)
+ for i, line in enumerate(include_lines):
+ if len(line) > self.state.document.settings.line_length_limit:
+ raise self.warning('"%s": line %d exceeds the'
+ ' line-length-limit.' % (path, i+1))
+
if 'literal' in self.options:
- # Don't convert tabs to spaces, if `tab_width` is positive.
+ # Don't convert tabs to spaces, if `tab_width` is negative.
if tab_width >= 0:
text = rawtext.expandtabs(tab_width)
else:
Modified: trunk/docutils/docutils/statemachine.py
===================================================================
--- trunk/docutils/docutils/statemachine.py 2020-09-13 17:29:38 UTC (rev 8564)
+++ trunk/docutils/docutils/statemachine.py 2020-09-14 10:26:03 UTC (rev 8565)
@@ -1505,12 +1505,12 @@
- `astring`: a multi-line string.
- `tab_width`: the number of columns between tab stops.
- `convert_whitespace`: convert form feeds and vertical tabs to spaces?
+ - `whitespace`: pattern object with the to-be-converted
+ whitespace characters (default [\\v\\f]).
"""
if convert_whitespace:
astring = whitespace.sub(' ', astring)
lines = [s.expandtabs(tab_width).rstrip() for s in astring.splitlines()]
- # TODO: test for too long lines (fixes bug #381):
- # for line in lines:
return lines
def _exception_data():
Modified: trunk/docutils/docutils/transforms/references.py
===================================================================
--- trunk/docutils/docutils/transforms/references.py 2020-09-13 17:29:38 UTC (rev 8564)
+++ trunk/docutils/docutils/transforms/references.py 2020-09-14 10:26:03 UTC (rev 8565)
@@ -663,11 +663,13 @@
def apply(self):
defs = self.document.substitution_defs
normed = self.document.substitution_names
- subreflist = list(self.document.traverse(nodes.substitution_reference))
nested = {}
+ line_length_limit = getattr(self.document.settings,
+ "line_length_limit", 10000)
subreflist = list(self.document.traverse(nodes.substitution_reference))
for ref in subreflist:
+ msg = ''
refname = ref['refname']
if refname in defs:
key = refname
@@ -678,6 +680,13 @@
msg = self.document.reporter.error(
'Undefined substitution referenced: "%s".'
% refname, base_node=ref)
+ else:
+ subdef = defs[key]
+ if len(subdef.astext()) > line_length_limit:
+ msg = self.document.reporter.error(
+ 'Substitution definition "%s" exceeds the'
+ ' line-length-limit.' % (key))
+ if msg:
msgid = self.document.set_id(msg)
prb = nodes.problematic(
ref.rawsource, ref.rawsource, refid=msgid)
@@ -686,7 +695,6 @@
ref.replace_self(prb)
continue
- subdef = defs[key]
parent = ref.parent
index = parent.index(ref)
if ('ltrim' in subdef.attributes
Added: trunk/docutils/test/test_parsers/test_rst/test_line_length_limit.py
===================================================================
--- trunk/docutils/test/test_parsers/test_rst/test_line_length_limit.py (rev 0)
+++ trunk/docutils/test/test_parsers/test_rst/test_line_length_limit.py 2020-09-14 10:26:03 UTC (rev 8565)
@@ -0,0 +1,76 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# $Id$
+# Author: David Goodger <go...@py...>
+# Copyright: This module has been placed in the public domain.
+
+"""
+Tests for inline markup in docutils/parsers/rst/states.py.
+Interpreted text tests are in a separate module, test_interpreted.py.
+"""
+from __future__ import absolute_import
+
+if __name__ == '__main__':
+ import __init__
+from test_parsers import DocutilsTestSupport
+
+
+def suite():
+ s = DocutilsTestSupport.ParserTestSuite(
+ suite_settings={'line_length_limit': 80})
+ s.generateTests(totest)
+ return s
+
+totest = {}
+
+totest['default'] = [
+["""\
+within the limit
+%s
+""" % ("x"*80),
+"""\
+<document source="test data">
+ <paragraph>
+ within the limit
+ %s
+""" % ("x"*80)],
+["""\
+above the limit
+%s
+""" % ("x"*81),
+"""\
+<document source="test data">
+ <system_message level="3" source="test data" type="ERROR">
+ <paragraph>
+ Line 2 exceeds the line-length-limit.
+"""],
+["""\
+Include Test
+============
+
+.. include:: docutils.conf
+ :literal:
+
+A paragraph.
+""",
+"""\
+<document source="test data">
+ <section ids="include-test" names="include\\ test">
+ <title>
+ Include Test
+ <system_message level="2" line="4" source="test data" type="WARNING">
+ <paragraph>
+ "docutils.conf": line 5 exceeds the line-length-limit.
+ <literal_block xml:space="preserve">
+ .. include:: docutils.conf
+ :literal:
+ <paragraph>
+ A paragraph.
+"""],
+]
+
+
+if __name__ == '__main__':
+ import unittest
+ unittest.main(defaultTest='suite')
Property changes on: trunk/docutils/test/test_parsers/test_rst/test_line_length_limit.py
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: trunk/docutils/test/test_parsers/test_rst/test_line_length_limit_default.py
===================================================================
--- trunk/docutils/test/test_parsers/test_rst/test_line_length_limit_default.py (rev 0)
+++ trunk/docutils/test/test_parsers/test_rst/test_line_length_limit_default.py 2020-09-14 10:26:03 UTC (rev 8565)
@@ -0,0 +1,52 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# $Id$
+# Author: David Goodger <go...@py...>
+# Copyright: This module has been placed in the public domain.
+
+"""
+Tests for inline markup in docutils/parsers/rst/states.py.
+Interpreted text tests are in a separate module, test_interpreted.py.
+"""
+from __future__ import absolute_import
+
+if __name__ == '__main__':
+ import __init__
+from test_parsers import DocutilsTestSupport
+
+
+def suite():
+ s = DocutilsTestSupport.ParserTestSuite()
+ s.generateTests(totest)
+ return s
+
+totest = {}
+
+totest['default'] = [
+["""\
+within the limit
+%s
+""" % ("x"*10000),
+"""\
+<document source="test data">
+ <paragraph>
+ within the limit
+ %s
+""" % ("x"*10000)],
+["""\
+above the limit
+%s
+""" % ("x"*10001),
+"""\
+<document source="test data">
+ <system_message level="3" source="test data" type="ERROR">
+ <paragraph>
+ Line 2 exceeds the line-length-limit.
+"""],
+]
+
+
+if __name__ == '__main__':
+ import unittest
+ unittest.main(defaultTest='suite')
Property changes on: trunk/docutils/test/test_parsers/test_rst/test_line_length_limit_default.py
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
Added: trunk/docutils/test/test_transforms/test_substitution_expansion_length_limit.py
===================================================================
--- trunk/docutils/test/test_transforms/test_substitution_expansion_length_limit.py (rev 0)
+++ trunk/docutils/test/test_transforms/test_substitution_expansion_length_limit.py 2020-09-14 10:26:03 UTC (rev 8565)
@@ -0,0 +1,71 @@
+#! /usr/bin/env python
+
+# $Id$
+# Author: David Goodger <go...@py...>
+# Copyright: This module has been placed in the public domain.
+
+"""
+Tests for docutils.transforms.references.Substitutions.
+"""
+from __future__ import absolute_import
+
+if __name__ == '__main__':
+ import __init__
+from test_transforms import DocutilsTestSupport
+from docutils.transforms.references import Substitutions
+from docutils.parsers.rst import Parser
+
+
+def suite():
+ parser = Parser()
+ s = DocutilsTestSupport.TransformTestSuite(parser,
+ suite_settings={'line_length_limit': 80})
+ s.generateTests(totest)
+ return s
+
+# pseudoxml representation of the substitution definition content:
+a = ' lol'
+b = ' 10^1 \n' + '\n \n'.join(10 * [a])
+c = ' 10^2 \n' + '\n \n'.join(10 * [b])
+
+totest = {}
+
+totest['substitutions'] = ((Substitutions,), [
+["""\
+The billion laughs attack for ReStructuredText:
+
+.. |a| replace:: lol
+.. |b| replace:: 10^1 |a| |a| |a| |a| |a| |a| |a| |a| |a| |a|
+.. |c| replace:: 10^2 |b| |b| |b| |b| |b| |b| |b| |b| |b| |b|
+.. ...
+
+|a| |c| continuation text
+""",
+"""\
+<document source="test data">
+ <paragraph>
+ The billion laughs attack for ReStructuredText:
+ <substitution_definition names="a">
+ lol
+ <substitution_definition names="b">
+{}
+ <substitution_definition names="c">
+{}
+ <comment xml:space="preserve">
+ ...
+ <paragraph>
+ lol
+ \n\
+ <problematic ids="id2" refid="id1">
+ |c|
+ continuation text
+ <system_message backrefs="id2" ids="id1" level="3" line="9" source="test data" type="ERROR">
+ <paragraph>
+ Substitution definition "c" exceeds the line-length-limit.
+""".format(b, c)],
+])
+
+
+if __name__ == '__main__':
+ import unittest
+ unittest.main(defaultTest='suite')
Property changes on: trunk/docutils/test/test_transforms/test_substitution_expansion_length_limit.py
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+Author Date Id Revision
\ No newline at end of property
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
