Test that the code examples in the docs run and match their output

MANIFEST.in

@@ -2,6 +2,7 @@ include requirements.txt
 include LICENSE
 include README.md
 include tests.py
+include doc/*.rst
 include property_tests.py
 include pytest.ini
 include ext_tests.py

doc/commandline.rst

@@ -21,11 +21,13 @@ JSON files ::
       FILE1
       FILE2
 
-    optional arguments:
-      -h, --help             show this help message and exit
-      --indent INDENT        Indent output by n spaces
-      -u, --preserve-unicode Output Unicode character as-is without using Code Point
-      -v, --version          show program's version number and exit
+    options:
+      -h, --help            show this help message and exit
+      --indent INDENT       Indent output by n spaces
+      -u, --preserve-unicode
+                            Output Unicode character as-is without using Code
+                            Point
+      -v, --version         show program's version number and exit
 
 Example
 ^^^^^^^
@@ -41,24 +43,24 @@ Example
 
     # show patch in "dense" representation
     $ jsondiff a.json b.json
-    [{"path": "/a/2", "value": 3, "op": "add"}, {"path": "/b", "op": "remove"}, {"path": "/c", "value": 100, "op": "add"}]
+    [{"op": "remove", "path": "/b"}, {"op": "add", "path": "/c", "value": 100}, {"op": "add", "path": "/a/2", "value": 3}]
 
     # show patch with some indentation
     $ jsondiff a.json b.json --indent=2
     [
       {
-        "path": "/a/2",
-        "value": 3,
-        "op": "add"
+        "op": "remove",
+        "path": "/b"
       },
       {
-        "path": "/b",
-        "op": "remove"
+        "op": "add",
+        "path": "/c",
+        "value": 100
       },
       {
-        "path": "/c",
-        "value": 100,
-        "op": "add"
+        "op": "add",
+        "path": "/a/2",
+        "value": 3
       }
     ]
 
@@ -69,21 +71,23 @@ Example
 
 The program ``jsonpatch`` is used to apply JSON patches on JSON files. ::
 
-    usage: jsonpatch [-h] [--indent INDENT] [-v] ORIGINAL PATCH
+    usage: jsonpatch [-h] [--indent INDENT] [-b] [-i] [-v] [-u] ORIGINAL [PATCH]
 
-    Apply a JSON patch on a JSON files
+    Apply a JSON patch on a JSON file
 
     positional arguments:
-      ORIGINAL         Original file
-      PATCH            Patch file
-
-    optional arguments:
-     -h, --help             show this help message and exit
-     --indent INDENT        Indent output by n spaces
-     -b, --backup           Back up ORIGINAL if modifying in-place
-     -i, --in-place         Modify ORIGINAL in-place instead of to stdout
-     -v, --version          show program's version number and exit
-     -u, --preserve-unicode Output Unicode character as-is without using Code Point
+      ORIGINAL              Original file
+      PATCH                 Patch file (read from stdin if omitted)
+
+    options:
+      -h, --help            show this help message and exit
+      --indent INDENT       Indent output by n spaces
+      -b, --backup          Back up ORIGINAL if modifying in-place
+      -i, --in-place        Modify ORIGINAL in-place instead of to stdout
+      -v, --version         show program's version number and exit
+      -u, --preserve-unicode
+                            Output Unicode character as-is without using Code
+                            Point
 
 Example
 ^^^^^^^
@@ -110,7 +114,6 @@ Example
     # pipe result into new file
     $ jsonpatch a.json patch.json --indent=2 > c.json
 
-    # c.json now equals b.json
+    # c.json now equals b.json, so there is no difference to print
     $ jsondiff b.json c.json
-    []
 

doc/tutorial.rst

@@ -19,16 +19,25 @@ patch object from it.
 
     >>> import jsonpatch
     >>> patch = jsonpatch.JsonPatch([
-        {'op': 'add', 'path': '/foo', 'value': 'bar'},
-        {'op': 'add', 'path': '/baz', 'value': [1, 2, 3]},
-        {'op': 'remove', 'path': '/baz/1'},
-        {'op': 'test', 'path': '/baz', 'value': [1, 3]},
-        {'op': 'replace', 'path': '/baz/0', 'value': 42},
-        {'op': 'remove', 'path': '/baz/1'},
-    ])
+    ...     {'op': 'add', 'path': '/foo', 'value': 'bar'},
+    ...     {'op': 'add', 'path': '/baz', 'value': [1, 2, 3]},
+    ...     {'op': 'remove', 'path': '/baz/1'},
+    ...     {'op': 'test', 'path': '/baz', 'value': [1, 3]},
+    ...     {'op': 'replace', 'path': '/baz/0', 'value': 42},
+    ...     {'op': 'remove', 'path': '/baz/1'},
+    ... ])
 
     # or equivalently
-    >>> patch = jsonpatch.JsonPatch.from_string('[{"op": "add", ....}]')
+    >>> patch = jsonpatch.JsonPatch.from_string('''[
+    ...     {"op": "add", "path": "/foo", "value": "bar"},
+    ...     {"op": "add", "path": "/baz", "value": [1, 2, 3]},
+    ...     {"op": "remove", "path": "/baz/1"},
+    ...     {"op": "test", "path": "/baz", "value": [1, 3]},
+    ...     {"op": "replace", "path": "/baz/0", "value": 42},
+    ...     {"op": "remove", "path": "/baz/1"}
+    ... ]''')
+    >>> patch.apply({})
+    {'foo': 'bar', 'baz': [42]}
 
 Another way is to *diff* two objects.
 
@@ -177,38 +186,45 @@ support for Python's ``Decimal`` type, can be used to create a custom
     >>> import simplejson
 
     >>> class DecimalJsonPatch(jsonpatch.JsonPatch):
-            @staticmethod
-            def json_dumper(obj):
-                return simplejson.dumps(obj)
-
-            @staticmethod
-            def json_loader(obj):
-                return simplejson.loads(obj, use_decimal=True,
-                                        object_pairs_hook=jsonpatch.multidict)
+    ...     @staticmethod
+    ...     def json_dumper(obj):
+    ...         return simplejson.dumps(obj)
+    ...
+    ...     @staticmethod
+    ...     def json_loader(obj):
+    ...         return simplejson.loads(obj, use_decimal=True,
+    ...                                 object_pairs_hook=jsonpatch.multidict)
 
     >>> src = {}
     >>> dst = {'bar': decimal.Decimal('1.10')}
     >>> patch = DecimalJsonPatch.from_diff(src, dst)
     >>> doc = {'foo': 1}
-    >>> result = patch.apply(doc)
+    >>> patch.apply(doc)
     {'foo': 1, 'bar': Decimal('1.10')}
 
 Instead of subclassing it is also possible to pass a dump function to
 ``from_diff``:
 
+.. code-block:: python
+
     >>> patch = jsonpatch.JsonPatch.from_diff(src, dst, dumps=simplejson.dumps)
 
 a dumps function to ``to_string``:
 
+.. code-block:: python
+
     >>> serialized_patch = patch.to_string(dumps=simplejson.dumps)
+    >>> serialized_patch
     '[{"op": "add", "path": "/bar", "value": 1.10}]'
 
 and load  function to ``from_string``:
 
+.. code-block:: python
+
     >>> import functools
     >>> loads = functools.partial(simplejson.loads, use_decimal=True,
-                                  object_pairs_hook=jsonpatch.multidict)
-    >>> patch.from_string(serialized_patch, loads=loads)
+    ...                           object_pairs_hook=jsonpatch.multidict)
+    >>> patch = jsonpatch.JsonPatch.from_string(serialized_patch, loads=loads)
     >>> doc = {'foo': 1}
-    >>> result = patch.apply(doc)
+    >>> patch.apply(doc)
     {'foo': 1, 'bar': Decimal('1.10')}

requirements-dev.txt

@@ -1,3 +1,4 @@
 coverage
 wheel
 hypothesis
+simplejson

tests.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import contextlib
 import copy
+import io
 import json
 import decimal
 import doctest
@@ -10,6 +12,7 @@
 import jsonpatch_cli
 import jsonpointer
 import random
+import shlex
 import sys
 import os
 import stat
@@ -1930,7 +1933,31 @@ class JsonPatch(jsonpatch.JsonPatch):
         self.assertEqual(res, {})
 
 
-class CommandLineTests(unittest.TestCase):
+class ScriptRunner:
+    """ Runs the command line programs of this checkout """
+
+    def run_python(self, code, *args, stdin='', utf8=True, cwd=None):
+        here = os.path.dirname(os.path.abspath(__file__))
+        # the scripts have to import this jsonpatch, not an installed one
+        pythonpath = [here] + [p for p in [os.environ.get('PYTHONPATH')] if p]
+        env = dict(os.environ, PYTHONPATH=os.pathsep.join(pythonpath),
+                   PYTHONUTF8='1')
+        if not utf8:
+            # Exercise locale-dependent file I/O while keeping pipes in UTF-8.
+            env.update(PYTHONUTF8='0', PYTHONIOENCODING='utf-8',
+                       LC_ALL='C', PYTHONCOERCECLOCALE='0')
+        return subprocess.run(
+            [sys.executable, '-c', code] + list(args),
+            input=stdin, capture_output=True, encoding='utf-8', env=env,
+            cwd=cwd)
+
+    def run_script(self, script, *args, stdin='', utf8=True, cwd=None):
+        return self.run_python(
+            'import jsonpatch_cli; jsonpatch_cli.{0}_main()'.format(script),
+            *args, stdin=stdin, utf8=utf8, cwd=cwd)
+
+
+class CommandLineTests(ScriptRunner, unittest.TestCase):
     """ The jsondiff and jsonpatch command line programs """
 
     def setUp(self):
@@ -1948,25 +1975,6 @@ def read(self, path):
         with open(path, encoding='utf-8') as f:
             return json.load(f)
 
-    def run_python(self, code, *args, stdin='', utf8=True):
-        here = os.path.dirname(os.path.abspath(__file__))
-        # the scripts have to import this jsonpatch, not an installed one
-        pythonpath = [here] + [p for p in [os.environ.get('PYTHONPATH')] if p]
-        env = dict(os.environ, PYTHONPATH=os.pathsep.join(pythonpath),
-                   PYTHONUTF8='1')
-        if not utf8:
-            # Exercise locale-dependent file I/O while keeping pipes in UTF-8.
-            env.update(PYTHONUTF8='0', PYTHONIOENCODING='utf-8',
-                       LC_ALL='C', PYTHONCOERCECLOCALE='0')
-        return subprocess.run(
-            [sys.executable, '-c', code] + list(args),
-            input=stdin, capture_output=True, encoding='utf-8', env=env)
-
-    def run_script(self, script, *args, stdin='', utf8=True):
-        return self.run_python(
-            'import jsonpatch_cli; jsonpatch_cli.{0}_main()'.format(script),
-            *args, stdin=stdin, utf8=utf8)
-
     def test_jsondiff(self):
         src = self.write('a.json', {'a': [1, 2], 'b': 0})
         dst = self.write('b.json', {'a': [1, 2, 3], 'c': 100})
@@ -2261,6 +2269,92 @@ def test_version(self):
                 self.assertEqual(res.stdout, '{0} {1}\n'.format(script, jsonpatch.__version__))
 
 
+class DocumentationTests(ScriptRunner, unittest.TestCase):
+    """ The code examples in the documentation run and give the shown output """
+
+    doc_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'doc')
+
+    def doc_path(self, name):
+        path = os.path.join(self.doc_dir, name)
+        if not os.path.exists(path):
+            self.skipTest('documentation not available')
+        return path
+
+    def test_tutorial(self):
+        path = self.doc_path('tutorial.rst')
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out):
+            failed, attempted = doctest.testfile(
+                path, module_relative=False, optionflags=doctest.ELLIPSIS)
+        self.assertGreater(attempted, 0)
+        self.assertEqual(failed, 0, out.getvalue())
+
+    def shell_sessions(self, path):
+        """ Yield (command, expected output) for each "$ " line of the
+        shell examples in ``.. code-block:: bash`` blocks """
+        with open(path, encoding='utf-8') as f:
+            lines = f.read().splitlines()
+        in_block = False
+        command, output = None, []
+        for line in lines + ['']:
+            if line.startswith('.. code-block:: bash'):
+                in_block = True
+                continue
+            if in_block and line and not line[0].isspace():
+                in_block = False
+            text = line.strip()
+            if command is not None and (not in_block or not text or
+                                        text.startswith(('$ ', '#'))):
+                yield command, output
+                command, output = None, []
+            if not in_block:
+                continue
+            if text.startswith('$ '):
+                command = text[2:]
+            elif command is not None:
+                # keep indentation relative to the code block
+                output.append(line[4:])
+
+    def test_commandline(self):
+        path = self.doc_path('commandline.rst')
+        tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp.cleanup)
+        sessions = list(self.shell_sessions(path))
+        self.assertGreater(len(sessions), 0)
+
+        for command, output in sessions:
+            with self.subTest(command=command):
+                args = shlex.split(command)
+                expected = ''.join(line + '\n' for line in output)
+                program, args = args[0], args[1:]
+
+                if program == 'cat':
+                    # "cat" introduces the input files of the examples
+                    filename = os.path.join(tmp.name, args[0])
+                    if not os.path.exists(filename):
+                        with open(filename, 'w', encoding='utf-8') as f:
+                            f.write(expected)
+                    with open(filename, encoding='utf-8') as f:
+                        self.assertEqual(f.read(), expected)
+                    continue
+
+                self.assertIn(program, ['jsondiff', 'jsonpatch'])
+                redirect = None
+                if '>' in args:
+                    redirect = args[args.index('>') + 1]
+                    args = args[:args.index('>')]
+
+                res = self.run_script(program, *args, cwd=tmp.name)
+                self.assertEqual(res.stderr, "")
+
+                if redirect:
+                    with open(os.path.join(tmp.name, redirect), 'w',
+                              encoding='utf-8') as f:
+                        f.write(res.stdout)
+                    res.stdout = ''
+                self.assertEqual(res.stdout, expected)
+
+
 if __name__ == '__main__':
     modules = ['jsonpatch']
 
@@ -2284,6 +2378,7 @@ def get_suite():
         suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(CustomJsonPointerTests))
         suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(CustomOperationTests))
         suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(CommandLineTests))
+        suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(DocumentationTests))
         return suite