Parse environment= using shlex in posix mode if possible

Closes #328
References #873
Closes #1613

Author: mnaberez

CHANGES.rst

@@ -1,6 +1,13 @@
 4.4.0.dev0 (Next Release)
 -------------------------
 
+- Parsing ``environment=`` in the config file now uses ``shlex`` in POSIX
+  mode instead of legacy mode to allow for escaped quotes in the values.
+  However, on Python 2 before 2.7.13 and Python 3 before 3.5.3, POSIX mode
+  can't be used because of a `bug <https://bugs.python.org/issue21999>`_
+  in ``shlex``.  If ``supervisord`` is run on a Python version with the bug,
+  it will fall back to legacy mode.  Patch by Stefan Friesel.
+
 - ``supervisorctl`` now reads extra files included via the ``[include]``
   section in ``supervisord.conf`` like ``supervisord`` does.  This allows
   the ``[supervisorctl]`` section or ``[ctlplugin:x]`` sections to be in

supervisor/compat.py

@@ -150,6 +150,26 @@ def is_text_stream(stream):
 except ImportError: # pragma: no cover
     from HTMLParser import HTMLParser
 
+# Begin check for working shlex posix mode
+
+# https://github.com/Supervisor/supervisor/issues/328
+# https://github.com/Supervisor/supervisor/issues/873
+# https://bugs.python.org/issue21999
+
+from shlex import shlex as _shlex
+
+_shlex_posix_expectations = {
+    'foo="",bar=a': ['foo', '=', '', ',', 'bar', '=', 'a'],
+    "'')abc":       ['', ')', 'abc']
+}
+
+shlex_posix_works = all(
+    list(_shlex(_input, posix=True)) == _expected
+    for _input, _expected in _shlex_posix_expectations.items()
+)
+
+# End check for working shlex posix mode
+
 # Begin importlib/setuptools compatibility code
 
 # Supervisor used pkg_resources (a part of setuptools) to load package

supervisor/datatypes.py

@@ -5,6 +5,7 @@
 import socket
 import shlex
 
+from supervisor.compat import shlex_posix_works
 from supervisor.compat import urlparse
 from supervisor.compat import long
 from supervisor.loggers import getLevelNumByDescription
@@ -68,7 +69,7 @@ def dict_of_key_value_pairs(arg):
     """ parse KEY=val,KEY2=val2 into {'KEY':'val', 'KEY2':'val2'}
         Quotes can be used to allow commas in the value
     """
-    lexer = shlex.shlex(str(arg))
+    lexer = shlex.shlex(str(arg), posix=shlex_posix_works)
     lexer.wordchars += '/.+-():'
 
     tokens = list(lexer)
@@ -81,7 +82,12 @@ def dict_of_key_value_pairs(arg):
         if len(k_eq_v) != 3 or k_eq_v[1] != '=':
             raise ValueError(
                 "Unexpected end of key/value pairs in value '%s'" % arg)
-        D[k_eq_v[0]] = k_eq_v[2].strip('\'"')
+
+        k, v = k_eq_v[0], k_eq_v[2]
+        if not shlex_posix_works:
+            v = v.strip('\'"')
+
+        D[k] = v
         i += 4
     return D
 

supervisor/tests/test_datatypes.py

@@ -8,6 +8,7 @@
 
 from supervisor.tests.base import Mock, patch, sentinel
 from supervisor.compat import maxint
+from supervisor.compat import shlex_posix_works
 
 from supervisor import datatypes
 
@@ -164,6 +165,16 @@ def test_handles_newlines_inside_quotes(self):
         expected = {'foo': 'a\nb\nc'}
         self.assertEqual(actual, expected)
 
+    def test_handles_quotes_inside_quotes(self):
+        func = lambda: datatypes.dict_of_key_value_pairs('foo="\'\\""')
+
+        if shlex_posix_works:
+            actual = func()
+            expected = {'foo': '\'"'}
+            self.assertEqual(actual, expected)
+        else:
+            self.assertRaises(ValueError, func)
+
     def test_handles_empty_inside_quotes(self):
         actual = datatypes.dict_of_key_value_pairs('foo=""')
         expected = {'foo': ''}