Get rid of the PyRuntime class

PyRuntime was an attempt to run the tests against multiple pythons.

However, it was clumsy, and contained hardcoded values (for python 2.7)

This patch removes it, instead simply building with sys.executable

Doing so means that we're now building and running the selftests against
the version of python that the plugin was built against, rather than
hardcoding /usr/bin/python2.7

In particular, when building the plugin for Python 3, it means that the
selftests are now testing compiling against the Python 3 headers.

Doing this located some real bugs for this case, which this patch also
fixes.

Author: davidmalcolm

cpybuilder.py

@@ -516,51 +516,3 @@ def from_text(cls, txt):
                              micro=int(m.group(3)),
                              releaselevel=m.group(4),
                              serial=int(m.group(5)))
-
-class PyRuntime:
-    def __init__(self, executable, config):
-        self.executable = executable
-        self.config = config
-
-        pydebug = self.run_command('import sys; print(hasattr(sys, "getobjects"))').strip()
-        assert pydebug in ('False', 'True')
-        self.pydebug = (pydebug == 'True')
-
-        versiontext = self.run_command('import sys; print(sys.version_info)').strip()
-        self.versioninfo = PyVersionInfo.from_text(versiontext)
-
-    def is_py3k(self):
-        return self.versioninfo.major == 3
-
-    def get_build_flags(self):
-        return self.get_config_value(['--cflags', '--ldflags'])
-
-    def get_config_value(self, flags):
-        args = [self.config] + flags
-        p = Popen(args, stdout=PIPE, stderr=PIPE)
-        out, err = p.communicate()
-        if six.PY3:
-            out = out.decode()
-            err = err.decode()
-        return ' '.join(out.splitlines()).strip()
-
-    def run_command(self, cmd, checkoutput=True):
-        p = Popen([self.executable, '-c', cmd],
-                  stdout=PIPE, stderr=PIPE)
-        out, err = p.communicate()
-        if six.PY3:
-            out = out.decode()
-            err = err.decode()
-        if p.returncode != 0:
-            raise PyRuntimeError(self, cmd, out, err, p)
-        return out
-
-    #def compile_simple_module(self, sm):
-
-    def get_module_filename(self, name):
-        # FIXME: this is a Fedora-ism:
-        # FIXME: support for 3.2 onwards also?
-        if self.pydebug:
-            return '%s_d.so' % name
-        else:
-            return '%s.so' % name

libcpychecker/PyArg_ParseTuple.py

@@ -327,8 +327,14 @@ def from_string(cls, fmt_string, with_size_t):
                 else:
                     result.add_argument('u', [Py_UNICODE().pointer.pointer])
             elif c == 'S':
-                result.add_argument('S', [(get_PyStringObject().pointer.pointer,
-                                           get_PyObject().pointer.pointer)])
+                if is_py3k():
+                    # S (bytes) [PyBytesObject *] (or PyObject *)
+                    result.add_argument('S', [(get_PyBytesObject().pointer.pointer,
+                                               get_PyObject().pointer.pointer)])
+                else:
+                    # S (string) [PyStringObject *] (or PyObject *)
+                    result.add_argument('S', [(get_PyStringObject().pointer.pointer,
+                                               get_PyObject().pointer.pointer)])
             elif c == 'U':
                 result.add_argument('U', [(get_PyUnicodeObject().pointer.pointer,
                                            get_PyObject().pointer.pointer)])

libcpychecker/refcounts.py

@@ -104,7 +104,7 @@ def get_transitions_for_function_call(self, state, stmt):
         # for this state by assigning it with a special "DeallocatedMemory"
         # value
         # Clear the value for any fields within the region:
-        for k, v in region.fields.iteritems():
+        for k, v in region.fields.items():
             if v in new.value_for_region:
                 del new.value_for_region[v]
         # Set the default value for the whole region to be "DeallocatedMemory"
@@ -631,7 +631,7 @@ def dump_region(region, title):
         print('    %s' % endstate.exception_rvalue)
 
         if i + 1 < len(traces):
-            print
+            sys.stdout.write('\n')
 
 def check_refcounts(fun, dump_traces=False, show_traces=False):
     """

libcpychecker/types.py

@@ -21,6 +21,15 @@
 import gcc
 from gccutils import get_global_typedef
 
+def is_py3k():
+    """
+    Is the Python.h we're compiling against python 3?
+    """
+    if get_global_typedef('PyStringObject'):
+        return False
+    else:
+        return True
+
 def get_Py_ssize_t():
     return get_global_typedef('Py_ssize_t')
 
@@ -54,3 +63,7 @@ def get_PyUnicodeObject():
 
 def get_Py_complex():
     return get_global_typedef('Py_complex')
+
+# Python 3:
+def get_PyBytesObject():
+    return get_global_typedef('PyBytesObject')

run-test-suite.py

@@ -63,7 +63,8 @@ def __init__(self, exppath):
                 # replace long literals with int literals:
                 expdata = re.sub('([0-9]+)L', '\g<1>', expdata)
                 expdata = re.sub('(0x[0-9a-f]+)L', '\g<1>', expdata)
-
+                expdata = expdata.replace('struct PyStringObject',
+                                          'struct PyBytesObject')
             # The expected data is for 64-bit builds of Python
             # Fix it up for 32-bit builds as necessary:
             if six.MAXSIZE == 0x7fffffff:
@@ -239,7 +240,7 @@ def exclude_test(test):
 failed_tests = []
 for testdir in testdirs:
     try:
-        print('%s: ' % testdir),
+        sys.stdout.write('%s: ' % testdir)
         run_test(testdir)
         print('OK')
         num_passes += 1

testcpybuilder.py

@@ -15,6 +15,8 @@
 #   along with this program.  If not, see
 #   <http://www.gnu.org/licenses/>.
 
+
+from distutils import sysconfig as sc
 import os
 import shutil
 from subprocess import Popen, PIPE
@@ -26,13 +28,13 @@
 
 from cpybuilder import *
 
-
-# FIXME: this will need tweaking:
-pyruntimes = [PyRuntime('/usr/bin/python2.7', '/usr/bin/python2.7-config'),
-              #PyRuntime('/usr/bin/python2.7-debug', '/usr/bin/python2.7-debug-config'),
-              #PyRuntime('/usr/bin/python3.2mu', '/usr/bin/python3.2mu-config'),
-              #PyRuntime('/usr/bin/python3.2dmu', '/usr/bin/python3.2dmu-config')
-              ]
+def get_module_filename(name):
+    # FIXME: this is a Fedora-ism:
+    # FIXME: support for 3.2 onwards also?
+    if hasattr(sys, "getobjects"):
+        return '%s_d.so' % name
+    else:
+        return '%s.so' % name
 
 class CompilationError(CommandError):
     def __init__(self, bm):
@@ -43,33 +45,33 @@ def _describe_activity(self):
         return 'compiling: %s' % ' '.join(self.bm.args)
 
 class BuiltModule:
-    """A test build of a SimpleModule for a PyRuntime, done in a tempdir"""
-    def __init__(self, sm, runtime):
+    """A test build of a SimpleModule using sys.executable, done in a tempdir"""
+    def __init__(self, sm):
         self.sm = sm
-        self.runtime = runtime
 
-    def write_src(self, extra_cflags = None):
+    def write_src(self, modname, extra_cflags = None):
         self.tmpdir = tempfile.mkdtemp()
 
-        self.srcfile = os.path.join(self.tmpdir, 'example.c')
-        self.modfile = os.path.join(self.tmpdir, self.runtime.get_module_filename('example'))
+        self.srcfile = os.path.join(self.tmpdir, '%s.c' % modname)
+        self.modfile = os.path.join(self.tmpdir, get_module_filename(modname))
 
         f = open(self.srcfile, 'w')
         f.write(self.sm.cu.as_str())
         f.close()
 
 
     def compile_src(self, extra_cflags = None):
-        cflags = self.runtime.get_build_flags()
         self.args = ['gcc']
         self.args += ['-o', self.modfile]
-        self.args += cflags.split()
+        self.args +=  ['-I' + sc.get_python_inc(),
+                       '-I' + sc.get_python_inc(plat_specific=True)]
+        self.args += sc.get_config_var('CFLAGS').split()
         self.args += ['-Wall',  '-Werror'] # during testing
         self.args += ['-shared'] # not sure why this is necessary
         if extra_cflags:
             self.args += extra_cflags
         self.args += [self.srcfile]
-        #print self.args
+        # print(self.args)
 
         env = dict(os.environ)
         env['LC_ALL'] = 'C'
@@ -85,24 +87,29 @@ def compile_src(self, extra_cflags = None):
             raise CompilationError(self)
 
         assert os.path.exists(self.modfile)
+        # print(self.modfile)
 
-    def build(self, extra_cflags = None):
-        self.write_src()
+    def build(self, modname, extra_cflags = None):
+        self.write_src(modname)
         self.compile_src(extra_cflags)
 
-    def run_command(self, cmd):
-        """Run the command (using the appropriate PyRuntime), adjusting sys.path first"""
-        out = self.runtime.run_command('import sys; sys.path.append("%s"); %s' % (self.tmpdir, cmd))
-        return out
-
     def cleanup(self):
         shutil.rmtree(self.tmpdir)
 
 class SimpleTest(unittest.TestCase):
-    #def __init__(self, pyruntime):
-    #    self.pytun
+
+    # We'll be manipulating sys.path during the test
+    # Save a copy, and restore it after each test:
+    def setUp(self):
+        self.saved_sys_path = sys.path
+        sys.path = sys.path[:]
+
+    def tearDown(self):
+        sys.path = self.saved_sys_path
+
     def test_simple_compilation(self):
         # Verify building and running a trivial module (against multiple Python runtimes)
+        MODNAME = 'simple_compilation'
         sm = SimpleModule()
 
         sm.cu.add_decl("""
@@ -123,23 +130,24 @@ def test_simple_compilation(self):
                                              METH_VARARGS, 'Return a greeting.')])
         sm.cu.add_defn(methods.c_defn())
 
-        sm.add_module_init('example', modmethods=methods, moddoc='This is a doc string')
+        sm.add_module_init(MODNAME, modmethods=methods, moddoc='This is a doc string')
         # print sm.cu.as_str()
 
-        for runtime in pyruntimes:
-            # Build the module:
-            bm = BuiltModule(sm, runtime)
-            bm.build()
+        # Build the module:
+        bm = BuiltModule(sm)
+        bm.build(MODNAME)
 
-            # Verify that it built:
-            out = bm.run_command('import example; print(example.hello())')
-            self.assertEqual(out, "Hello world!\n")
+        # Verify that it built:
+        sys.path.append(bm.tmpdir)
+        import simple_compilation
+        self.assertEqual(simple_compilation.hello(), 'Hello world!')
 
-            # Cleanup successful test runs:
-            bm.cleanup()
+        # Cleanup successful test runs:
+        bm.cleanup()
 
     def test_module_with_type(self):
         # Verify an extension with a type
+        MODNAME = 'module_with_type'
         sm = SimpleModule()
 
         sm.cu.add_decl("""
@@ -164,20 +172,21 @@ def test_module_with_type(self):
                            struct_name = 'struct PyExampleType',
                            tp_repr = 'example_Example_repr')
 
-        sm.add_module_init('example', modmethods=None, moddoc='This is a doc string')
+        sm.add_module_init(MODNAME, modmethods=None, moddoc='This is a doc string')
         # print sm.cu.as_str()
 
-        for runtime in pyruntimes:
-            # Build the module:
-            bm = BuiltModule(sm, runtime)
-            bm.build()
+        # Build the module:
+        bm = BuiltModule(sm)
+        bm.build(MODNAME)
 
-            # Verify that it built:
-            out = bm.run_command('import example; print(repr(example.ExampleType()))')
-            self.assertEqual(out, "example.ExampleType('')\n")
+        # Verify that it built:
+        sys.path.append(bm.tmpdir)
+        import module_with_type
+        self.assertEqual(repr(module_with_type.ExampleType()),
+                         "example.ExampleType('')")
 
-            # Cleanup successful test runs:
-            bm.cleanup()
+        # Cleanup successful test runs:
+        bm.cleanup()
 
     def test_version_parsing(self):
         vi  = PyVersionInfo.from_text("sys.version_info(major=2, minor=7, micro=1, releaselevel='final', serial=0)")

testcpychecker.py

@@ -23,12 +23,9 @@
 
 import six
 
-from testcpybuilder import BuiltModule, PyRuntime, SimpleModule, CompilationError
+from testcpybuilder import BuiltModule, SimpleModule, CompilationError
 from cpybuilder import PyMethodTable, PyMethodDef, METH_VARARGS
 
-#FIXME:
-pyruntime = PyRuntime('/usr/bin/python2.7', '/usr/bin/python2.7-config')
-
 class ExpectedErrorNotFound(CompilationError):
     def __init__(self, expected_err, actual_err, bm):
         CompilationError.__init__(self, bm)
@@ -59,7 +56,7 @@ def compile_src(self, bm):
                                      '-fplugin-arg-python-script=cpychecker.py'])
 
     def build_module(self, bm):
-        bm.write_src()
+        bm.write_src('example')
         self.compile_src(bm)
 
     def assertNoErrors(self, src):
@@ -68,7 +65,7 @@ def assertNoErrors(self, src):
         else:
             sm = SimpleModule()
             sm.cu.add_defn(src)
-        bm = BuiltModule(sm, pyruntime)
+        bm = BuiltModule(sm)
         self.build_module(bm)
         bm.cleanup()
         return bm
@@ -79,9 +76,9 @@ def assertFindsError(self, src, experr):
         else:
             sm = SimpleModule()
             sm.cu.add_defn(src)
-        bm = BuiltModule(sm, pyruntime)
+        bm = BuiltModule(sm)
         try:
-            bm.write_src()
+            bm.write_src('example')
             experr = experr.replace('$(SRCFILE)', bm.srcfile)
             self.compile_src(bm)
         except CompilationError:
@@ -127,7 +124,11 @@ def test_finding_htons_error(self):
         return NULL;
     }
     x2 = (int)htons((short)x1);
+#if PY_MAJOR_VERSION >= 3
+    return PyLong_FromLong(x2);
+#else
     return PyInt_FromLong(x2);
+#endif
 }
 """
         self.assertFindsError(src,