Comply with flake8 linter

Reformats Python code according to pep8, and fixes up flake8 warnings. Adds a lint script that can be run to fix up the code to be conformant in the future. Updates the Travis CI tests to run flake8 as part of the automated tests
rtollert · Aug 1, 2019 · fe509fe · fe509fe
1 parent 7b3e0b6
commit fe509fe
Show file tree

Hide file tree

Showing 10 changed files with 217 additions and 110 deletions.
diff --git a/.pep8speaks.yml b/.pep8speaks.yml
@@ -0,0 +1,3 @@
+scanner:
+  diff_only: True
+  linter: flake8
diff --git a/.travis.yml b/.travis.yml
@@ -28,11 +28,14 @@ install:
 - pip3 install coverage
 - pip3 install coveralls
 - pip3 install requests
+- pip3 install flake8
 before_script:
 - printf '\nimport coverage\ncoverage.current_coverage = coverage.process_startup()\n'
   >> "/home/travis/virtualenv/python${TRAVIS_PYTHON_VERSION}/lib/python${TRAVIS_PYTHON_VERSION}/sitecustomize.py"
 - rm -f .coverage-report.*
-script: COVERAGE_PROCESS_START=${TRAVIS_BUILD_DIR}/.coveragerc ci/test.py -vb PyrexImage_$(echo $TEST_IMAGE | sed 's/\W/_/g')
+script:
+- flake8 $(git ls-files '*.py')
+- COVERAGE_PROCESS_START=${TRAVIS_BUILD_DIR}/.coveragerc ci/test.py -vb PyrexImage_$(echo $TEST_IMAGE | sed 's/\W/_/g')
 after_success:
 - coverage3 combine
 - coveralls

diff --git a/DEVELOPING.md b/DEVELOPING.md
@@ -1,6 +1,23 @@
 # Developer Guide
 Pyrex development information and processes
 
+## Linting
+Pyrex conforms to [PEP8](https://pep8.org/) for all Python code, and also make
+use of [flake8](https://pypi.org/project/flake8/) as a linter. Please ensure
+all code conforms to this. There is helpful tool that will report any places
+where the code is non-conformant in the project root:
+
+```shell
+./lint
+```
+
+If you would like to tool to automatically reformat code to comply with PEP8,
+pass the `--reformat` option:
+
+```
+./lint --reformat
+```
+
 ## Testing
 Pyrex has a comprehensive test suite that can be used to test all generated
 Docker images. Some external test data is required to perform the test. To

diff --git a/ci/deploy_docker.py b/ci/deploy_docker.py
@@ -7,9 +7,11 @@
 import subprocess
 import sys
 
+
 def main():
     parser = argparse.ArgumentParser(description='Deploy docker images')
-    parser.add_argument('--login', action='store_true', help='Login to Dockerhub using the environment variables $DOCKER_USERNAME and $DOCKER_PASSWORD')
+    parser.add_argument('--login', action='store_true',
+                        help='Login to Dockerhub using the environment variables $DOCKER_USERNAME and $DOCKER_PASSWORD')
     parser.add_argument('image', metavar='IMAGE[:TAG]', help='The image to build and push')
 
     args = parser.parse_args()
@@ -37,7 +39,8 @@ def main():
                 print("$%s is missing from the environment. Images will not be deployed" % v)
                 return 0
 
-        with subprocess.Popen(['docker', 'login', '--username', os.environ['DOCKER_USERNAME'], '--password-stdin'], stdin=subprocess.PIPE) as p:
+        with subprocess.Popen(['docker', 'login', '--username', os.environ['DOCKER_USERNAME'],
+                               '--password-stdin'], stdin=subprocess.PIPE) as p:
             try:
                 p.communicate(os.environ['DOCKER_PASSWORD'].encode('utf-8'), timeout=60)
             except subprocess.TimeoutExpired:
@@ -52,12 +55,13 @@ def main():
     print("Deploying %s..." % name)
 
     # Get a login token for the docker registry and download the manifest
-    token = requests.get("https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull" % repo, json=True).json()["token"]
+    token = requests.get("https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull" %
+                         repo, json=True).json()["token"]
     manifest = requests.get(
-            "https://registry.hub.docker.com/v2/%s/manifests/%s" % (repo, tag),
-            headers={"Authorization": "Bearer %s" % token},
-            json=True
-            ).json()
+        "https://registry.hub.docker.com/v2/%s/manifests/%s" % (repo, tag),
+        headers={"Authorization": "Bearer %s" % token},
+        json=True
+    ).json()
 
     found_manifest = (manifest.get('name', '') == repo)
 
@@ -80,7 +84,7 @@ def main():
 
     try:
         subprocess.check_call(['docker', 'build'] + docker_build_args)
-    except subprocess.CalledProcessError as e:
+    except subprocess.CalledProcessError:
         print("Building failed!")
         return 1
 
@@ -91,18 +95,19 @@ def main():
         test_name = 'PyrexImage_' + re.sub(r'\W', '_', image)
 
         subprocess.check_call(['%s/test.py' % this_dir, '-vbf', test_name], env=env, cwd=os.path.join(this_dir, '..'))
-    except subprocess.CalledProcessError as e:
+    except subprocess.CalledProcessError:
         print("Testing failed!")
         return 1
 
     print("Pushing", name)
     try:
         subprocess.check_call(['docker', 'push', name])
-    except subprocess.CalledProcessError as e:
+    except subprocess.CalledProcessError:
         print("Pushing failed!")
         return 1
 
     return 0
 
+
 if __name__ == "__main__":
     sys.exit(main())
diff --git a/ci/test.py b/ci/test.py
@@ -20,7 +20,6 @@
 import pwd
 import re
 import shutil
-import stat
 import subprocess
 import sys
 import tempfile
@@ -30,17 +29,19 @@
 
 PYREX_ROOT = os.path.join(os.path.dirname(__file__), '..')
 sys.path.append(PYREX_ROOT)
-import pyrex
+import pyrex  # NOQA
 
 TEST_PREBUILT_TAG_ENV_VAR = 'TEST_PREBUILT_TAG'
 
+
 def skipIfPrebuilt(func):
     def wrapper(self, *args, **kwargs):
         if os.environ.get(TEST_PREBUILT_TAG_ENV_VAR, ''):
             self.skipTest('Test does not apply to prebuilt images')
         return func(self, *args, **kwargs)
     return wrapper
 
+
 class PyrexTest(object):
     def setUp(self):
         self.build_dir = os.path.abspath(os.path.join(PYREX_ROOT, 'build'))
@@ -166,6 +167,7 @@ def assertPyrexContainerShellPTY(self, *args, returncode=0, env=None, quiet_init
         container_cmd_file = self._write_container_command(args)
         host_cmd_file = self._write_host_command(['pyrex-shell %s' % container_cmd_file], quiet_init)
         stdout = []
+
         def master_read(fd):
             while True:
                 data = os.read(fd, 1024)
@@ -187,16 +189,19 @@ def master_read(fd):
                 os.environ.clear()
                 os.environ.update(old_env)
 
-        self.assertFalse(os.WIFSIGNALED(status), msg='%s died from a signal: %s' % (' '.join(args), os.WTERMSIG(status)))
+        self.assertFalse(os.WIFSIGNALED(status), msg='%s died from a signal: %s' %
+                         (' '.join(args), os.WTERMSIG(status)))
         self.assertTrue(os.WIFEXITED(status), msg='%s exited abnormally' % ' '.join(args))
         self.assertEqual(os.WEXITSTATUS(status), returncode, msg='%s failed' % ' '.join(args))
         return b''.join(stdout)
 
+
 class PyrexImageType_base(PyrexTest):
     """
     Base image tests. All images that derive from a -base image should derive
     from this class
     """
+
     def test_init(self):
         self.assertPyrexHostCommand('true')
 
@@ -241,7 +246,8 @@ def test_no_docker_build(self):
         self.assertPyrexHostCommand('true', env=env)
 
         # Verify that pyrex won't allow you to try and use docker later
-        output = self.assertPyrexHostCommand('PYREX_DOCKER=1 bitbake', returncode=1, capture=True, env=env).decode('utf-8')
+        output = self.assertPyrexHostCommand('PYREX_DOCKER=1 bitbake', returncode=1,
+                                             capture=True, env=env).decode('utf-8')
         self.assertIn('Docker was not enabled when the environment was setup', output)
 
     def test_bad_docker(self):
@@ -310,7 +316,8 @@ def read_fifo():
         thread = threading.Thread(target=read_fifo)
         thread.start()
         try:
-            self.assertPyrexContainerShellCommand('echo "$(id -u):$(id -g):$(id -un):$(id -gn):$USER:$GROUP" > %s' % fifo)
+            self.assertPyrexContainerShellCommand(
+                'echo "$(id -u):$(id -g):$(id -un):$(id -gn):$USER:$GROUP" > %s' % fifo)
         finally:
             thread.join()
 
@@ -448,7 +455,8 @@ def test_version_tag(self):
             tag = os.environ['TRAVIS_TAG']
         else:
             try:
-                tags = subprocess.check_output(['git', '-C', PYREX_ROOT, 'tag', '-l', '--points-at', 'HEAD']).decode('utf-8').splitlines()
+                tags = subprocess.check_output(['git', '-C', PYREX_ROOT, 'tag', '-l',
+                                                '--points-at', 'HEAD']).decode('utf-8').splitlines()
                 if tags:
                     tag = tags[0]
             except subprocess.CalledProcessError:
@@ -470,7 +478,8 @@ def test_tag_overwrite(self):
 
         self.assertPyrexHostCommand('true', returncode=1)
 
-        output = self.assertSubprocess(['docker', 'images', '-q', conf['config']['tag']], capture=True).decode('utf-8').strip()
+        output = self.assertSubprocess(['docker', 'images', '-q', conf['config']['tag']],
+                                       capture=True).decode('utf-8').strip()
         self.assertEqual(output, "", msg="Tagged image found!")
 
     def test_pty(self):
@@ -484,25 +493,28 @@ def test_invalid_term(self):
         env['TERM'] = bad_term
         output = self.assertPyrexContainerShellPTY('true', env=env).decode('utf-8').strip()
         self.assertIn('$TERM has an unrecognized value of "%s"' % bad_term, output)
-        self.assertPyrexContainerShellPTY('/usr/bin/infocmp %s > /dev/null' % bad_term, env=env, returncode=1, quiet_init=True)
+        self.assertPyrexContainerShellPTY('/usr/bin/infocmp %s > /dev/null' %
+                                          bad_term, env=env, returncode=1, quiet_init=True)
 
     def test_required_terms(self):
         # Tests that a minimum set of terminals are supported
         REQUIRED_TERMS = (
-                'dumb',
-                'vt100',
-                'xterm',
-                'xterm-256color'
-                )
+            'dumb',
+            'vt100',
+            'xterm',
+            'xterm-256color'
+        )
 
         env = os.environ.copy()
         for t in REQUIRED_TERMS:
             with self.subTest(term=t):
                 env['TERM'] = t
-                output = self.assertPyrexContainerShellPTY('echo $TERM', env=env, quiet_init=True).decode('utf-8').strip()
+                output = self.assertPyrexContainerShellPTY(
+                    'echo $TERM', env=env, quiet_init=True).decode('utf-8').strip()
                 self.assertEqual(output, t, msg='Bad $TERM found in container!')
 
-                output = self.assertPyrexContainerShellPTY('/usr/bin/infocmp %s > /dev/null' % t, env=env).decode('utf-8').strip()
+                output = self.assertPyrexContainerShellPTY(
+                    '/usr/bin/infocmp %s > /dev/null' % t, env=env).decode('utf-8').strip()
                 self.assertNotIn('$TERM has an unrecognized value', output)
 
     def test_tini(self):
@@ -516,8 +528,10 @@ def test_guest_image(self):
         (image_name, image_version, _) = self.test_image.split('-')
 
         # Capture the LSB release information.
-        dist_id_str = self.assertPyrexContainerCommand('lsb_release -i', quiet_init=True, capture=True).decode('utf-8').rstrip()
-        release_str = self.assertPyrexContainerCommand('lsb_release -r', quiet_init=True, capture=True).decode('utf-8').rstrip()
+        dist_id_str = self.assertPyrexContainerCommand(
+            'lsb_release -i', quiet_init=True, capture=True).decode('utf-8').rstrip()
+        release_str = self.assertPyrexContainerCommand(
+            'lsb_release -r', quiet_init=True, capture=True).decode('utf-8').rstrip()
 
         self.assertRegex(dist_id_str.lower(), r'^distributor id:\s+' + re.escape(image_name))
         self.assertRegex(release_str.lower(), r'^release:\s+' + re.escape(image_version) + r'(\.|$)')
@@ -539,16 +553,20 @@ def test_envvars(self):
         env = os.environ.copy()
         env['TEST_ENV'] = test_string
 
-        s = self.assertPyrexContainerShellCommand('echo $TEST_ENV', env=env, quiet_init=True, capture=True).decode('utf-8').rstrip()
+        s = self.assertPyrexContainerShellCommand(
+            'echo $TEST_ENV', env=env, quiet_init=True, capture=True).decode('utf-8').rstrip()
         self.assertEqual(s, test_string)
 
-        s = self.assertPyrexContainerShellCommand('echo $TEST_ENV2', env=env, quiet_init=True, capture=True).decode('utf-8').rstrip()
+        s = self.assertPyrexContainerShellCommand(
+            'echo $TEST_ENV2', env=env, quiet_init=True, capture=True).decode('utf-8').rstrip()
         self.assertEqual(s, '')
 
+
 class PyrexImageType_oe(PyrexImageType_base):
     """
     Tests images designed for building OpenEmbedded
     """
+
     def test_bitbake_parse(self):
         self.assertPyrexHostCommand('bitbake -p')
 
@@ -561,8 +579,10 @@ def test_templateconf_abs(self):
 
         self.assertTrue(os.path.isabs(template_dir))
 
-        shutil.copyfile(os.path.join(PYREX_ROOT, 'poky/meta-poky/conf/local.conf.sample'), os.path.join(template_dir, 'local.conf.sample'))
-        shutil.copyfile(os.path.join(PYREX_ROOT, 'poky/meta-poky/conf/bblayers.conf.sample'), os.path.join(template_dir, 'bblayers.conf.sample'))
+        shutil.copyfile(os.path.join(PYREX_ROOT, 'poky/meta-poky/conf/local.conf.sample'),
+                        os.path.join(template_dir, 'local.conf.sample'))
+        shutil.copyfile(os.path.join(PYREX_ROOT, 'poky/meta-poky/conf/bblayers.conf.sample'),
+                        os.path.join(template_dir, 'bblayers.conf.sample'))
 
         test_string = 'set_by_test.%d' % threading.get_ident()
 
@@ -581,7 +601,8 @@ def test_templateconf_abs(self):
         env['TEMPLATECONF'] = template_dir
         env['TEST_ENV'] = test_string
 
-        s = self.assertPyrexContainerShellCommand('echo $TEST_ENV', env=env, quiet_init=True, capture=True).decode('utf-8').rstrip()
+        s = self.assertPyrexContainerShellCommand(
+            'echo $TEST_ENV', env=env, quiet_init=True, capture=True).decode('utf-8').rstrip()
         self.assertEqual(s, test_string)
 
     def test_templateconf_rel(self):
@@ -590,8 +611,10 @@ def test_templateconf_rel(self):
 
         self.assertTrue(os.path.isabs(template_dir))
 
-        shutil.copyfile(os.path.join(PYREX_ROOT, 'poky/meta-poky/conf/local.conf.sample'), os.path.join(template_dir, 'local.conf.sample'))
-        shutil.copyfile(os.path.join(PYREX_ROOT, 'poky/meta-poky/conf/bblayers.conf.sample'), os.path.join(template_dir, 'bblayers.conf.sample'))
+        shutil.copyfile(os.path.join(PYREX_ROOT, 'poky/meta-poky/conf/local.conf.sample'),
+                        os.path.join(template_dir, 'local.conf.sample'))
+        shutil.copyfile(os.path.join(PYREX_ROOT, 'poky/meta-poky/conf/bblayers.conf.sample'),
+                        os.path.join(template_dir, 'bblayers.conf.sample'))
 
         test_string = 'set_by_test.%d' % threading.get_ident()
 
@@ -610,13 +633,15 @@ def test_templateconf_rel(self):
         env['TEMPLATECONF'] = os.path.relpath(template_dir, os.path.join(PYREX_ROOT, 'poky'))
         env['TEST_ENV'] = test_string
 
-        s = self.assertPyrexContainerShellCommand('echo $TEST_ENV', env=env, quiet_init=True, capture=True).decode('utf-8').rstrip()
+        s = self.assertPyrexContainerShellCommand(
+            'echo $TEST_ENV', env=env, quiet_init=True, capture=True).decode('utf-8').rstrip()
         self.assertEqual(s, test_string)
 
 
 TEST_IMAGES = ('ubuntu-14.04-base', 'ubuntu-16.04-base', 'ubuntu-18.04-base', 'centos-7-base',
                'ubuntu-14.04-oe', 'ubuntu-16.04-oe', 'ubuntu-18.04-oe')
 
+
 def add_image_tests():
     for image in TEST_IMAGES:
         (_, _, image_type) = image.split('-')
@@ -627,6 +652,7 @@ def add_image_tests():
         name = 'PyrexImage_' + re.sub(r'\W', '_', image)
         setattr(self, name, type(name, (parent, unittest.TestCase), {'test_image': image}))
 
+
 add_image_tests()
 
 if __name__ == "__main__":