Update output when project not found

caniusepython3/__init__.py

@@ -21,33 +21,7 @@
 
 import multiprocessing
 
-
 try:
     CPU_COUNT = max(2, multiprocessing.cpu_count())
 except NotImplementedError:  #pragma: no cover
     CPU_COUNT = 2
-
-
-def check(requirements_paths=[], metadata=[], projects=[]):
-    """Return True if all of the specified dependencies have been ported to Python 3.
-
-    The requirements_paths argument takes a sequence of file paths to
-    requirements files. The 'metadata' argument takes a sequence of strings
-    representing metadata. The 'projects' argument takes a sequence of project
-    names.
-
-    Any project that is not listed on PyPI will be considered ported.
-    """
-    dependencies = []
-    dependencies.extend(projects_.projects_from_requirements(requirements_paths))
-    dependencies.extend(projects_.projects_from_metadata(metadata))
-    dependencies.extend(projects)
-
-    manual_overrides = pypi.manual_overrides()
-
-    for dependency in dependencies:
-        if dependency in manual_overrides:
-            continue
-        elif not pypi.supports_py3(dependency):
-            return False
-    return True

caniusepython3/__main__.py

@@ -68,34 +68,47 @@ def projects_from_cli(args):
 
 def message(blockers):
     """Create a sequence of key messages based on what is blocking."""
+    encoding = getattr(sys.stdout, 'encoding', '')
+    if encoding:
+        encoding = encoding.lower()
+
     if not blockers:
-        encoding = getattr(sys.stdout, 'encoding', '')
-        if encoding:
-            encoding = encoding.lower()
         if encoding == 'utf-8':
             # party hat
             flair = "\U0001F389  "
         else:
             flair = ''
-        return [flair +
-                'You have 0 projects blocking you from using Python 3!']
+
+        return [flair + 'You have no projects known to block you from using Python 3!']
+
     flattened_blockers = set()
     for blocker_reasons in blockers:
         for blocker in blocker_reasons:
             flattened_blockers.add(blocker)
-    need = 'You need {0} project{1} to transition to Python 3.'
-    formatted_need = need.format(len(flattened_blockers),
-                      's' if len(flattened_blockers) != 1 else '')
+
+    if encoding == 'utf-8':
+        # sad face
+        flair = "\U00002639  "
+    else:
+        flair = ''
+
+    need = '{0}You need {1} project{2} to transition to Python 3.'
+    formatted_need = need.format(
+        flair,
+        len(flattened_blockers),
+        's' if len(flattened_blockers) != 1 else ''
+    )
     can_port = ('Of {0} {1} project{2}, {3} {4} no direct dependencies '
-                'blocking {5} transition:')
+                'known to block {5} transition:')
     formatted_can_port = can_port.format(
             'those' if len(flattened_blockers) != 1 else 'that',
             len(flattened_blockers),
             's' if len(flattened_blockers) != 1 else '',
             len(blockers),
             'have' if len(blockers) != 1 else 'has',
             'their' if len(blockers) != 1 else 'its')
-    return formatted_need, formatted_can_port
+
+    return [formatted_need, formatted_can_port]
 
 
 def pprint_blockers(blockers):
@@ -118,21 +131,64 @@ def pprint_blockers(blockers):
     return pprinted
 
 
+def output_not_on_pypi(not_on_pypi):
+    lines = []
+
+    if len(not_on_pypi) > 0:
+        compat_status = 'The following {0} project{1} could not be found on pypi.org:'.format(
+            len(not_on_pypi),
+            's' if len(not_on_pypi) != 1 else '',
+        )
+        lines.append(compat_status)
+
+        for project in not_on_pypi:
+            lines.append(' {0}'.format(project))
+
+    return lines
+
+
+def output_not_on_distlib(not_on_distlib):
+    lines = []
+
+    if len(not_on_distlib) > 0:
+        compat_status = 'The depedencies of following {0} project{1} could not be found using distlib:'.format(
+            len(not_on_distlib),
+            's' if len(not_on_distlib) != 1 else '',
+        )
+        lines.append(compat_status)
+
+        for project in not_on_distlib:
+            lines.append(' {0}'.format(project))
+
+    return lines
+
+
 def check(projects):
     """Check the specified projects for Python 3 compatibility."""
     log = logging.getLogger('ciu')
     log.info('{0} top-level projects to check'.format(len(projects)))
     print('Finding and checking dependencies ...')
-    blockers = dependencies.blockers(projects)
+    blockers, not_on_pypi, not_on_distlib = dependencies.blockers(projects)
 
     print('')
     for line in message(blockers):
         print(line)
 
-    print('')
+    if len(blockers) > 0:
+        print('')
     for line in pprint_blockers(blockers):
         print(' ', line)
 
+    if len(not_on_pypi) > 0:
+        print('')
+    for line in output_not_on_pypi(not_on_pypi):
+        print(line)
+
+    if len(not_on_distlib) > 0:
+        print('')
+    for line in output_not_on_distlib(not_on_distlib):
+        print(line)
+
     return len(blockers) == 0
 
 

caniusepython3/dependencies.py

@@ -16,6 +16,7 @@
 
 import distlib.locators
 import packaging.utils
+import urllib
 
 import caniusepython3 as ciu
 from caniusepython3 import pypi
@@ -57,7 +58,6 @@ def dependencies(project_name):
     log.info('Locating dependencies for {}'.format(project_name))
     located = distlib.locators.locate(project_name, prereleases=True)
     if not located:
-        log.warning('{0} not found'.format(project_name))
         return None
     return {packaging.utils.canonicalize_name(pypi.just_name(dep))
             for dep in located.run_requires}
@@ -75,25 +75,32 @@ def supports_py3(project_name):
 
     check = []
     evaluated = set(overrides)
+    not_on_pypi = []
+
     for project in project_names:
         log.info('Checking top-level project: {0} ...'.format(project))
         evaluated.add(project)
-        if not supports_py3(project):
+        ported = supports_py3(project)
+
+        if ported is False:
             check.append(project)
+        elif ported is None:
+            not_on_pypi.append(project)
+
     reasons = {project: None for project in check}
+
     thread_pool_executor = concurrent.futures.ThreadPoolExecutor(
             max_workers=ciu.CPU_COUNT)
+    not_on_distlib = []
     with thread_pool_executor as executor:
         while len(check) > 0:
             new_check = []
             for parent, deps in zip(check, executor.map(dependencies, check)):
                 if deps is None:
-                    # Can't find any results for a project, so ignore it so as
-                    # to not accidentally consider indefinitely that a project
-                    # can't port.
-                    del reasons[parent]
+                    # Can't find any results for a project.
+                    not_on_distlib.append(parent)
                     continue
-                log.info('Dependencies of {0}: {1}'.format(project, deps))
+                log.info('Dependencies of {0}: {1}'.format(parent, deps))
                 unchecked_deps = []
                 for dep in deps:
                     if dep in evaluated:
@@ -104,11 +111,14 @@ def supports_py3(project_name):
                                   executor.map(supports_py3,
                                                unchecked_deps))
                 for dep, ported in deps_status:
-                    if not ported:
+                    if ported is False:
                         reasons[dep] = parent
                         new_check.append(dep)
+                    elif ported is None:
+                        not_on_pypi.append(parent)
                     # Make sure there's no data race in recording a dependency
                     # has been evaluated but not reported somewhere.
                     evaluated.add(dep)
             check = new_check
-    return reasons_to_paths(reasons)
+
+    return reasons_to_paths(reasons), not_on_pypi, not_on_distlib

caniusepython3/pypi.py

@@ -81,10 +81,7 @@ def supports_py3(project_name):
     log.info("Checking {} ...".format(project_name))
     request = requests.get("https://pypi.org/pypi/{}/json".format(project_name))
     if request.status_code >= 400:
-        log = logging.getLogger("ciu")
-        log.warning("problem fetching {}, assuming ported ({})".format(
-                        project_name, request.status_code))
-        return True
+        return None  # Represents unknown.
     response = request.json()
     return any(c.startswith("Programming Language :: Python :: 3")
                for c in response["info"]["classifiers"])

caniusepython3/test/test_cli.py

@@ -149,7 +149,7 @@ def test_message_plural(self):
         self.assertEqual(2, len(messages))
         want = 'You need 2 projects to transition to Python 3.'
         self.assertEqual(messages[0], want)
-        want = ('Of those 2 projects, 2 have no direct dependencies blocking '
+        want = ('Of those 2 projects, 2 have no direct dependencies known to block '
                 'their transition:')
         self.assertEqual(messages[1], want)
 
@@ -159,22 +159,37 @@ def test_message_singular(self):
         self.assertEqual(2, len(messages))
         want = 'You need 1 project to transition to Python 3.'
         self.assertEqual(messages[0], want)
-        want = ('Of that 1 project, 1 has no direct dependencies blocking '
+        want = ('Of that 1 project, 1 has no direct dependencies known to block '
                 'its transition:')
         self.assertEqual(messages[1], want)
 
+    def test_message_with_unknowns(self):
+        unknowns = [['A']]
+        messages = ciu_main.message([])
+        expected = ['You have no projects known to block you from using Python 3!']
+        self.assertEqual(expected, messages)
+
+    def test_output_unknowns(self):
+        unknowns = ['A']
+        outputs = ciu_main.output_not_on_pypi(unknowns)
+        self.assertEqual(2, len(outputs))
+        want = 'The following 1 project could not be found on pypi.org:'
+        self.assertEqual(outputs[0], want)
+        want = (' ' + unknowns[0][0])
+        self.assertEqual(outputs[1], want)
+
     @mock.patch('sys.stdout', autospec=True)
     def test_message_no_blockers_flair_on_utf8_terminal(self, mock_stdout):
         mock_stdout.encoding = 'UTF-8'
         messages = ciu_main.message([])
-        expected = ['\U0001f389  You have 0 projects blocking you from using Python 3!']
+        expected = ['\U0001f389  You have no projects known to block you from using Python 3!']
         self.assertEqual(expected, messages)
 
     @mock.patch('sys.stdout', autospec=True)
     def test_message_no_blockers(self, mock_stdout):
         mock_stdout.encoding = None
         messages = ciu_main.message([])
-        expected = ['You have 0 projects blocking you from using Python 3!']
+        expected = ['You have no projects known to block you from using Python 3!']
         self.assertEqual(expected, messages)
 
     def test_pprint_blockers(self):
@@ -203,7 +218,7 @@ def test_verbose_output(self):
         ciu_main.projects_from_cli(['-v', '-p', 'ipython'])
         self.assertTrue(logging.getLogger('ciu').isEnabledFor(logging.INFO))
 
-    @mock.patch('caniusepython3.dependencies.blockers', lambda projects: ['blocker'])
+    @mock.patch('caniusepython3.dependencies.blockers', lambda projects: (['blocker'], [], []))
     def test_nonzero_return_code(self):
         args = ['--projects', 'foo', 'bar.baz']
         with self.assertRaises(SystemExit) as context:

caniusepython3/test/test_command.py

@@ -42,7 +42,7 @@ def test_extras_require(self):
         got = frozenset(cmd._dependencies())
         self.assertEqual(got, frozenset(['pip']))
 
-    @mock.patch('caniusepython3.dependencies.blockers', lambda projects: ['blocker'])
+    @mock.patch('caniusepython3.dependencies.blockers', lambda projects: (['blocker'], [], []))
     def test_nonzero_return_code(self):
         cmd = make_command({'install_requires': ['pip']})
         with self.assertRaises(SystemExit) as context:

caniusepython3/test/test_dependencies.py

@@ -67,7 +67,7 @@ def __init__(self, run_requires):
 class NetworkTests(unittest.TestCase):
 
     def test_blockers(self):
-        got = frozenset(dependencies.blockers(['ralph_scrooge']))
+        got = frozenset(dependencies.blockers(['ralph_scrooge'])[0])
         want = frozenset([('ralph', 'ralph_scrooge'), ('ralph-assets', 'ralph_scrooge')])
         self.assertEqual(got, want)
 
@@ -80,11 +80,11 @@ def test_dependencies_no_project(self):
         self.assertIsNone(got)
 
     def test_blockers_no_project(self):
-        got = dependencies.blockers(['asdfsadfdsfsdffdfadf'])
+        got = dependencies.blockers(['asdfsadfdsfsdffdfadf'])[0]
         self.assertEqual(got, frozenset())
 
     def test_manual_overrides(self):
-        self.assertEqual(dependencies.blockers(["unittest2"]), frozenset())
+        self.assertEqual(dependencies.blockers(["unittest2"])[0], frozenset())
 
 
 if __name__ == '__main__':

caniusepython3/test/test_pypi.py

@@ -65,5 +65,4 @@ class NetworkTests(unittest.TestCase):
     def test_supports_py3(self):
         self.assertTrue(pypi.supports_py3("caniusepython3"))
         self.assertFalse(pypi.supports_py3("pil"))
-        # Unfound projects are considered ported.
-        self.assertTrue(pypi.supports_py3("sdfffavsafasvvavfdvavfavf"))
+        self.assertEqual(pypi.supports_py3("sdfffavsafasvvavfdvavfavf"), None)