Support dependencies in pyproject.toml (#13)

* Update minimum_dependency_generator.py

* support pyproject toml

* fix integration test

* fix integration test

* fix filename

* fix pip

* fix output

* clean list 1

* better unit test

* better name

* lint

Author: Gaurav Sheni

.github/workflows/integration_tests.yml

@@ -97,29 +97,59 @@ jobs:
         uses: ./
         with:
           paths: 'setup_example.cfg'
-          options: 'install_requires setup_requires'
+          options: 'install_requires'
           extras_require: 'dev test'
           output_filepath: 'output_filepath.txt'
-      - name: Save the output minimum install_requires, setup_requires and dev, test requirements
+      - name: Save the output minimum install_requires and dev, test requirements
         run: printf "${{ steps.min_dep_gen_6.outputs.min_reqs }}" >> generated-min-setupcfg-multiple-reqs.txt
       - name: Verify output
         run: |
+
           echo "${{ steps.min_dep_gen_6.outputs.min_reqs }}" | grep -c "scipy==1.3.3"
           echo "${{ steps.min_dep_gen_6.outputs.min_reqs }}" | grep -c "pandas==1.3.0"
           echo "${{ steps.min_dep_gen_6.outputs.min_reqs }}" | grep -c "featuretools==1.5.0"
           echo "${{ steps.min_dep_gen_6.outputs.min_reqs }}" | grep -c "woodwork==0.8.1"
-          echo "${{ steps.min_dep_gen_6.outputs.min_reqs }}" | grep -c "setuptools==47"
-          echo "${{ steps.min_dep_gen_6.outputs.min_reqs }}" | grep -c "wheel==0.37.1"
           echo "${{ steps.min_dep_gen_6.outputs.min_reqs }}" | grep -c "autopep8==1.6.0"
           echo "${{ steps.min_dep_gen_6.outputs.min_reqs }}" | grep -c "pip==22.0.2"
+          echo "${{ steps.min_dep_gen_6.outputs.min_reqs }}" | grep -c "wheel==0.37.1"
           echo "${{ steps.min_dep_gen_6.outputs.min_reqs }}" | grep -c "pytest==5.2.0"
       - name: Check file existence
-        id: check_files
+        id: check_files_1
         uses: andstor/file-existence-action@v1
         with:
           files: "output_filepath.txt"
       - name: File exists
-        if: steps.check_files.outputs.files_exists == 'false'
+        if: steps.check_files_1.outputs.files_exists == 'false'
         run: exit 1
       - name: Print text file
         run: cat output_filepath.txt
+      - name: Run Minimum Dependency Generator with pyproject.toml
+        id: min_dep_gen_7
+        uses: ./
+        with:
+          paths: 'pyproject_example.toml'
+          options: 'dependencies'
+          extras_require: 'dev test'
+          output_filepath: 'output_filepath_toml.txt'
+      - name: Save the output minimum install_requires and dev, test requirements
+        run: printf "${{ steps.min_dep_gen_7.outputs.min_reqs }}" >> generated-min-pyproject-multiple-reqs.txt
+      - name: Verify output
+        run: |
+          echo "${{ steps.min_dep_gen_7.outputs.min_reqs }}" | grep -c "scipy==1.3.3"
+          echo "${{ steps.min_dep_gen_7.outputs.min_reqs }}" | grep -c "pandas==1.3.0"
+          echo "${{ steps.min_dep_gen_7.outputs.min_reqs }}" | grep -c "featuretools==1.5.0"
+          echo "${{ steps.min_dep_gen_7.outputs.min_reqs }}" | grep -c "woodwork==0.8.1"
+          echo "${{ steps.min_dep_gen_7.outputs.min_reqs }}" | grep -c "wheel==0.37.1"
+          echo "${{ steps.min_dep_gen_7.outputs.min_reqs }}" | grep -c "autopep8==1.6.0"
+          echo "${{ steps.min_dep_gen_7.outputs.min_reqs }}" | grep -c "pip==22.0.2"
+          echo "${{ steps.min_dep_gen_7.outputs.min_reqs }}" | grep -c "pytest==5.2.0"
+      - name: Check file existence
+        id: check_files_2
+        uses: andstor/file-existence-action@v1
+        with:
+          files: "output_filepath_toml.txt"
+      - name: Print text file
+        run: cat output_filepath_toml.txt
+      - name: File exists
+        if: steps.check_files_2.outputs.files_exists == 'false'
+        run: exit 1

README.md

@@ -39,7 +39,7 @@ steps:
     uses: alteryx/[email protected]
     with:
       paths: 'setup.cfg'
-      options: 'install_requires setup_requires'
+      options: 'install_requires'
       extras_require: 'dev test'
       output_filepath: 'generated-min-reqs.txt'
 ```

minimum_dependency_generator/minimum_dependency_generator.py

@@ -2,6 +2,7 @@
 import os
 from collections import defaultdict
 
+import tomli
 from packaging.requirements import Requirement
 from packaging.specifiers import Specifier
 
@@ -50,7 +51,7 @@ def determine_package_name(package):
     return name
 
 
-def clean_cfg_section(section):
+def clean_section(section):
     section = section.split('\n')
     section = [x for x in section if len(x) > 1]
     return section
@@ -112,10 +113,26 @@ def parse_setup_cfg(paths, options, extras_require):
     extras_require = clean_list_length_one(extras_require)
     if options and len(options) > 0:
         for option in options:
-            requirements += clean_cfg_section(config['options'][option])
+            requirements += clean_section(config['options'][option])
     if extras_require and len(extras_require) > 0:
         for extra in extras_require:
-            requirements += clean_cfg_section(config['options.extras_require'][extra])
+            requirements += clean_section(config['options.extras_require'][extra])
+    return requirements
+
+
+def parse_pyproject_toml(paths, options, extras_require):
+    requirements = []
+    with open(paths[0], "rb") as f:
+        toml_dict = tomli.load(f)
+
+    options = clean_list_length_one(options)
+    extras_require = clean_list_length_one(extras_require)
+    if options and len(options) > 0:
+        for option in options:
+            requirements += toml_dict['project'][option]
+    if extras_require and len(extras_require) > 0:
+        for extra in extras_require:
+            requirements += toml_dict['project']['optional-dependencies'][extra]
     return requirements
 
 
@@ -125,6 +142,8 @@ def generate_min_requirements(paths, options=None, extras_require=None, output_f
 
     if len(paths) == 1 and paths[0].endswith('.cfg') and os.path.basename(paths[0]).startswith('setup'):
         requirements = parse_setup_cfg(paths, options, extras_require)
+    elif len(paths) == 1 and paths[0].endswith('.toml') and os.path.basename(paths[0]).startswith('pyproject'):
+        requirements = parse_pyproject_toml(paths, options, extras_require)
     else:
         requirements = parse_requirements_text_file(paths)
 

minimum_dependency_generator/tests/conftest.py

@@ -44,3 +44,51 @@ def setuptools():
 @pytest.fixture(scope="session", autouse=True)
 def p_ytest_dep():
     return "pytest >= 5.2.0"
+
+
+@pytest.fixture(scope="session", autouse=True)
+def cfg_str(dask_dep, pandas_dep, woodwork_dep, numpy_lower, ploty_dep, numpy_upper, p_ytest_dep):
+    setup_cfg_str = f'''\
+    [metadata]
+    name = example_package
+
+    [options]
+    install_requires =
+        {dask_dep}
+        {pandas_dep}
+        {woodwork_dep}
+        {numpy_lower}
+
+    [options.extras_require] =
+    dev =
+        {ploty_dep}
+        {numpy_upper}
+    test =
+        {p_ytest_dep}
+    '''
+    return setup_cfg_str
+
+
+@pytest.fixture(scope="session", autouse=True)
+def toml_cfg(pandas_dep, woodwork_dep, numpy_upper, ploty_dep, p_ytest_dep, dask_dep, numpy_lower):
+    pyproject_str = f'''\
+    [project]
+    name = "example_package"
+    requires-python = ">=3.7,<3.10"
+    dependencies = [
+        "{pandas_dep}",
+        "{woodwork_dep}",
+        "{numpy_upper}",
+    ]
+
+    [project.optional-dependencies]
+    test = [
+        "{ploty_dep}",
+        "{p_ytest_dep}",
+    ]
+    dev = [
+        "{dask_dep}",
+        "{numpy_lower}",
+    ]
+    '''
+    return pyproject_str

minimum_dependency_generator/tests/test_setup_cfg.py renamed to minimum_dependency_generator/tests/test_cfg_toml.py

@@ -1,41 +1,34 @@
 import tempfile
 
+import pytest
+
 from ..minimum_dependency_generator import generate_min_requirements
 
 
-def test_with_setup_cfg(
-    dask_dep, pandas_dep, woodwork_dep, numpy_lower, ploty_dep, numpy_upper, setuptools, p_ytest_dep
+@pytest.mark.parametrize(
+    "file_prefix,file_extension,options",
+    [('setup', 'cfg', 'install_requires'), ('pyproject', 'toml', 'dependencies')],
+)
+def test_with_toml_cfg(
+    file_prefix, file_extension, options, cfg_str, toml_cfg
 ):
-    setup_cfg_str = f'''\
-    [metadata]
-    name = example_package
-
-    [options]
-    install_requires =
-        {dask_dep}
-        {pandas_dep}
-        {woodwork_dep}
-        {numpy_lower}
-    setup_requires =
-        {setuptools}
-
-    [options.extras_require] =
-    koalas =
-        {ploty_dep}
-        {numpy_upper}
-    test =
-        {p_ytest_dep}
-    '''
+    file_str = toml_cfg
+    if file_prefix == 'setup':
+        file_str = cfg_str
     with tempfile.NamedTemporaryFile(
-        mode="w", suffix=".cfg", prefix="setup"
-    ) as setup_cfg_f:
-        setup_cfg_f.write(setup_cfg_str)
-        setup_cfg_f.flush()
+        mode="w", suffix="." + file_extension, prefix=file_prefix
+    ) as pyproject_file:
+        pyproject_file.write(file_str)
+        pyproject_file.flush()
 
-        paths = [setup_cfg_f.name]
-        options = ['install_requires setup_requires']
-        extra_requires = ['koalas test']
+        paths = [pyproject_file.name]
+        options = [options]
+        extra_requires = ['test dev']
         min_requirements = generate_min_requirements(paths, options, extra_requires)
+    verify_min_reqs_cfg_toml(min_requirements)
+
+
+def verify_min_reqs_cfg_toml(min_requirements):
     assert '-r' not in min_requirements
     assert '.txt' not in min_requirements
     assert 'core-requirements.txt' not in min_requirements
@@ -48,7 +41,6 @@ def test_with_setup_cfg(
         "pandas==0.24.1",
         "plotly==4.14.0",
         "pytest==5.2.0",
-        "setuptools==47",
         "woodwork==0.0.11",
     ]
     expected_min_reqs = sorted(expected_min_reqs)

pyproject_example.toml

@@ -0,0 +1,20 @@
+[project]
+name = "my_custom_pkg"
+dependencies = [
+    "scipy>=1.3.3",
+    "pandas>=1.3.0",
+    "featuretools>=1.5.0,<2.0.0",
+    "woodwork>=0.8.1",
+]
+
+[project.optional-dependencies]
+dev = [
+    "autopep8>=1.6.0",
+    "pandas<1.4.0",
+]
+
+test = [
+    "pip>=22.0.2",
+    "wheel>=0.37.1",
+    "pytest>=5.2.0",
+]

requirements.txt

@@ -1,3 +1,4 @@
 packaging>=20.9
 requests>=2.25.1
-wheel>=0.36.2
+wheel>=0.36.2
+tomli>=2.0.1

setup_example.cfg

@@ -7,10 +7,6 @@ install_requires =
     pandas >= 1.3.0
     featuretools >= 1.5.0, <2.0.0
     woodwork >= 0.8.1
-setup_requires =
-    setuptools >= 47
-    wheel
-python_requires = >= 3.8, <4
 
 [options.extras_require]
 dev =