Enable memory profiling

Author: kbattocchi

azure-pipelines.yml

@@ -48,163 +48,54 @@ jobs:
     name: output
     displayName: 'Determine type of code change'
 
-- template: azure-pipelines-steps.yml
-  parameters:
-    versions: ['3.6']
-    images: ['ubuntu-18.04']
-    package: '-e .[all]'
-    job:
-      job: 'Docs'
-      displayName: 'Build documentation'  
-      dependsOn: 'EvalChanges'
-      condition: eq(dependencies.EvalChanges.outputs['output.buildDocs'], 'True')
-      steps:
-      - script: 'sudo apt-get -yq install graphviz'
-        displayName: 'Install graphviz'
-
-      - script: 'pip install sklearn-contrib-lightning'
-        displayName: 'Install lightning'        
-
-      - script: 'pip install git+https://github.com/slundberg/shap.git@d1d2700acc0259f211934373826d5ff71ad514de'
-        displayName: 'Install specific version of shap'  
-
-      - script: 'pip install "sphinx < 3.2" sphinx_rtd_theme'
-        displayName: 'Install sphinx'
-
-      - script: 'python setup.py build_sphinx -W'
-        displayName: 'Build documentation'
-
-      - publish: 'build/sphinx/html'
-        artifact: 'Documentation'
-        displayName: 'Publish documentation as artifact'
-
-      - script: 'python setup.py build_sphinx -b doctest'
-        displayName: 'Run doctests'
-
 - template: azure-pipelines-steps.yml
   parameters:
     versions: ['3.8']
     images: ['ubuntu-18.04']
-    package: '-e .[tf]'
-    job:
-      job: 'Notebooks'
-      dependsOn: 'EvalChanges'
-      condition: eq(dependencies.EvalChanges.outputs['output.buildNbs'], 'True')
-      steps:
-      # Work around https://github.com/pypa/pip/issues/9542
-      - script: 'pip install -U numpy'
-        displayName: 'Upgrade numpy'
-
-      - script: 'pip install pytest-runner && python setup.py pytest'
-        displayName: 'Unit tests'
-        env:
-          PYTEST_ADDOPTS: '-m "notebook"'
-
-      - task: PublishTestResults@2
-        displayName: 'Publish Test Results **/test-results.xml'
-        inputs:
-          testResultsFiles: '**/test-results.xml'
-          testRunTitle: 'Notebooks'
-        condition: succeededOrFailed()
-
-# - job: 'AutoML'
-#   dependsOn: 'EvalChanges'
-#   condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True')
-#   variables:
-#     python.version: '3.6'
-#   pool:
-#     vmImage: 'ubuntu-18.04'
-#   steps:
-#   - template: azure-pipelines-steps.yml
-#     parameters:
-#       body:
-#         - task: AzureCLI@2
-#           displayName: 'AutoML tests'
-#           inputs:
-#             azureSubscription: 'automl'
-#             scriptLocation: 'inlineScript'
-#             scriptType: 'pscore'
-#             powerShellIgnoreLASTEXITCODE: '' # string for now due to https://github.com/microsoft/azure-pipelines-tasks/issues/12266
-#             inlineScript: |
-#               $env:SUBSCRIPTION_ID = az account show --query id -o tsv
-#               python setup.py pytest
-#           env:
-#             WORKSPACE_NAME: 'testWorkspace'
-#             RESOURCE_GROUP: 'testingAutoMLEconML'
-#             PYTEST_ADDOPTS: '-m "automl" -n 0'
-#             COVERAGE_PROCESS_START: 'setup.cfg'
-
-#         - task: PublishTestResults@2
-#           displayName: 'Publish Test Results **/test-results.xml'
-#           inputs:
-#             testResultsFiles: '**/test-results.xml'
-#             testRunTitle: 'AutoML'
-#           condition: succeededOrFailed()
-#       package: '.[automl]'
-
-- template: azure-pipelines-steps.yml
-  parameters:
-    versions: ['3.8']
-    images: ['macOS-10.15']
-    job:
-      job: 'Linting'
-      dependsOn: 'EvalChanges'
-      condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True')
-      steps:
-      - script: 'pip install pycodestyle && pycodestyle econml'
-        failOnStderr: true
-        displayName: Linting
-
-- template: azure-pipelines-steps.yml
-  parameters:
     package: '-e .[tf,plt]'
     job:
       job: Tests_no_dml
       dependsOn: 'EvalChanges'
       condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True')
       displayName: 'Run tests (non-DML)'
       steps:
-      - script: 'pip install pytest-runner && python setup.py pytest'
+      - script: 'pip install pytest-runner filprofiler && fil-profile run setup.py pytest'
         displayName: 'Unit tests'
         env:
-          PYTEST_ADDOPTS: '-m "not (notebook or automl or dml)"'
+          PYTEST_ADDOPTS: '-m "not (notebook or automl or dml or half)"'
           COVERAGE_PROCESS_START: 'setup.cfg'
       - task: PublishTestResults@2
         displayName: 'Publish Test Results **/test-results.xml'
         inputs:
           testResultsFiles: '**/test-results.xml'
           testRunTitle: 'Python $(python.version), image $(imageName)'
         condition: succeededOrFailed()
-
-      - task: PublishCodeCoverageResults@1
-        displayName: 'Publish Code Coverage Results'
-        inputs:
-          codeCoverageTool: Cobertura
-          summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
+      - publish: 'fil-result'
+        artifact: 'Profiling'
+        displayName: 'Publish profiling results as artifact'
 
 - template: azure-pipelines-steps.yml
   parameters:
+    versions: ['3.8']
+    images: ['ubuntu-18.04']
     package: '-e .[tf,plt]'
     job:
-      job: Tests_dml
+      job: Tests_no_dml
       dependsOn: 'EvalChanges'
       condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True')
-      displayName: 'Run tests (DML)'
+      displayName: 'Run tests (non-DML)'
       steps:
-      - script: 'pip install pytest-runner && python setup.py pytest'
+      - script: 'pip install pytest-runner filprofiler && fil-profile run setup.py pytest'
         displayName: 'Unit tests'
         env:
-          PYTEST_ADDOPTS: '-m "dml and (not (notebook or automl))"'
+          PYTEST_ADDOPTS: '-m "half and (not (notebook or automl or dml))"'
           COVERAGE_PROCESS_START: 'setup.cfg'
       - task: PublishTestResults@2
         displayName: 'Publish Test Results **/test-results.xml'
         inputs:
           testResultsFiles: '**/test-results.xml'
           testRunTitle: 'Python $(python.version), image $(imageName)'
         condition: succeededOrFailed()
-
-      - task: PublishCodeCoverageResults@1
-        displayName: 'Publish Code Coverage Results'
-        inputs:
-          codeCoverageTool: Cobertura
-          summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
+      - publish: 'fil-result'
+        artifact: 'Profiling'
+        displayName: 'Publish profiling results as artifact'

econml/tests/test_ate_inference.py

@@ -9,6 +9,7 @@
 from econml.inference import BootstrapInference
 
 
+@pytest.mark.half
 class TestATEInference(unittest.TestCase):
 
     @classmethod

econml/tests/test_bootstrap.py

@@ -13,6 +13,7 @@
 import joblib
 
 
+@pytest.mark.half
 class TestBootstrap(unittest.TestCase):
 
     def test_with_sklearn(self):

econml/tests/test_dowhy.py

@@ -11,6 +11,7 @@
 from sklearn.linear_model import LinearRegression, LogisticRegression, Lasso
 
 
+@pytest.mark.half
 class TestDowhy(unittest.TestCase):
 
     def _get_data(self):

econml/tests/test_drlearner.py

@@ -24,6 +24,7 @@
 import econml.tests.utilities  # bugfix for assertWarns
 
 
+@pytest.mark.half
 class TestDRLearner(unittest.TestCase):
 
     @classmethod

econml/tests/test_linear_model.py

@@ -16,6 +16,7 @@
 from sklearn.base import clone
 
 
+@pytest.mark.half
 class TestLassoExtensions(unittest.TestCase):
     """Test WeightedLasso."""
 

econml/tests/test_montecarlo.py

@@ -12,6 +12,7 @@
 import numpy as np
 
 
+@pytest.mark.half
 class TestMonteCarlo(unittest.TestCase):
 
     def test_montecarlo(self):

econml/tests/test_orf.py

@@ -14,6 +14,7 @@
 from econml.sklearn_extensions.linear_model import WeightedLassoCVWrapper
 
 
+@pytest.mark.half
 class TestOrthoForest(unittest.TestCase):
 
     @classmethod

econml/tests/test_ortho_learner.py

@@ -12,6 +12,7 @@
 import pytest
 
 
+@pytest.mark.half
 class TestOrthoLearner(unittest.TestCase):
 
     def test_crossfit(self):

econml/tests/test_rscorer.py

@@ -19,6 +19,7 @@ def _fit_model(name, model, Y, T, X):
     return name, model.fit(Y, T, X=X)
 
 
+@pytest.mark.half
 class TestRScorer(unittest.TestCase):
 
     def _get_data(self):

econml/tests/test_statsmodels.py

@@ -21,6 +21,7 @@
 from sklearn.preprocessing import PolynomialFeatures
 
 
+@pytest.mark.half
 class StatsModelsOLS:
     """
     Helper class to wrap a StatsModels OLS model to conform to the sklearn API.

econml/tests/test_two_stage_least_squares.py

@@ -14,6 +14,7 @@
 from sklearn.preprocessing import PolynomialFeatures
 
 
+@pytest.mark.half
 class Test2SLS(unittest.TestCase):
 
     def test_hermite_shape(self):

pyproject.toml

@@ -9,10 +9,11 @@ requires = [
 build-backend = "setuptools.build_meta"
 
 [tool.pytest.ini_options]
-addopts = "--junitxml=junit/test-results.xml -n auto --strict-markers --cov-config=setup.cfg --cov=econml --cov-report=xml"
+addopts = "--junitxml=junit/test-results.xml --strict-markers"
 markers = [    
     "slow",
     "notebook",
     "automl",
-    "dml"
+    "dml",
+    "half"
 ]