Merge pull request #23 from DoubleML/s-update-irm-scripts

Update irm scripts

Author: SvenKlaassen

.github/workflows/apo_sim.yml

@@ -17,7 +17,8 @@ jobs:
     strategy:
       matrix:
         script: [
-          'scripts/irm/irm_apo_coverage.py',
+          'scripts/irm/apo.py',
+          'scripts/irm/apos.py',
         ]
 
     steps:
@@ -47,28 +48,37 @@ jobs:
       with:
         ref: ${{ env.TARGET_BRANCH }}
 
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        version: "0.7.8"
+  
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version-file: "monte-cover/pyproject.toml"
 
-    - name: Install dependencies
+    - name: Install Monte-Cover
       run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        cd monte-cover
+        uv venv
+        uv sync
 
     - name: Install DoubleML from correct branch
       run: |
-        pip uninstall -y doubleml
-        pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}"
+        source monte-cover/.venv/bin/activate
+        uv pip uninstall doubleml
+        uv pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}"
 
     - name: Set up Git configuration
       run: |
         git config --global user.name 'github-actions'
         git config --global user.email '[email protected]'
 
     - name: Run scripts
-      run: python ${{ matrix.script }}
+      run: |
+        source monte-cover/.venv/bin/activate
+        uv run ${{ matrix.script }}
 
     - name: Commit any existing changes
       run: |

.github/workflows/irm_sim.yml

@@ -17,10 +17,10 @@ jobs:
     strategy:
       matrix:
         script: [
-          'scripts/irm/irm_ate_coverage.py',
-          'scripts/irm/irm_atte_coverage.py',
-          'scripts/irm/irm_cate_coverage.py',
-          'scripts/irm/irm_gate_coverage.py',
+          'scripts/irm/irm_ate.py',
+          'scripts/irm/irm_atte.py',
+          'scripts/irm/irm_cate.py',
+          'scripts/irm/irm_gate.py',
           'scripts/irm/irm_ate_sensitivity.py',
           'scripts/irm/irm_atte_sensitivity.py',
         ]
@@ -52,28 +52,37 @@ jobs:
       with:
         ref: ${{ env.TARGET_BRANCH }}
 
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        version: "0.7.8"
+
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version-file: "monte-cover/pyproject.toml"
 
-    - name: Install dependencies
+    - name: Install Monte-Cover
       run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        cd monte-cover
+        uv venv
+        uv sync
 
     - name: Install DoubleML from correct branch
       run: |
-        pip uninstall -y doubleml
-        pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}"
+        source monte-cover/.venv/bin/activate
+        uv pip uninstall doubleml
+        uv pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}"
 
     - name: Set up Git configuration
       run: |
         git config --global user.name 'github-actions'
         git config --global user.email '[email protected]'
 
     - name: Run scripts
-      run: python ${{ matrix.script }}
+      run: |
+        source monte-cover/.venv/bin/activate
+        uv run ${{ matrix.script }}
 
     - name: Commit any existing changes
       run: |

.github/workflows/quant_sim.yml

@@ -17,9 +17,9 @@ jobs:
     strategy:
       matrix:
         script: [
-          'scripts/irm/cvar_coverage.py',
-          'scripts/irm/pq_coverage.py',
-          'scripts/irm/lpq_coverage.py',
+          'scripts/irm/cvar.py',
+          'scripts/irm/pq.py',
+          'scripts/irm/lpq.py',
         ]
 
     steps:
@@ -49,28 +49,37 @@ jobs:
       with:
         ref: ${{ env.TARGET_BRANCH }}
 
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        version: "0.7.8"
+
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version-file: "monte-cover/pyproject.toml"
 
-    - name: Install dependencies
+    - name: Install Monte-Cover
       run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
+        cd monte-cover
+        uv venv
+        uv sync
 
     - name: Install DoubleML from correct branch
       run: |
-        pip uninstall -y doubleml
-        pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}"
+        source monte-cover/.venv/bin/activate
+        uv pip uninstall doubleml
+        uv pip install "doubleml @ git+https://github.com/DoubleML/doubleml-for-py@${{ env.DML_BRANCH }}"
 
     - name: Set up Git configuration
       run: |
         git config --global user.name 'github-actions'
         git config --global user.email '[email protected]'
 
     - name: Run scripts
-      run: python ${{ matrix.script }}
+      run: |
+        source monte-cover/.venv/bin/activate
+        uv run ${{ matrix.script }}
 
     - name: Commit any existing changes
       run: |

.github/workflows/ssm_sim.yml

@@ -17,8 +17,8 @@ jobs:
     strategy:
       matrix:
         script: [
-          'scripts/irm/ssm_mar_ate_coverage.py',
-          'scripts/irm/ssm_nonignorable_ate_coverage.py',
+          'scripts/ssm/ssm_mar_ate.py',
+          'scripts/ssm/ssm_nonig_ate.py',
         ]
 
     steps:
@@ -73,7 +73,7 @@ jobs:
 
     - name: Commit any existing changes
       run: |
-        git add results/irm
+        git add results/ssm
         git commit -m "Update results from script: ${{ matrix.script }}" || echo "No changed results to commit"
 
     - name: Wait random time

doc/irm/apo.qmd

@@ -30,7 +30,7 @@ The simulations are based on the  the [make_irm_data_discrete_treatments](https:
 
 ```{python}
 #| echo: false
-metadata_file = '../../results/irm/irm_apo_coverage_metadata.csv'
+metadata_file = '../../results/irm/apo_metadata.csv'
 metadata_df = pd.read_csv(metadata_file)
 print(metadata_df.T.to_string(header=False))
 ```
@@ -41,7 +41,7 @@ print(metadata_df.T.to_string(header=False))
 #| echo: false
 
 # set up data
-df_apo = pd.read_csv("../../results/irm/irm_apo_coverage_apo.csv", index_col=None)
+df_apo = pd.read_csv("../../results/irm/apo_coverage.csv", index_col=None)
 
 assert df_apo["repetition"].nunique() == 1
 n_rep_apo = df_apo["repetition"].unique()[0]
@@ -88,7 +88,7 @@ The non-uniform results (coverage, ci length and bias) refer to averaged values
 
 ```{python}
 #| echo: false
-metadata_file = '../../results/irm/irm_apo_coverage_metadata.csv'
+metadata_file = '../../results/irm/apos_metadata.csv'
 metadata_df = pd.read_csv(metadata_file)
 print(metadata_df.T.to_string(header=False))
 ```
@@ -99,7 +99,7 @@ print(metadata_df.T.to_string(header=False))
 #| echo: false
 
 # set up data
-df_apos = pd.read_csv("../../results/irm/irm_apo_coverage_apos.csv", index_col=None)
+df_apos = pd.read_csv("../../results/irm/apos_coverage.csv", index_col=None)
 
 assert df_apos["repetition"].nunique() == 1
 n_rep_apos = df_apos["repetition"].unique()[0]
@@ -144,7 +144,7 @@ The non-uniform results (coverage, ci length and bias) refer to averaged values
 
 ```{python}
 #| echo: false
-metadata_file = '../../results/irm/irm_apo_coverage_metadata.csv'
+metadata_file = '../../results/irm/apos_metadata.csv'
 metadata_df = pd.read_csv(metadata_file)
 print(metadata_df.T.to_string(header=False))
 ```
@@ -155,7 +155,7 @@ print(metadata_df.T.to_string(header=False))
 #| echo: false
 
 # set up data
-df_contrast = pd.read_csv("../../results/irm/irm_apo_coverage_apos_contrast.csv", index_col=None)
+df_contrast = pd.read_csv("../../results/irm/apos_causal_contrast.csv", index_col=None)
 
 assert df_contrast["repetition"].nunique() == 1
 n_rep_contrast = df_contrast["repetition"].unique()[0]

doc/irm/irm.qmd

@@ -86,7 +86,7 @@ As for the ATE, the simulations are based on the  the [make_irm_data](https://do
 
 ```{python}
 #| echo: false
-metadata_file = '../../results/irm/irm_atte_coverage_metadata.csv'
+metadata_file = '../../results/irm/irm_atte_metadata.csv'
 metadata_df = pd.read_csv(metadata_file)
 print(metadata_df.T.to_string(header=False))
 ```
@@ -135,7 +135,7 @@ generate_and_show_styled_table(
 
 ## Sensitivity
 
-The simulations are based on the  the ADD-DGP with $10,000$ observations. As the DGP is nonlinear, we will only use corresponding learners. Since the DGP includes an unobserved confounder, we would expect a bias in the ATE estimates, leading to low coverage of the true parameter.
+The simulations are based on the  the [make_confounded_irm_data](https://docs.doubleml.org/stable/api/generated/doubleml.datasets.make_confounded_irm_data.html#doubleml.datasets.make_confounded_irm_data)-DGP with $5,000$ observations. Since the DGP includes an unobserved confounder, we would expect a bias in the ATE estimates, leading to low coverage of the true parameter.
 
 The confounding is set such that both sensitivity parameters are approximately $cf_y=cf_d=0.1$, such that the robustness value $RV$ should be approximately $10\%$.
 Further, the corresponding confidence intervals are one-sided (since the direction of the bias is unkown), such that only one side should approximate the corresponding coverage level (here only the lower coverage is relevant since the bias is positive). Remark that for the coverage level the value of $\rho$ has to be correctly specified, such that the coverage level will be generally (significantly) larger than the nominal level under the conservative choice of $|\rho|=1$.
@@ -157,7 +157,7 @@ print(metadata_df.T.to_string(header=False))
 #| echo: false
 
 # set up data and rename columns
-df_ate_sens = pd.read_csv("../../results/irm/irm_ate_sensitivity.csv", index_col=None)
+df_ate_sens = pd.read_csv("../../results/irm/irm_ate_sensitivity_coverage.csv", index_col=None)
 
 assert df_ate_sens["repetition"].nunique() == 1
 n_rep_ate_sens = df_ate_sens["repetition"].unique()[0]
@@ -211,7 +211,7 @@ print(metadata_df.T.to_string(header=False))
 #| echo: false
 
 # set up data
-df_atte_sens = pd.read_csv("../../results/irm/irm_atte_sensitivity.csv", index_col=None)
+df_atte_sens = pd.read_csv("../../results/irm/irm_atte_sensitivity_coverage.csv", index_col=None)
 
 assert df_atte_sens["repetition"].nunique() == 1
 n_rep_atte_sens = df_atte_sens["repetition"].unique()[0]

doc/irm/qte.qmd

@@ -31,7 +31,7 @@ The non-uniform results (coverage, ci length and bias) refer to averaged values
 
 ```{python}
 #| echo: false
-metadata_file = '../../results/irm/pq_coverage_metadata.csv'
+metadata_file = '../../results/irm/pq_metadata.csv'
 metadata_df = pd.read_csv(metadata_file)
 print(metadata_df.T.to_string(header=False))
 ```
@@ -42,7 +42,7 @@ print(metadata_df.T.to_string(header=False))
 #| echo: false
 
 # set up data
-df_qte = pd.read_csv("../../results/irm/pq_coverage_qte.csv", index_col=None)
+df_qte = pd.read_csv("../../results/irm/pq_effect_coverage.csv", index_col=None)
 
 assert df_qte["repetition"].nunique() == 1
 n_rep_qte = df_qte["repetition"].unique()[0]
@@ -85,7 +85,7 @@ generate_and_show_styled_table(
 #| echo: false
 
 # set up data
-df_pq0 = pd.read_csv("../../results/irm/pq_coverage_pq0.csv", index_col=None)
+df_pq0 = pd.read_csv("../../results/irm/pq_Y0_coverage.csv", index_col=None)
 
 assert df_pq0["repetition"].nunique() == 1
 n_rep_pq0 = df_pq0["repetition"].unique()[0]
@@ -125,7 +125,7 @@ generate_and_show_styled_table(
 #| echo: false
 
 # set up data and rename columns
-df_pq1 = pd.read_csv("../../results/irm/pq_coverage_pq1.csv", index_col=None)
+df_pq1 = pd.read_csv("../../results/irm/pq_Y1_coverage.csv", index_col=None)
 
 assert df_pq1["repetition"].nunique() == 1
 n_rep_pq1 = df_pq1["repetition"].unique()[0]
@@ -161,15 +161,15 @@ generate_and_show_styled_table(
 
 ## LQTE
 
-The results are based on a location-scale model as described the corresponding [Example](https://docs.doubleml.org/stable/examples/py_double_ml_pq.html#Local-Potential-Quantiles-(LPQs)) with $10,000$ observations.
+The results are based on a location-scale model as described the corresponding [Example](https://docs.doubleml.org/stable/examples/py_double_ml_pq.html#Local-Potential-Quantiles-(LPQs)) with $5,000$ observations.
 
 The non-uniform results (coverage, ci length and bias) refer to averaged values over all quantiles (point-wise confidende intervals).
 
 ::: {.callout-note title="Metadata"  collapse="true"}
 
 ```{python}
 #| echo: false
-metadata_file = '../../results/irm/lpq_coverage_metadata.csv'
+metadata_file = '../../results/irm/lpq_metadata.csv'
 metadata_df = pd.read_csv(metadata_file)
 print(metadata_df.T.to_string(header=False))
 ```
@@ -180,7 +180,7 @@ print(metadata_df.T.to_string(header=False))
 #| echo: false
 
 # set up data
-df_lqte = pd.read_csv("../../results/irm/lpq_coverage_lqte.csv", index_col=None)
+df_lqte = pd.read_csv("../../results/irm/lpq_effect_coverage.csv", index_col=None)
 
 assert df_lqte["repetition"].nunique() == 1
 n_rep_lqte = df_lqte["repetition"].unique()[0]
@@ -222,7 +222,7 @@ generate_and_show_styled_table(
 #| echo: false
 
 # set up data
-df_lpq0 = pd.read_csv("../../results/irm/lpq_coverage_lpq0.csv", index_col=None)
+df_lpq0 = pd.read_csv("../../results/irm/lpq_Y0_coverage.csv", index_col=None)
 
 assert df_lpq0["repetition"].nunique() == 1
 n_rep_lpq0 = df_lpq0["repetition"].unique()[0]
@@ -262,7 +262,7 @@ generate_and_show_styled_table(
 #| echo: false
 
 # set up data
-df_lpq1 = pd.read_csv("../../results/irm/lpq_coverage_lpq1.csv", index_col=None)
+df_lpq1 = pd.read_csv("../../results/irm/lpq_Y1_coverage.csv", index_col=None)
 
 assert df_lpq1["repetition"].nunique() == 1
 n_rep_lpq1 = df_lpq1["repetition"].unique()[0]
@@ -306,7 +306,7 @@ The non-uniform results (coverage, ci length and bias) refer to averaged values
 
 ```{python}
 #| echo: false
-metadata_file = '../../results/irm/cvar_coverage_metadata.csv'
+metadata_file = '../../results/irm/cvar_metadata.csv'
 metadata_df = pd.read_csv(metadata_file)
 print(metadata_df.T.to_string(header=False))
 ```
@@ -317,7 +317,7 @@ print(metadata_df.T.to_string(header=False))
 #| echo: false
 
 # set up data
-df_cvar_qte = pd.read_csv("../../results/irm/cvar_coverage_qte.csv", index_col=None)
+df_cvar_qte = pd.read_csv("../../results/irm/cvar_effect_coverage.csv", index_col=None)
 
 assert df_cvar_qte["repetition"].nunique() == 1
 n_rep_cvar_qte = df_cvar_qte["repetition"].unique()[0]
@@ -359,7 +359,7 @@ generate_and_show_styled_table(
 #| echo: false
 
 # set up data
-df_cvar_pq0 = pd.read_csv("../../results/irm/cvar_coverage_pq0.csv", index_col=None)
+df_cvar_pq0 = pd.read_csv("../../results/irm/cvar_Y0_coverage.csv", index_col=None)
 
 assert df_cvar_pq0["repetition"].nunique() == 1
 n_rep_cvar_pq0 = df_cvar_pq0["repetition"].unique()[0]
@@ -399,7 +399,7 @@ generate_and_show_styled_table(
 #| echo: false
 
 # set up data
-df_cvar_pq1 = pd.read_csv("../../results/irm/cvar_coverage_pq1.csv", index_col=None)
+df_cvar_pq1 = pd.read_csv("../../results/irm/cvar_Y1_coverage.csv", index_col=None)
 
 assert df_cvar_pq1["repetition"].nunique() == 1
 n_rep_cvar_pq1 = df_cvar_pq1["repetition"].unique()[0]