scrapsheet.html

<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en"><head>

<meta charset="utf-8">
<meta name="generator" content="quarto-1.6.39">

<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes">


<title>Scrapsheet</title>
<style>
code{white-space: pre-wrap;}
span.smallcaps{font-variant: small-caps;}
div.columns{display: flex; gap: min(4vw, 1.5em);}
div.column{flex: auto; overflow-x: auto;}
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
ul.task-list{list-style: none;}
ul.task-list li input[type="checkbox"] {
  width: 0.8em;
  margin: 0 0.8em 0.2em -1em; /* quarto-specific, see https://github.com/quarto-dev/quarto-cli/issues/4556 */ 
  vertical-align: middle;
}
/* CSS for syntax highlighting */
pre > code.sourceCode { white-space: pre; position: relative; }
pre > code.sourceCode > span { line-height: 1.25; }
pre > code.sourceCode > span:empty { height: 1.2em; }
.sourceCode { overflow: visible; }
code.sourceCode > span { color: inherit; text-decoration: inherit; }
div.sourceCode { margin: 1em 0; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
pre > code.sourceCode { white-space: pre-wrap; }
pre > code.sourceCode > span { display: inline-block; text-indent: -5em; padding-left: 5em; }
}
pre.numberSource code
  { counter-reset: source-line 0; }
pre.numberSource code > span
  { position: relative; left: -4em; counter-increment: source-line; }
pre.numberSource code > span > a:first-child::before
  { content: counter(source-line);
    position: relative; left: -1em; text-align: right; vertical-align: baseline;
    border: none; display: inline-block;
    -webkit-touch-callout: none; -webkit-user-select: none;
    -khtml-user-select: none; -moz-user-select: none;
    -ms-user-select: none; user-select: none;
    padding: 0 4px; width: 4em;
  }
pre.numberSource { margin-left: 3em;  padding-left: 4px; }
div.sourceCode
  {   }
@media screen {
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
}
</style>


<script src="scrapsheet_files/libs/clipboard/clipboard.min.js"></script>
<script src="scrapsheet_files/libs/quarto-html/quarto.js"></script>
<script src="scrapsheet_files/libs/quarto-html/popper.min.js"></script>
<script src="scrapsheet_files/libs/quarto-html/tippy.umd.min.js"></script>
<script src="scrapsheet_files/libs/quarto-html/anchor.min.js"></script>
<link href="scrapsheet_files/libs/quarto-html/tippy.css" rel="stylesheet">
<link href="scrapsheet_files/libs/quarto-html/quarto-syntax-highlighting-e26003cea8cd680ca0c55a263523d882.css" rel="stylesheet" id="quarto-text-highlighting-styles">
<script src="scrapsheet_files/libs/bootstrap/bootstrap.min.js"></script>
<link href="scrapsheet_files/libs/bootstrap/bootstrap-icons.css" rel="stylesheet">
<link href="scrapsheet_files/libs/bootstrap/bootstrap-8a79a254b8e706d3c925cde0a310d4f0.min.css" rel="stylesheet" append-hash="true" id="quarto-bootstrap" data-mode="light">

  <script src="https://cdnjs.cloudflare.com/polyfill/v3/polyfill.min.js?features=es6"></script>
  <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml-full.js" type="text/javascript"></script>

<script type="text/javascript">
const typesetMath = (el) => {
  if (window.MathJax) {
    // MathJax Typeset
    window.MathJax.typeset([el]);
  } else if (window.katex) {
    // KaTeX Render
    var mathElements = el.getElementsByClassName("math");
    var macros = [];
    for (var i = 0; i < mathElements.length; i++) {
      var texText = mathElements[i].firstChild;
      if (mathElements[i].tagName == "SPAN") {
        window.katex.render(texText.data, mathElements[i], {
          displayMode: mathElements[i].classList.contains('display'),
          throwOnError: false,
          macros: macros,
          fleqn: false
        });
      }
    }
  }
}
window.Quarto = {
  typesetMath
};
</script>

</head>

<body class="fullcontent">

<div id="quarto-content" class="page-columns page-rows-contents page-layout-article">

<main class="content" id="quarto-document-content">

<header id="title-block-header" class="quarto-title-block default">
<div class="quarto-title">
<h1 class="title">Scrapsheet</h1>
</div>


<div class="quarto-title-meta">

    
  </div>
  

</header>


<section id="other-places" class="level2">
<h2 class="anchored" data-anchor-id="other-places">Other Places</h2>
<ul>
<li>bbq chips</li>
<li>Restaurants
<ul>
<li>Spinelli’s
<ul>
<li>1 supreme w/o mush</li>
<li>1 supreme</li>
</ul></li>
</ul></li>
</ul>
</section>
<section id="grocery-list" class="level2">
<h2 class="anchored" data-anchor-id="grocery-list">Grocery list</h2>
<ul>
<li>banana</li>
<li>celery</li>
<li>lettuce</li>
<li>tomato</li>
<li>Pastry</li>
<li>Buns</li>
<li>bread - dad sourdough</li>
<li>cheese and summer sausage</li>
<li>3 - yogurts</li>
<li>Snack
<ul>
<li>1- dried fruit</li>
<li>1 - nut</li>
<li>3 - dark choc</li>
</ul></li>
<li>Frozen
<ul>
<li>2 - fries</li>
<li>1 - pizza</li>
<li>2 - mini pizza</li>
<li>2 - ice cream</li>
<li>1 - pot pie</li>
<li>2 - burrito</li>
<li>1 - breaded fish</li>
<li>1 - veg</li>
</ul></li>
<li>3 - juice</li>
<li>3 - creamer</li>
<li>1 - beano</li>
</ul>
</section>
<section id="misc" class="level2">
<h2 class="anchored" data-anchor-id="misc">Misc</h2>
<ul>
<li>Hierarchical Bootstrap
<ul>
<li>bootstrap types
<ul>
<li><a href="https://www.r-bloggers.com/2019/09/understanding-bootstrap-confidence-interval-output-from-the-r-boot-package/" class="uri">https://www.r-bloggers.com/2019/09/understanding-bootstrap-confidence-interval-output-from-the-r-boot-package/</a></li>
</ul></li>
<li><a href="https://radlfabs.github.io/posts/thesis/">Uncertainty quantification for cross-validation</a>
<ul>
<li>Procedure as described (<a href="https://radlfabs.github.io/posts/thesis/#ref-davison_bootstrap_1997">Davison and Hinkley 1997</a>; <a href="https://radlfabs.github.io/posts/thesis/#ref-goldstein_bootstrapping_2010">Goldstein 2010</a>)
<ul>
<li><p>Sample with replacement from the fold indices –&gt;</p></li>
<li><p>Sample with replacement from the validation preds of the newly set of fold indices</p></li>
<li><p>Calculate CV estimate on the new set of validation preds</p></li>
<li><p>CI Variants: basic, normal, studentized, percentile</p></li>
</ul></li>
</ul></li>
<li>My understanding
<ol type="1">
<li>Sample fold indices w/replacement</li>
<li>For each sampled fold’s validation set, sample the predictions w/replacement</li>
<li>For each sampled fold’s predictions on the validation set, calculate the score</li>
<li>Average the scores across the folds.</li>
<li>Repeat 1-10K times</li>
<li>Calculate CI variant on the distribution of averaged scores</li>
</ol></li>
<li>Questions
<ul>
<li>How many bootstrap iterations?</li>
</ul></li>
</ul></li>
<li>Positron
<ul>
<li>Resources
<ul>
<li>2024 Posit conference <a href="https://www.youtube.com/watch?v=8uRcB34Hhsw&amp;ab_channel=PositPBC">video</a> (7 10-min talks?)</li>
</ul></li>
</ul></li>
</ul>
</section>
<section id="drawdown-implied-correlation-dic" class="level2">
<h2 class="anchored" data-anchor-id="drawdown-implied-correlation-dic">Drawdown Implied Correlation (DIC)</h2>
<ul>
<li><p>Notes from</p>
<ul>
<li><a href="https://cssanalytics.wordpress.com/2024/12/23/drawdown-implied-correlations-part-1/">Drawdown Implied Correlations (Part 1)</a></li>
<li><a href="https://cssanalytics.wordpress.com/2025/01/09/drawdown-implied-correlations-part-2-generalized-downside-implied-correlations/">Drawdown Implied Correlations Part 2: Generalized Downside Implied Correlations</a></li>
<li><a href="https://cssanalytics.wordpress.com/2025/01/21/iterative-psd-shrinkage-ips/">Iterative PSD Shrinkage (IPS)</a></li>
</ul></li>
<li><p>Formula</p>
<p><span class="math display">\[
\text{DIC} = \frac{4\text{MDD}_{AB}^2-\text{MDD}_{A}^2-\text{MDD}_{B}^2}{2\cdot \text{MDD}_{A} \cdot \text{MDD}_{B}}
\]</span></p>
<ul>
<li><span class="math inline">\(\text{MDD}\)</span> is the Maximum Drawdown</li>
<li><span class="math inline">\(A\)</span> is an asset, <span class="math inline">\(B\)</span> is an asset and <span class="math inline">\(AB\)</span> is a portfolio with both <span class="math inline">\(A\)</span> and <span class="math inline">\(B\)</span></li>
</ul></li>
<li><p>Asset returns are not Normal — they have “fat tails”</p></li>
<li><p>Two assets can lose money simultaneously, even while maintaining a negative (pearson) correlation, leaving the portfolio exposed to significant losses.</p>
<ul>
<li>We typically rely on a dynamic or rolling correlation to measure diversification</li>
</ul></li>
<li><p>In contrast to correlations and volatility, drawdowns are nonlinear and path-dependent making them complementary for risk analysis.</p></li>
<li><p>For the DIC measure, you can certainly use drawdowns entirely within a lookback window to keep the measure mathematically consistent, but it is recommended that you use a much bigger window for calculation to avoid a lot of noise.</p>
<ul>
<li>Regardless using drawdowns from all-time highs will slighly change the final values in such a way that they can be more negative than -1 which is why you need to bound the DIC between 1 and -1 to provide a practical correlation measure.</li>
</ul></li>
<li><p>When constructing a portfolio with multiple assets, the portfolio’s drawdown series (the peak-to-trough losses of the combined portfolio) behaves differently than the individual drawdown series of the constituent assets. This difference arises from how the assets interact in a portfolio.</p>
<ul>
<li>The drawdown of the portfolio is not simply the sum or average of individual asset drawdowns; instead, it reflects the combined behavior of the assets as they interact over time. Two or more assets in the portfolio may experience drawdowns at different times or to different extents, and their drawdown implied correlations will directly influence how the portfolio’s total drawdown evolves.</li>
<li>If two assets experience drawdowns simultaneously, their joint drawdown will be greater than what you would expect from either asset alone, this will lead to a measurement of high correlation.</li>
<li>If the portfolio drawdown is moderate to low compared to the individual assets’ drawdowns this will lead to a measurement of low correlation.</li>
<li>Therefore, the drawdown of the portfolio can reflect behavior and interactions between assets that individual asset drawdowns and returns cannot capture. This is the key reason correlating individual asset drawdowns will not fully explain the portfolio drawdowns.</li>
</ul></li>
<li><p>Single Reference Process</p>
<ol type="1">
<li>Drawdown Calculation for Each Asset:
<ul>
<li><p>For Asset A and Asset B, calculate the drawdowns from their respective all-time highs over a rolling window (e.g., 60 days).</p></li>
<li><p>For the joint time series (AB), calculate the combined drawdown from all-time highs over the same 60-day window.</p></li>
</ul></li>
<li>Find Maximum Drawdown for AB:
<ul>
<li><p>Identify the maximum drawdown for the joint time series (AB) over the 60-day rolling window.</p></li>
<li><p>Retrieve the corresponding drawdown values for Asset A and Asset B on the same day that the maximum drawdown for AB occurs.</p></li>
</ul></li>
<li>Compute the DIC:
<ul>
<li><p>Calculate the implied correlation between the drawdowns of A, B, and AB on the specific day.</p></li>
<li><p>This gives the DIC for the pair of assets based on the maximum drawdown for the joint time series (AB).</p></li>
</ul></li>
</ol></li>
<li><p>The DIC can be calculated using only one drawdown point while you need a minimum of 3 data points to compute a correlation between drawdowns.</p></li>
<li><p>The “standard” version of the DIC which uses the max drawdown over some window.</p></li>
<li><p>Note you can certainly use the top % or drawdowns above a threshold as well. But because we are only looking at maximum drawdowns with this variation, in order to create a rolling daily measurement I suggest a slight modification to the original calculation by using a “triple point” reference.</p></li>
<li><p>Triple Reference</p>
<ul>
<li><p>This means we are going to look at three reference points which represent the maximum drawdown for each asset and the portfolio. The purpose of a triple reference is to get as much information as possible from a shorter window and increase accuracy while reducing indicator volatility.</p></li>
<li><p>Calculating DIC at the point of maximum drawdown (individually) for A , B, and the joint series AB and averaging the three results.<br>
(need pic)</p></li>
<li><p>The maximum drawdown for AB could be influenced by an unusually strong movement in one asset, which might not reflect the risk dynamics between A and B themselves. By averaging the DIC from the three scenarios (max drawdown of AB, A, and B), you smooth out this potential bias and get a more robust measure of correlation.</p></li>
</ul></li>
<li><p>Triple Reference Process</p>
<ol type="1">
<li><p>Find the point of Max Drawdown for Asset A:</p>
<ul>
<li><p>Now, repeat the process but for Asset A as the reference. Find the maximum drawdown for A over the same 60-day rolling window.</p></li>
<li><p>Retrieve the corresponding drawdown values for Asset B and AB on the same day that the maximum drawdown for A occurs. Calculate the DIC using the exact same formula.</p></li>
</ul></li>
<li><p>Find the point of Max Drawdown for Asset B:</p>
<ul>
<li><p>Similarly, find the maximum drawdown for Asset B over the same 60-day window.</p></li>
<li><p>Retrieve the corresponding drawdown values for A and AB on the same day as the maximum drawdown for B. Calculate the DIC using the same formula.</p></li>
</ul></li>
<li><p>Calculate and Average DICs:</p>
<ul>
<li><p>You now have three DICs: one from the maximum drawdown for AB, one from the maximum drawdown for A, and one from the maximum drawdown for B.</p></li>
<li><p>The final DIC is the average of these three DICs, providing a comprehensive view of the correlation during drawdown periods for both individual assets and their joint performance.</p></li>
</ul></li>
</ol></li>
</ul>
</section>
<section id="dbt" class="level2">
<h2 class="anchored" data-anchor-id="dbt">DBT</h2>
<section id="dbt-expectations" class="level3">
<h3 class="anchored" data-anchor-id="dbt-expectations">dbt-expectations</h3>
<ul>
<li>Feaures
<ul>
<li>Free package</li>
<li>Integrates into already existing dbt project</li>
<li>Assertive testing</li>
</ul></li>
<li>Set-Up
<ul>
<li><p>Specify in package.yml</p>
<div class="sourceCode" id="cb1"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="fu">packages</span><span class="kw">:</span></span>
<span id="cb1-2"><a href="#cb1-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">package</span><span class="kw">:</span><span class="at"> calogica/dbt_expectations</span></span>
<span id="cb1-3"><a href="#cb1-3" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">version</span><span class="kw">:</span><span class="at"> </span><span class="fl">0.10.4</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div></li>
<li><p>Run <code>dbt deps</code> to install</p></li>
</ul></li>
</ul>
<section id="tests" class="level4">
<h4 class="anchored" data-anchor-id="tests">Tests</h4>
<ul>
<li>Tests can be specified on a model, source, seed, or column in one of your YAML files</li>
<li>Source Data
<ul>
<li><p>Always apply your tests to the source data if possible</p></li>
<li><p>Because sources are defined in your YAML files, this is where you will want to write your tests</p>
<div class="sourceCode" id="cb2"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb2-1"><a href="#cb2-1" aria-hidden="true" tabindex="-1"></a><span class="fu">sources</span><span class="kw">:</span></span>
<span id="cb2-2"><a href="#cb2-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">name</span><span class="kw">:</span><span class="at"> company_customers</span></span>
<span id="cb2-3"><a href="#cb2-3" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">database</span><span class="kw">:</span><span class="at"> company</span></span>
<span id="cb2-4"><a href="#cb2-4" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">schema</span><span class="kw">:</span><span class="at"> customers</span></span>
<span id="cb2-5"><a href="#cb2-5" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">description</span><span class="kw">:</span><span class="at"> </span><span class="st">"Contains personal customer information for company"</span></span>
<span id="cb2-6"><a href="#cb2-6" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">tables</span><span class="kw">:</span></span>
<span id="cb2-7"><a href="#cb2-7" aria-hidden="true" tabindex="-1"></a><span class="at">      </span><span class="kw">-</span><span class="at"> </span><span class="fu">name</span><span class="kw">:</span><span class="at"> addresses</span></span>
<span id="cb2-8"><a href="#cb2-8" aria-hidden="true" tabindex="-1"></a><span class="at">        </span><span class="fu">description</span><span class="kw">:</span><span class="at"> </span><span class="st">"Customer addresses for the company"</span></span>
<span id="cb2-9"><a href="#cb2-9" aria-hidden="true" tabindex="-1"></a><span class="at">        </span><span class="fu">tests</span><span class="kw">:</span></span>
<span id="cb2-10"><a href="#cb2-10" aria-hidden="true" tabindex="-1"></a><span class="at">          </span><span class="kw">-</span><span class="at"> </span><span class="fu">dbt_expectations.expect_table_column_count_to_be_between</span><span class="kw">:</span></span>
<span id="cb2-11"><a href="#cb2-11" aria-hidden="true" tabindex="-1"></a><span class="at">              </span><span class="fu">min_value</span><span class="kw">:</span><span class="at"> </span><span class="dv">1</span><span class="at"> </span></span>
<span id="cb2-12"><a href="#cb2-12" aria-hidden="true" tabindex="-1"></a><span class="at">              </span><span class="fu">max_value</span><span class="kw">:</span><span class="at"> </span><span class="dv">10</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<ul>
<li>Specify the test under one of the source’s tables and not the source itself</li>
<li>check to make sure the <span class="var-text">customers.addresses</span> table has between 1-10 columns.</li>
</ul></li>
</ul></li>
<li>Models
<ul>
<li><p>Adding tests to your complex data models is great for ensuring your data is as expected <em>after</em> the transformation process.</p></li>
<li><p>Similar sytax to specifying in Source Data</p>
<div class="sourceCode" id="cb3"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="fu">models</span><span class="kw">:</span></span>
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">name</span><span class="kw">:</span><span class="at"> stg_addresses</span></span>
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">description</span><span class="kw">:</span><span class="at"> </span><span class="st">"Customer addresses for the company"</span></span>
<span id="cb3-4"><a href="#cb3-4" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">tests</span><span class="kw">:</span></span>
<span id="cb3-5"><a href="#cb3-5" aria-hidden="true" tabindex="-1"></a><span class="at">      </span><span class="kw">-</span><span class="at"> </span><span class="fu">dbt_expectations.expect_table_column_count_to_be_between</span><span class="kw">:</span></span>
<span id="cb3-6"><a href="#cb3-6" aria-hidden="true" tabindex="-1"></a><span class="at">          </span><span class="fu">min_value</span><span class="kw">:</span><span class="at"> </span><span class="dv">1</span><span class="at"> </span></span>
<span id="cb3-7"><a href="#cb3-7" aria-hidden="true" tabindex="-1"></a><span class="at">          </span><span class="fu">max_value</span><span class="kw">:</span><span class="at"> </span><span class="dv">10</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<ul>
<li>Tests on the addresses staging model</li>
</ul></li>
</ul></li>
<li>Column
<ul>
<li><p>Can only apply the test to one column</p></li>
<li><p>Preferrable on source data</p></li>
<li><p>Make sure the column names of your sources and models are fully documented before you can implement column tests.</p>
<div class="sourceCode" id="cb4"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="fu">models</span><span class="kw">:</span></span>
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">name</span><span class="kw">:</span><span class="at"> stg_addresses</span></span>
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">description</span><span class="kw">:</span><span class="at"> </span><span class="st">"Customer addresses for the company"</span></span>
<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a><span class="at">    </span><span class="fu">columns</span><span class="kw">:</span></span>
<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a><span class="at">      </span><span class="kw">-</span><span class="at"> </span><span class="fu">name</span><span class="kw">:</span><span class="at"> address_id </span></span>
<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a><span class="at">        </span><span class="fu">description</span><span class="kw">:</span><span class="at"> </span><span class="st">"The primary key of this table"</span></span>
<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a><span class="at">        </span><span class="fu">tests</span><span class="kw">:</span></span>
<span id="cb4-8"><a href="#cb4-8" aria-hidden="true" tabindex="-1"></a><span class="at">          </span><span class="kw">-</span><span class="at"> dbt_expectations.expect_column_to_exist</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<ul>
<li>Tests whether <span class="var-text">address_id</span> exists as a column within the model</li>
</ul></li>
</ul></li>
<li>Check for Recent Data
<ul>
<li><p>Applies to one column only</p></li>
<li><p>Recommendation: Making this interval 3 days max, so you can catch the issue at the source fairly quickly</p></li>
<li><p>Use Case: FiveTran shows data connector syncs working, but you aren’t sure</p>
<div class="sourceCode" id="cb5"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb5-1"><a href="#cb5-1" aria-hidden="true" tabindex="-1"></a><span class="fu">tests</span><span class="kw">:</span></span>
<span id="cb5-2"><a href="#cb5-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">dbt_expectations.expect_row_values_to_have_recent_data</span><span class="kw">:</span></span>
<span id="cb5-3"><a href="#cb5-3" aria-hidden="true" tabindex="-1"></a><span class="at">        </span><span class="fu">datepart</span><span class="kw">:</span><span class="at"> day</span></span>
<span id="cb5-4"><a href="#cb5-4" aria-hidden="true" tabindex="-1"></a><span class="at">        </span><span class="fu">interval</span><span class="kw">:</span><span class="at"> </span><span class="dv">3</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<ul>
<li>If there is no new data from the last 3 days then the test will throw an error.</li>
</ul></li>
</ul></li>
<li>Compare Column Values
<ul>
<li><p>Applies to models, seeds, and sources</p></li>
<li><p>Compares if the value in column A is greater than the value in column B.</p>
<div class="sourceCode" id="cb6"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="fu">tests</span><span class="kw">:</span></span>
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">dbt_expectations.expect_column_pair_values_A_to_be_greater_than_B</span><span class="kw">:</span></span>
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a><span class="at">      </span><span class="fu">column_A</span><span class="kw">:</span><span class="at"> total_amount</span></span>
<span id="cb6-4"><a href="#cb6-4" aria-hidden="true" tabindex="-1"></a><span class="at">      </span><span class="fu">column_B</span><span class="kw">:</span><span class="at"> sub_total</span></span>
<span id="cb6-5"><a href="#cb6-5" aria-hidden="true" tabindex="-1"></a><span class="at">      </span><span class="fu">or_equal</span><span class="kw">:</span><span class="at"> </span><span class="ch">False</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<ul>
<li>If the source contains high-quality data, the total should always be greater than the subtotal</li>
</ul></li>
</ul></li>
<li>Check Column Type
<ul>
<li><p>Use Case: If your source is a spreadsheet, they are particularly prone to these types of data entry errors</p>
<div class="sourceCode" id="cb7"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="fu">tests</span><span class="kw">:</span></span>
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">dbt_expectations.expect_column_values_to_be_of_type</span><span class="kw">:</span></span>
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a><span class="at">      </span><span class="fu">column_type</span><span class="kw">:</span><span class="at"> timestamp_ntz</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<ul>
<li>This makes sure the time stamp is a format.</li>
<li>Different time stamps across different models and data sources can become an issue with joins, etc.</li>
</ul></li>
</ul></li>
<li>Add Row Conditions
<ul>
<li><p>These can be added to other tests</p></li>
<li><p>A commin condition is “id is not null”</p>
<div class="sourceCode" id="cb8"><pre class="sourceCode yaml code-with-copy"><code class="sourceCode yaml"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="fu">tests</span><span class="kw">:</span></span>
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a><span class="at">  </span><span class="kw">-</span><span class="at"> </span><span class="fu">dbt_expectations.expect_column_values_to_be_in_set</span><span class="kw">:</span></span>
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a><span class="at">      </span><span class="fu">value_set</span><span class="kw">:</span><span class="at"> </span><span class="kw">[</span><span class="st">'cat'</span><span class="kw">,</span><span class="st">'dog'</span><span class="kw">,</span><span class="st">'pig'</span><span class="kw">]</span></span>
<span id="cb8-4"><a href="#cb8-4" aria-hidden="true" tabindex="-1"></a><span class="at">      </span><span class="fu">quote_values</span><span class="kw">:</span><span class="at"> </span><span class="ch">true</span><span class="at"> </span></span>
<span id="cb8-5"><a href="#cb8-5" aria-hidden="true" tabindex="-1"></a><span class="at">      </span><span class="fu">row_condition</span><span class="kw">:</span><span class="at"> </span><span class="st">"animal_id is not null"</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div>
<ul>
<li>This test only looks at the column values whose row does <em>not</em>have a null <span class="var-text">animal_id</span>.</li>
</ul></li>
</ul></li>
</ul>
</section>
</section>
</section>
<section id="functional-data-analysis-and-forecasting" class="level2">
<h2 class="anchored" data-anchor-id="functional-data-analysis-and-forecasting">Functional Data Analysis and Forecasting</h2>
<ul>
<li>Each observation in a functional dataset consists of a collection of points representing a continuous curve or surface over a compact domain (e.g., a fixed length of time or region of space).</li>
<li>Functional data provide detailed information of a continuous process</li>
<li>Apply a dimension reduction technique, such as fPCA, and use a subset of the transformed features as predictor variables
<ul>
<li>Only accounts for vertical functional variability, where <em>vertical</em> variability (also known as y or amplitude variability) is the variability in the height of functions</li>
</ul></li>
<li>Horizontal variability (also known as x or phase variability) is the variability in the location of peaks and valleys of the functions.</li>
<li>Requires enough data so that smoothing functions can accurately interpolate the curve.</li>
<li>Me
<ul>
<li>Seems like uber-nonlinear modeling. The coefficients and the predictors are smoothing functions. No flexibility on which terms get smoothers — all do. Also, the RHS is an integral.</li>
<li>Seems like this can be used for multivariate/group time series forecasting or as a feature reduction technique</li>
</ul></li>
<li>Packages
<ul>
<li><span style="color: #990000">{</span><a href="https://cran.r-project.org/web/packages/refund/index.html" style="color: #990000">refund</a><span style="color: #990000">}</span> - Methods for regression for functional data, including function-on-scalar, scalar-on-function, and function-on-function regression.</li>
<li><span style="color: #990000">{</span><a href="https://astamm.github.io/roahd/" style="color: #990000">roahd</a><span style="color: #990000">}</span> - The Robust Analysis of High-dimensional Data package allows to use a set of statistical tools for the exploration and robustification of univariate and multivariate functional datasets through the use of depth-based statistical methods.
<ul>
<li>Functions for generating functional data</li>
<li>Band depths and modified band depths,</li>
<li>Modified band depths for multivariate functional data,</li>
<li>Epigraph and hypograph indexes,</li>
<li>Spearman and Kendall’s correlation indexes for functional data,</li>
<li>Confidence intervals and tests on Spearman’s correlation coefficients for univariate and multivariate functional data.</li>
</ul></li>
<li><span style="color: #990000">{</span><a href="https://cran.r-project.org/web/packages/veesa/index.html" style="color: #990000">veesa</a><span style="color: #990000">}</span> (<a href="https://arxiv.org/abs/2501.07602">Paper</a>) - Pipeline for Explainable Machine Learning with Functional Data
<ul>
<li>Accounts for the vertical and horizontal variability in the functional data</li>
<li>Provides an explanation in the original data space of how the model uses variability in the functional data for prediction</li>
</ul></li>
<li><a href="https://cran.r-project.org/web/packages/hdftsa/index.html">hdftsa</a> - High-Dimensional Functional Time Series Analysis</li>
</ul></li>
<li>Types of Functional Data
<ul>
<li>age-specific mortality rates (Shang et all, 2024)
<ul>
<li><a href="https://arxiv.org/abs/2411.12423">Nonstationary functional time series forecasting</a></li>
<li><a href="https://arxiv.org/abs/2305.19749">Forecasting high-dimensional functional time series: Application to sub-national age-specific mortality</a>
<ul>
<li>age- and sex-specific mortality rates in the United States, France, and Japan, in which there are 51 states, 95 departments, and 47 prefectures</li>
</ul></li>
</ul></li>
<li>heights of children measured over time (Ramsay and Silverman, 2005)</li>
<li>silhouettes of animals extracted from images (Srivastava and Klassen, 2016)</li>
<li>glucose monitoring (Danne et al., 2017)</li>
<li>fitness tracking (Henriksen et al., 2018)</li>
<li>environmental sensors (Butts-Wilmsmeyer, Rapp and Guthrie, 2020)</li>
</ul></li>
</ul>
</section>
<section id="gaussian-processes" class="level2">
<h2 class="anchored" data-anchor-id="gaussian-processes">Gaussian Processes</h2>
<ul>
<li>Notes from
<ul>
<li><a href="https://arxiv.org/abs/2411.05869">Compactly-supported nonstationary kernels for computing exact Gaussian processes on big data</a>
<ul>
<li>alternative kernel that can discover and encode both sparsity and nonstationarity</li>
</ul></li>
</ul></li>
<li>A preeminent framework for stochastic function approximation, statistical modeling of real-world measurements, and non-parametric and nonlinear regression within machine learning (ML) and surrogate modeling.</li>
<li>Unlike many other machine learning methods, GPs include an implicit characterization of uncertainty</li>
<li>Traditional implementations of GPs involve stationary kernels (also termed covariance functions) that limit their flexibility and exact methods for inference that prevent application to data sets with more than about ten thousand points. (paper and its packages fix this)
<ul>
<li>Other methods to fix this are generally difficult to implement for large data sets due to their large numbers of hyperparameters which leads to overfitting and the need for specialized algorithms for training</li>
</ul></li>
</ul>
</section>
<section id="wavelets" class="level2">
<h2 class="anchored" data-anchor-id="wavelets">Wavelets</h2>
<ul>
<li><p>Notes from</p>
<ul>
<li><a href="https://arxiv.org/abs/2406.05012">TrendLSW: Trend and Spectral Estimation of Nonstationary Time Series in R</a></li>
<li>ChatGPT
<ul>
<li><a href="https://chatgpt.com/c/672cd99f-7480-8002-80cd-39263f52950b">My link</a></li>
<li><a href="https://chatgpt.com/share/672cddd9-ff2c-8002-a105-be470cc41dc2">Public Link</a></li>
</ul></li>
</ul></li>
<li><p>Wavelets are particularly suited for analyzing nonstationary time series because they can capture both time and frequency information.</p></li>
<li><p>Spectral estimation</p>
<ul>
<li>Spectral estimation is a technique used to analyze the frequency content of time series data, particularly focusing on how the variance (or power) is distributed across different frequencies. This is especially useful in nonstationary time series, where statistical properties, such as the mean and variance, change over time.</li>
<li>In classical settings, spectral estimation often involves the Fourier transform, where stationary processes are assumed. For nonstationary processes, wavelet-based methods are popular.</li>
<li>Usage
<ul>
<li>Identifying underlying periodicities, understanding the evolution of variance across different time periods, and detecting anomalies or regime shifts in time series.</li>
<li>If the time series contains periodic behavior (such as seasonal patterns), the spectral plot will show high power at the corresponding frequency</li>
<li>In an EWS plot, you may see that a certain frequency band has high power only during certain time intervals, indicating a time-localized periodicity.</li>
<li>A flat or consistent spectral plot indicates stationarity, while time-varying plots (especially with wavelet-based approaches) show how different scales contribute at different times, revealing nonstationarity. An increasing or decreasing trend in a particular frequency band over time might indicate a nonstationary process.</li>
<li>Sudden spikes or drops in spectral power at specific times and frequencies could indicate anomalies, abrupt changes, or unusual behavior in the time series.
<ul>
<li>A sudden burst (i.e.&nbsp;transient, not consistent) of power at a low scales (high-frequency) may indicate an abrupt event, such as a machine breakdown in industrial monitoring data</li>
<li>If the burst occurs at high scales (low frequencies), it may indicate a sudden, large-scale trend change, such as a long-term shift or event.</li>
</ul></li>
<li>In economic time series, higher scales (low frequencies) might represent long-term economic cycles, while lower scales (high frequencies) might correspond to short-term market volatility.</li>
<li>Analysts can detect sub-seasonal or irregular cycles that might not be immediately obvious.</li>
<li>If the model’s spectral plot aligns with the observed data’s spectral plot, this indicates a good fit. Discrepancies in power or patterns suggest areas where the model might need improvement.
<ul>
<li>e.g.&nbsp;After fitting a time series model, an analyst might generate a simulated spectral plot and compare it with the real data to see if the model captures both the trends and the variabilities at different scales.</li>
</ul></li>
<li>Time-frequency or time-scale plots provide a localized view of how spectral properties change, enabling analysts to detect events or features that occur intermittently.
<ul>
<li>e.g.&nbsp;In seismic data, an EWS plot could reveal bursts of energy at different scales corresponding to earthquake tremors and aftershocks.</li>
</ul></li>
<li>High power at certain frequencies suggests strong periodicity or correlation at corresponding time lags.
<ul>
<li>e.g.&nbsp;In biological signals, changes in autocorrelation can be related to transitions between different physiological states (e.g., sleep stages).</li>
</ul></li>
</ul></li>
</ul></li>
<li><p>Scales</p>
<ul>
<li>Scales refer to the different levels of resolution at which the data is analyzed. These scales are analogous to frequencies in Fourier analysis but provide a time-localized view of how signal components of different frequencies evolve over time.</li>
<li>Each scale <span class="math inline">\(j\)</span> corresponds to a specific level of detail, and the evolutionary wavelet spectrum <span class="math inline">\(S_j(z)\)</span>estimates the power at scale <span class="math inline">\(j\)</span> over time <span class="math inline">\(z\)</span>. This allows for the identification of how variance at different frequencies changes across time, which is crucial for analyzing nonstationary time series.</li>
<li>The wavelet function at a particular scale acts like a band-pass filter, capturing specific ranges of frequencies. Lower scales capture higher-frequency details (short-term fluctuations), while higher scales capture lower-frequency trends (long-term patterns).</li>
<li>At higher scales, the wavelet stretches over a longer portion of the time series, capturing slower, low-frequency changes. At lower scales, the wavelet is more compressed, capturing faster, high-frequency changes. Unlike Fourier transforms, which assume a fixed frequency range, wavelets allow for more adaptive, localized analysis.</li>
<li><strong>Low Scales (High Frequencies)</strong>: Capture fine, fast-varying details, like noise or short-term oscillations.</li>
</ul>
<!-- -->
<ul>
<li><p><strong>High Scales (Low Frequencies)</strong>: Capture broad, slow-varying features, such as long-term trends or cycles.</p></li>
<li><p>Scales to Periodicity</p>
<ul>
<li><p>Frequency Relationship<br>
<span class="math display">\[
f_j = \frac{k}{2^j \Delta t}
\]</span></p>
<ul>
<li><p><span class="math inline">\(f_j\)</span> : Frequency at scale <span class="math inline">\(j\)</span></p></li>
<li><p><span class="math inline">\(\Delta t\)</span> : Time step of your data (e.g.&nbsp;1 day for daily, 1 sec for seconds, etc.)</p></li>
<li><p><span class="math inline">\(k\)</span> : Constant that depends on the choice of wavelet</p>
<ul>
<li>e.g.&nbsp;Morlet wavelet, <span class="math inline">\(k \approx 1.03\)</span></li>
</ul></li>
</ul></li>
<li><p>Scale to Periodicity<br>
<span class="math display">\[
\begin{align}
P_j &amp;= \frac{1}{f_j} \\
&amp;\approx \frac{2^j \Delta t}{k}
\end{align}
\]</span></p></li>
<li><p><span class="ribbon-highlight">Example</span>: 1 sec data with power detected at <span class="math inline">\(S_5\)</span> (scale 5) using a Morlet wavelet<br>
<span class="math display">\[
P_5 \approx \frac{2^5 \times 1}{1.03} \approx 31.07 \;\mbox{sec}
\]</span></p>
<ul>
<li>If the high power persists over time in your wavelet spectrum (such as in an Evolutionary Wavelet Spectrum or Scalogram plot), it suggests a consistent periodic signal at that time scale.</li>
</ul>
<!-- -->
<ul>
<li>If the high power is transient, it indicates that the periodic behavior is localized in time, occurring only during certain time intervals.</li>
</ul></li>
</ul></li>
</ul></li>
<li><p>Trend Estimation</p>
<ul>
<li>Wavelets with more vanishing moments can handle smoother and more complex polynomial trends.</li>
</ul>
<!-- -->
<ul>
<li><p>Wavelets with fewer vanishing moments are better at capturing sharp, localized features but may not handle smooth trends as well.</p></li>
<li><p>Example: A wavelet with 4 vanishing moments means it can remove cubic trends from the data.</p></li>
<li><p>Choosing the number of vanishing moments</p>
<ul>
<li>For smooth trends: If the data is expected to have a smooth, slowly varying trend, you should choose a wavelet with a higher number of vanishing moments (e.g., 4 or more). This allows the wavelet to annihilate polynomial trends up to a higher degree and isolate the trend from noise or high-frequency components.</li>
</ul>
<!-- -->
<ul>
<li>For sharp changes or noise: If the data contains sharp changes or you’re more interested in detecting localized features, wavelets with fewer vanishing moments (e.g., 1 or 2) may be more appropriate.</li>
</ul></li>
</ul></li>
<li><p>Wavelet Types</p>
<ul>
<li><p><strong>Phase distortion</strong> occurs when the phase of different frequency components of a signal is altered unevenly, leading to a misalignment in the reconstructed signal (i.e.&nbsp;shifts in position). Important for trend estimation. The more symmetric a wavelet is, the less phase distortion it introduces</p>
<ul>
<li>The <strong>phase</strong> describes the position of the waveform relative to a reference point in time. For example, in a sine wave, the phase tells us where the peaks and troughs of the wave occur.</li>
<li>When a signal is passed through a filter or transformation (such as a wavelet or Fourier transform), each frequency component might experience a shift in its phase and by different amounts which results in the distortion.</li>
</ul></li>
<li><p>Wavelets with strong time localization allow them to detect sharp, abrupt changes</p></li>
<li><p>Guidelines</p>
<ul>
<li>Daubechies EP Wavelets: Choose for <em>sharp, localized features</em> (e.g.&nbsp;spike, discontinuities) or if phase shifts are not a major concern. They are ideal for compression or denoising while preserving features like discontinuities and detecting abrupt changes but may introduce phase distortion and boundary effects.
<ul>
<li>e.g.&nbsp;seismic , financial data</li>
</ul></li>
</ul>
<!-- -->
<ul>
<li><p>Daubechies LA Wavelets: Choose for <em>smooth trends</em> and when minimizing phase distortion is important. They are particularly effective for trend estimation and nonstationary data with long-term, smooth features.</p>
<ul>
<li>e.g.&nbsp;temperature changes, economic growth</li>
</ul></li>
</ul></li>
</ul></li>
</ul>
</section>
<section id="ranking-algo" class="level2">
<h2 class="anchored" data-anchor-id="ranking-algo">Ranking Algo</h2>
<ul>
<li>Notes from <a href="https://arxiv.org/abs/2408.10558">Multi-Attribute Preferences: A Transfer Learning Approach</a></li>
<li>preference data are typically elicited by individuals, whether in the form of pairwise comparisons, partial rankings or click-through data, which are aggregated into a single coherent ranking that best reflects these preferences</li>
<li>Use Cases
<ul>
<li>data consisting of hotel rankings, where consumers rank various attributes of hotels such as breakfast, hygiene, price, quality of service, but also their overall satisfaction with the hotel</li>
<li>different types of food that are ranked on various properties, such as different aspects of taste, smell, visual aspects, but also their overall ranking</li>
</ul></li>
<li>primary attribute - the main attribute of interest
<ul>
<li>Typically the overall preference or satisfaction, but not necessarily</li>
</ul></li>
<li>secondary attributes - The other attributes on which the objects are evaluated</li>
<li>jointly learning tasks
<ul>
<li>multi-task learning - concerns the improvement of multiple related learning tasks by borrowing relevant information among these tasks and therefore coincides with existing methods that aim to model multi-attribute preference data</li>
</ul></li>
<li>learning a single task
<ul>
<li>transfer learning - aims to optimise the efficiency of learning a single task, by utilising relevant information from other task</li>
<li>the single task of interest is called the target, whilst the other tasks are sources, and forms a parallel to the primary and secondary attributes</li>
<li>only the Gaussian graphical model and the Gaussian mixture model have been enriched by the transfer learning framework.</li>
</ul></li>
<li>Paper goals
<ul>
<li>Utilizing Bradley-Terry and its generalization the Plackett-Luce models – in order to improve inference on parameters underlying a primary attribute by utilising information contained in the secondary attributes
<ul>
<li>Models frequently used in pairwise comparison data</li>
<li>method is then incorporated into the transfer learning framework and extended upon, resulting in algorithms that generate estimates for the primary attribute with and without a known set of informative secondary attributes</li>
</ul></li>
<li>typically only a subset of the secondary attributes is useful when estimating the primary attribute parameters, we adapt the framework proposed by Tian and Feng, where we introduce an algorithm that is able to effectively infer the set of informative secondary attributes</li>
<li>Bradley-Terry<br>
<span class="math display">\[
\begin{aligned}
&amp;P(o_j &gt; o_l) = \frac{e^{\alpha_j}}{e^{\alpha_j} + e^{\alpha_l}} \\
&amp;\text{where} \; 1 \le j \ne l \le M
\end{aligned}
\]</span>
<ul>
<li>Each individual <span class="math inline">\(i\)</span> assigns their preference for one object <span class="math inline">\(j\)</span> over another object <span class="math inline">\(l\)</span> from a total pool of <span class="math inline">\(M\)</span> objects.</li>
<li>Assumes that underlying each object there exists some worth <span class="math inline">\(\alpha\)</span> that relates to its probability of being preferred over another objects.</li>
<li>These pairwise comparisons can be presented by an undirected graph <span class="math inline">\(\mathcal{G} =(\mathcal{V}, \mathcal{E})\)</span>, with vertices <span class="math inline">\(\mathcal{V} = \{1, \ldots, M\}\)</span> and edge set <span class="math inline">\(\mathcal{E}\)</span> that has the property that <span class="math inline">\((j, l) \in \mathcal{E}\)</span> if and only if objects <span class="math inline">\(j\)</span> and <span class="math inline">\(l\)</span> are compared at least once in the data. The following conditions are postulated for the pairwise comparison graph.</li>
</ul></li>
<li>Assumptions</li>
<li>Data
<ul>
<li>partial ranking: <span class="math inline">\(\{o_1 \gt \cdots \gt o_m\}\)</span></li>
<li>pairwise comparisons: <span class="math inline">\(\bigcap_{1 \le j \ne l \le M} \: \{o_j \gt o_l\}\)</span></li>
</ul></li>
</ul></li>
</ul>
</section>
<section id="trend-followingmomentum" class="level2">
<h2 class="anchored" data-anchor-id="trend-followingmomentum">Trend Following/Momentum</h2>
<ul>
<li>Notes from <a href="https://arxiv.org/abs/2407.13685">Beyond Trend Following: Deep Learning for Market Trend Prediction</a></li>
<li>Read
<ul>
<li><a href="https://quantpedia.com/designing-robust-trend-following-system/">Designing Robust Trend-Following System</a></li>
<li><a href="https://alphaarchitect.com/2024/12/portfolio-efficiency/">Diversifying Trend Following Strategies Improves Portfolio Efficiency</a>
<ul>
<li>Diversify across CTAs</li>
<li>It’s a tail hedge, so there will be long periods of loss — i.e.&nbsp;not for faint of heart</li>
<li>Improve the efficiency of their portfolio by also adding allocations to the other uncorrelated strategies</li>
</ul></li>
<li><a href="https://quantpedia.com/exploration-of-cta-momentum-strategies-using-etfs/">Exploration of CTA Momentum Strategies Using ETFs</a></li>
</ul></li>
<li>Trend following
<ul>
<li>Trend following or trend trading is an investment strategy based on the expectation of price movements to continue in the same direction: buy an asset when its price goes up, sell it when its price goes down.
<ul>
<li>a particular criterion to detect when prices move in a particular direction over time and every investor uses his own criterion</li>
</ul></li>
<li>Traditional trend following is usually done on futures. Just follow trends on a large, diversified set of futures markets, covering major asset classes.
<ul>
<li>Diversification is key: with multiple assets with low or negative correlations, you can achieve higher returns at a lower risk.</li>
</ul></li>
<li>Trend following on stocks can easily yield negative returns in the short side (when prices go down). When we trade only on the long side, it does not always add any real value.
<ul>
<li>Standard trend following is not expected to work with stocks, since their correlation is too high.</li>
</ul></li>
<li>Compared with a passive index ETF, trend following requires additional work and creates potential risks, yet it does not always yield actual benefits.
<ul>
<li>Trend following on single stocks, or a few of them, however, is not attractive for the risk you have to assume.</li>
</ul></li>
<li>Bear regime strategy (Meb Fabor on <a href="https://www.bloomberg.com/news/articles/2024-10-18/meb-faber-on-why-prudent-investors-keep-getting-punished?srnd=oddlots">Odd Lots</a>)</li>
</ul></li>
<li>momentum investing
<ul>
<li>When a stock price goes up for a while, the likelihood of rising higher is greater than the likelihood of falling. Likewise, a stock going up faster than other stocks is likely to keep going up faster than other stocks.</li>
<li>One explanation is that people who buy past winners and sell past losers temporarily move prices. An alternative explanation is that the market underreacts to information on short-term prospects but overreacts to information on long-term prospects.</li>
<li>Andreas Clenow employs the following trading rules on a weekly basis:
<ul>
<li>A.&nbsp;F. Clenow,&nbsp;Stocks on the Move: Beating the Market with Hedge Fund Momentum Strategies.Equilateral Publishing, 2015.</li>
<li>rank stocks on volatility-adjusted momentum (using an exponential 90-day regression, multiplied by its coefficient of determination),</li>
<li>calculate position sizes (targeting a daily move of 10 basis points),</li>
<li>check the index filter (S&amp;P 500 above its 200-day moving average), and build your portfolio.</li>
<li>Individual stocks are disqualified when they are below their 100-day moving average or have experienced a gap over 15%.</li>
<li>When, in the weekly portfolio rebalancing, a stock is no longer in the top 20% of the S&amp;P 500 ranking or fails to meet the qualification criteria (moving average and gap), it is sold. It is replaced by other stocks only if the index is in a positive trend. Twice per month, position sizes are also rebalanced to control risk.</li>
</ul></li>
</ul></li>
</ul>
</section>
<section id="clusteringhierarchical-ts" class="level2">
<h2 class="anchored" data-anchor-id="clusteringhierarchical-ts">Clustering/Hierarchical TS</h2>
<ul>
<li>Notes from <a href="https://arxiv.org/html/2404.06064v1">Constructing hierarchical time series through clustering: Is there an optimal way for forecasting?</a>
<ul>
<li>Code: <a href="https://github.com/AngelPone/project_hierarchy" class="uri">https://github.com/AngelPone/project_hierarchy</a></li>
</ul></li>
<li>The models used to obtain base forecasts and the reconciliation method are fixed throughout the experiments</li>
<li>coherent, that is they respect the aggregation constraints implied by the hierarchical structure. Coherent forecasts facilitate aligned decisions by agents acting upon different variables within the hierarchy. For example, consider a retail setting, where a warehouse manager supplies stock to individual store managers within their region. Forecasts could be incoherent when the warehouse manager forecasts low total demand while store managers forecast high demand, leading to supply shortages.</li>
<li>Clustered different representations (the original time series, forecast errors, features of both), different distance metrics (Euclidean, dynamic time warping), and different clustering paradigms (k-medioids, hierarchical).
<ul>
<li>For features, they used 56 features from {tsfeatures}</li>
<li>in-sample one-step-ahead forecast error as a representation of the time series, since a key step in MinT reconciliation is to estimate the <span class="math inline">\(\boldsymbol{W}_h\)</span> matrix.
<ul>
<li>It is important to note that raw time series and in-sample error representations are standardized to eliminate the impact of scale variations.</li>
</ul></li>
</ul></li>
<li>rq1
<ul>
<li>natural hierarchy outperforms the two-level hierarchy, and data-driven hierarchy via clustering can further improve forecast performance compared to natural hierarchy.
<ul>
<li>“grouping” is the idea that some correct subsets of series are chosen to form new middle-level series</li>
<li>“structure” of the hierarchy, includes the number of middle-level series, the depth of the hierarchy, and the distribution of group sizes in the middle layer(s).</li>
</ul></li>
<li>optimal clustering method depends on the dataset characteristics</li>
</ul></li>
<li>rq2
<ul>
<li>the driver of forecast improvement is the enlarged number of series in the hierarchy and/or its <strong>structure</strong>, rather than similarities between the time series (i.e.&nbsp;grouping).</li>
</ul></li>
<li>rq3
<ul>
<li>an equally-weighted combination of reconciled forecasts derived from multiple hierarchies improves forecast reconciliation performance</li>
<li>our approach averages not only different coherent forecasts, but also across hierarchies with completely different middle level series. This is possible since only coherent bottom and top level forecasts are averaged and evaluated.</li>
</ul></li>
<li>Section 2 describes the trace minimization reconciliation method (min T from {forecast})</li>
</ul>
</section>
<section id="lab-91" class="level2">
<h2 class="anchored" data-anchor-id="lab-91">lab 91</h2>
<ul>
<li>clvtools for prob type, h2o::automl for ML</li>
<li>agg, cohort, prob, ml, fcast</li>
<li>group by, summarize, pad_by_time, ungroup</li>
<li>lag - use horizon and use 2*horizon</li>
<li>rolling - 2,3,6 (uses lag parameters and 2; lags were 3 and 6 months with horizon = 3)</li>
<li>splits: timetk::time_series_split, cumulative = TRUE says use all previous data(?)</li>
</ul>
</section>
<section id="rhino" class="level2">
<h2 class="anchored" data-anchor-id="rhino">Rhino</h2>
<ul>
<li>rhinoverse.dev</li>
<li>opiniated project structure, development toolbox, guides you towards best practices</li>
<li><code>rhino::init()</code> or RStudio New Project wizard</li>
<li>github discussions for questions</li>
<li>Can use other UI packages and not just those in rhinoverse</li>
<li>Project structure
<ul>
<li>config.yml for different environments (e.g.&nbsp;dev, prod)</li>
<li>main.R with server and ui</li>
<li>view - modules that rely on reactivity</li>
<li>static - imgs</li>
<li>styles - sass files (css stuff)</li>
<li>dependencies - explicit list of packages</li>
<li>cypress - unit tests of functions</li>
</ul></li>
<li><code>options(shiny.qutoreload = TRUE)</code> - once you save, the app changes automatically</li>
<li>addins
<ul>
<li>formatting, lintr</li>
<li>create rhino module</li>
<li>build sass - automatically shows changes in app when changing and saving sass file</li>
<li>build javascript - same as build sass but for react components</li>
</ul></li>
<li>Uses {box} for function imports from packages and has a box linter</li>
<li>dependency management
<ul>
<li><code>pkg_install</code>/<code>remove</code> - install packages from everywhere and not just cran. Updates dependency.R and renv.lock</li>
</ul></li>
<li>Add react components with {shiny.react}</li>
</ul>
</section>
<section id="signature-transform" class="level2">
<h2 class="anchored" data-anchor-id="signature-transform">Signature Transform</h2>
<ul>
<li>Todo
<ul>
<li>Continue reading</li>
<li>Look at the separate papers from with the applied data examples are taken from</li>
<li>Go back to original Amazon paper and see if signature parts and its appendix make more sense.</li>
<li>Look at Discussion section in Signatory github and ask questions</li>
</ul></li>
<li>Misc
<ul>
<li><span class="math inline">\(e\)</span> is a monomial (pg 13)</li>
<li><span class="math inline">\(\lambda\)</span> is a real number (pg 13)</li>
<li><span class="math inline">\(\otimes\)</span> is defined as the the <em>joining</em> (i.e.&nbsp;concantenating) of multi-indexes of monomials: <span class="math inline">\(e_{i_1} \cdots e_{i_k} \otimes e_{j_1} \cdots e_{j_m} = e_{i_1} \cdots e_{i_k} e_{j_1} \cdots e_{j_m}\)</span> (pg 13)
<ul>
<li>Chen’s Identity: <span class="math inline">\(S(X*Y)_{a,c} = S(X)_{a,b} \otimes S(Y)_{b,c}\)</span> where <span class="math inline">\(X*Y\)</span> is the concantenation of two paths (pg 14)
<ul>
<li>So the signature of a concantenated path is equal to the circle-product of the signatures of the component paths.</li>
</ul></li>
<li><span class="math inline">\(\otimes n\)</span> is the n<sup>th</sup> power with respect to the circle product, <span class="math inline">\(\otimes\)</span> (pg15)<br>
</li>
</ul></li>
</ul></li>
<li>Workflow
<ul>
<li>Create a continuous path <span class="math inline">\(X_i\)</span> from each time-series <span class="math inline">\(\{Y_i\}\)</span> (row-wise)</li>
<li>If needed, make use of the lead-lag transform to account for the variability in data
<ul>
<li>Cumalative sum transform is another</li>
</ul></li>
<li>Compute the truncated signature <span class="math inline">\(S(X_i)|_L\)</span> of the path <span class="math inline">\(X_i\)</span> up to level <span class="math inline">\(L\)</span>
<ul>
<li>Either a Full or Log signature</li>
</ul></li>
<li>Standardize each signature column</li>
<li>Use the terms of signature <span class="math inline">\(\{S^I_i\}\)</span> as features</li>
</ul></li>
<li>Issue
<ul>
<li>Degeneracy in the terms of the signature causes this representation not to be unique and introducing a problem of colinearity of the signature terms.
<ul>
<li>Solution: LASSO, ridge or elastic net regularization</li>
<li>Paper uses a 2-step lasso where signature features are selected by LASSO. Then those selected features are used in a second regression with other predictors.</li>
</ul></li>
</ul></li>
<li>Signature
<ul>
<li><span class="math inline">\(S^{(1)}_{a,b} = X_b - X_a\)</span></li>
<li><span class="math inline">\(S^{(1,1)}_{a,b} = \frac{(X_b - X_a)^2}{2!}\)</span></li>
<li><span class="math inline">\(S^{(1,1,1)}_{a,b} = \frac{(X_b - X_a)^3}{3!}\)</span></li>
</ul></li>
<li>Cumulative + Lead Lag Signature Truncated to Level 2
<ul>
<li>Signature
<ul>
<li><span class="math inline">\(S(\tilde X)|_{L=2} = (1, S^{(1)}, S^{(2)}, S^{(1,1)}, S^{(1,2)}, S^{(2,1)}, S^{(2,2)})\)</span></li>
<li><span class="math inline">\(S^{(1)} = S^{(2)} = \sum_i^N X_i\)</span></li>
<li><span class="math inline">\(S^{(1,1)} = S^{(2,2)} = \frac{1}{2} \left(\sum_i^N X_i \right)^2\)</span></li>
<li><span class="math inline">\(S^{(1,2)} = \frac{1}{2} \left[\left(\sum_i^N X_i\right)^2 + \sum_i^N X_i^2 \right]\)</span></li>
<li><span class="math inline">\(S^{(1,2)} = \frac{1}{2} \left[\left(\sum_i^N X_i\right)^2 - \sum_i^N X_i^2 \right]\)</span></li>
</ul></li>
<li>Moments
<ul>
<li>Mean(X): <span class="math inline">\(\frac{1}{N}S^{(1)}\)</span></li>
<li>Var(X): <span class="math inline">\(-\frac{N+1}{N^2}S^{(1,2)} + \frac{N-1}{N^2}S^{(2,1)}\)</span></li>
</ul></li>
</ul></li>
<li>Lead Lag Signature Truncated to Level 2
<ul>
<li><span class="math inline">\(S(\tilde X)|_{L=2} = (1, S^{(1)}, S^{(2)}, S^{(1,1)}, S^{(1,2)}, S^{(2,1)}, S^{(2,2)})\)</span></li>
<li><span class="math inline">\(S^{(1)} = S^{(2)} = \sum_i^{N-1} (X_{i+1} - X_i)\)</span></li>
<li><span class="math inline">\(S^{(1,1)} = S^{(2,2)} = \frac{1}{2} \left(\sum_i^N (X_{i+1} - X_i) \right)^2\)</span></li>
<li><span class="math inline">\(S^{(1,2)} = \frac{1}{2} \left[\left(\sum_i^N (X_{i+1} - X_i)\right)^2 + \sum_i^N (X_{i+1} - X_i) \right]\)</span></li>
<li><span class="math inline">\(S^{(2,1)} = \frac{1}{2} \left[\left(\sum_i^N (X_{i+1} - X_i)\right)^2 - \sum_i^N (X_{i+1} - X_i) \right]\)</span></li>
</ul></li>
<li>Log Signature Truncated to Level 2<br>
<span class="math display">\[
\begin{aligned}
&amp;\log S(X) = (\Delta X, \Delta X, \frac{1}{2}\text{QV}(X))\\
&amp;\begin{aligned}
\text{where} \quad &amp;\Delta X = X_N - X_1 \\
&amp;\text{QV}(X)) = \sum_{i=1}^{N-1} (X_{i+1} - X_i)^2
\end{aligned}
\end{aligned}
\]</span>
<ul>
<li><p><span class="ribbon-highlight">Example</span>: eq 2.17, Calculation for Quadratic Variation (QV)</p>
<div class="sourceCode" id="cb9"><pre class="sourceCode r code-with-copy"><code class="sourceCode r"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a>x <span class="ot">&lt;-</span> <span class="fu">c</span>(<span class="dv">1</span>, <span class="dv">4</span>, <span class="dv">2</span>, <span class="dv">6</span>)</span>
<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a>x_lead <span class="ot">&lt;-</span> dplyr<span class="sc">::</span><span class="fu">lead</span>(x1)</span>
<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb9-4"><a href="#cb9-4" aria-hidden="true" tabindex="-1"></a>QV <span class="ot">&lt;-</span> <span class="cf">function</span>(x1, x2){</span>
<span id="cb9-5"><a href="#cb9-5" aria-hidden="true" tabindex="-1"></a>  x1_a <span class="ot">&lt;-</span> x1[<span class="sc">-</span><span class="fu">length</span>(x1)]</span>
<span id="cb9-6"><a href="#cb9-6" aria-hidden="true" tabindex="-1"></a>  x2_a <span class="ot">&lt;-</span> x2[<span class="sc">-</span><span class="fu">length</span>(x2)]</span>
<span id="cb9-7"><a href="#cb9-7" aria-hidden="true" tabindex="-1"></a>  sq_diff <span class="ot">&lt;-</span> <span class="cf">function</span>(j1, j2) {</span>
<span id="cb9-8"><a href="#cb9-8" aria-hidden="true" tabindex="-1"></a>    (j2 <span class="sc">-</span> j1)<span class="sc">^</span><span class="dv">2</span></span>
<span id="cb9-9"><a href="#cb9-9" aria-hidden="true" tabindex="-1"></a>  }</span>
<span id="cb9-10"><a href="#cb9-10" aria-hidden="true" tabindex="-1"></a>  purrr<span class="sc">::</span><span class="fu">map2_dbl</span>(x1_a, x2_a, sq_diff) <span class="sc">|&gt;</span> <span class="fu">sum</span>()</span>
<span id="cb9-11"><a href="#cb9-11" aria-hidden="true" tabindex="-1"></a>}</span>
<span id="cb9-12"><a href="#cb9-12" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb9-13"><a href="#cb9-13" aria-hidden="true" tabindex="-1"></a><span class="fu">QV</span>(x, x_lead)</span>
<span id="cb9-14"><a href="#cb9-14" aria-hidden="true" tabindex="-1"></a><span class="co">#&gt; [1] 29</span></span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div></li>
</ul></li>
<li>Questions
<ul>
<li><span class="math inline">\(\otimes n\)</span> makes no sense (pg 15) — just literally</li>
<li>Log transform makes no sense (pg 15, 16) in terms of applying it to data</li>
<li>Formula never provided lag-lead dimensions (pg 20)
<ul>
<li>What’s a lead-lag embedding and how are these things connected.</li>
</ul></li>
<li>How can it be -14.5 in 2.17 with QV is always positive? (pg 26)</li>
<li>eq 2.18, Standard Signature, <span class="math inline">\(S(X^2) = (1, 5, 5, 12.5, −2, 27, 12.5)\)</span> — How is this calculated? (pg 26)
<ul>
<li>S<sup>(1,2)</sup> and S<sup>(2,1)</sup> don’t match lead-lag algorithm. NO WHERE in the paper is the formula for these terms explicitly given. Think it might have to do with shuffle product maybe.</li>
</ul></li>
<li>What is the last column in the feature matrix (pg 31)? Paragraph makes it sound like it’s the mean.</li>
<li>In eq 2.34, what the fuck is going on in the 3rd dimension? It’s supposed to be an indicator variable (i.e.&nbsp;0 or 1)</li>
<li>Are there guidelines on when to use cum/lead-lag transforms and full/log signatures and Level 1,2, or 3?
<ul>
<li>In order to capture the quadratic variation of the price, the path is extended by means of a lead-lag transform ()</li>
</ul></li>
<li>Annoying phrases
<ul>
<li>One can easily rewrite (pg 26) - then just spits out an answer with no previous example of how it was obtained.</li>
</ul></li>
</ul></li>
<li>Full Signature
<ul>
<li>The terms of the signature are iterated integrals of a path, while the path is normally constructed by an interpolation of data points. One can compute such iterated integrals using several computational algorithms (cubature methods) which are generally straightforward to implement. (sect 2.3, pg 34)</li>
<li>Signature approach is to convert data into paths and then compute the iterated integrals of the resulting paths (sect 2.4.1, pg 35)</li>
</ul></li>
</ul>
</section>
<section id="reproducibility" class="level2">
<h2 class="anchored" data-anchor-id="reproducibility">Reproducibility</h2>
<ul>
<li>Notes from <a href="https://www.brodrigues.co/blog/2023-07-13-nix_for_r_part1/" class="uri">https://www.brodrigues.co/blog/2023-07-13-nix_for_r_part1/</a></li>
<li>To ensure that a project is reproducible you need to deal with at least four things:
<ul>
<li>Make sure that the required/correct version of R (or any other language) is installed</li>
<li>Make sure that the required versions of packages are installed</li>
<li>Make sure that system dependencies are installed (for example, you’d need a working Java installation to install the {rJava} R package on Linux)</li>
<li>Make sure that you can install all of this for the hardware you have on hand.</li>
</ul></li>
<li>Consensus seems to be a mixture of Docker to deal with system dependencies,<code>{renv}</code>for the packages (or<code>{groundhog}</code>, or a fixed CRAN snapshot like those <a href="https://packagemanager.posit.co/__docs__/user/get-repo-url/#ui-frozen-urls">Posit provides</a>) and the <a href="https://github.com/r-lib/rig">R installation manager</a> to install the correct version of R (unless you use a Docker image as base that already ships the required version by default). As for the last point, the only way out is to be able to compile the software for the target architecture.</li>
<li>Nix
<ul>
<li><p>a package manager for Linux distributions, macOS and apparently it even works on Windows if you enable WSL2.</p></li>
<li><p>huge package repository, over 80K packages</p></li>
<li><p>possible to install software in (relatively) isolated environments</p></li>
</ul></li>
</ul>
</section>
<section id="text-tiling" class="level2">
<h2 class="anchored" data-anchor-id="text-tiling">Text Tiling</h2>
<ul>
<li><p>Previous articles</p>
<ul>
<li>5 sentence chunks - Instead of creating chunks large enough to fit into a context window (langchain default), I propose that the chunk size should be the number of sentences it generally takes to express a discrete idea. This is because we will later embed this chunk of text, essentially distilling its semantic meaning into a vector. I currently use 5 sentences (but you can experiment with other numbers). I tend to have a 1-sentence overlap between chunks, just to ensure continuity so that each chunk has some contextual information about the previous chunk. (<a href="https://towardsdatascience.com/summarize-podcast-transcripts-and-long-texts-better-with-nlp-and-ai-e04c89d3b2cb">2-stage summarizing method</a>)</li>
<li>Chunk markdown documents by section using header tags (h1, h2, etc.) (<a href="https://towardsdatascience.com/how-i-turned-my-companys-docs-into-a-searchable-database-with-openai-4f2d34bd8736#8ed1">company doc searchable db</a>)</li>
<li>Chunk size = Context window size (langchain default)</li>
</ul></li>
<li><p><a href="https://www.nltk.org/_modules/nltk/tokenize/texttiling.html">nltk.tokenize.texttiling</a> - text tiling method from {{nltk}}</p></li>
<li><p>Created a test document that was the amalgam 4 different articles that can be used to test tiling method undefined</p></li>
</ul>
</section>
<section id="propensity-score-models" class="level2">
<h2 class="anchored" data-anchor-id="propensity-score-models">Propensity Score Models?</h2>
<ul>
<li>Articles
<ul>
<li>Stephen Senn paper on why propensity scores are redundant, <a href="https://onlinelibrary.wiley.com/doi/abs/10.1002/sim.3133" class="uri">https://onlinelibrary.wiley.com/doi/abs/10.1002/sim.3133</a>. I think this might only be for RCTs though.
<ul>
<li>From <a href="https://x.com/stephensenn/status/1788917201476423756">thread</a></li>
<li>Also, <a href="https://x.com/AleksanderMolak/status/1788479511958352322">thread</a> and a <a href="https://www.youtube.com/watch?v=nT_yCwXSz54&amp;ab_channel=CausalPythonwithAlexMolak">video</a> of 1hr podcast with Senn describing it</li>
</ul></li>
<li><a href="https://bsky.app/profile/noahgreifer.bsky.social/post/3lgiz64ikak2t">Thread</a> on covariate balancing propensity score (CBPS)</li>
</ul></li>
<li>bayesian vid (currently at 23:14)
<ul>
<li>I don’t remember which video this is</li>
</ul></li>
<li>in observational analysis its imagined that sample is drawn from a joint distribution of all the variables</li>
<li>in causal inf, imagine intervening to change Z, treatment, independent of X, confounders.</li>
<li>Frequentist method: G-Estimation
<ul>
<li><p>Models</p>
<ul>
<li>propensity score models, <span class="math inline">\(b(x;\gamma)\)</span>
<ul>
<li><p>Equivalent to <span class="math inline">\(\text{Pr} [Z = 1 |x]\)</span></p></li>
<li><p>Estimating Equation for <span class="math inline">\(\gamma\)</span></p>
<p><span class="math display">\[
\sum \limits_{i=1}^n x_i^T (z_i - b(x_i; \gamma)) = 0
\]</span></p></li>
</ul></li>
<li>treatment free mean model, <span class="math inline">\(\mu_0(x; \beta)\)</span>
<ul>
<li>Used for doubly robust estimation</li>
</ul></li>
<li>treatment effect (or blip) model <span class="math inline">\(\tau z\)</span>, which can be extended to <span class="math inline">\(z\mu_1(x;\tau)\)</span></li>
</ul></li>
<li><p>Propensity Score Regression<br>
</p>
<p><span class="math display">\[
Y = Z \tau + b(X; \hat{\gamma}) \phi + \epsilon
\]</span></p>
<ul>
<li><span class="math inline">\(\phi\)</span> is the estimated coefficient for the propensity score model</li>
</ul></li>
</ul></li>
<li>Bayesian
<ul>
<li>There are other ways but this procedure is recommended</li>
<li>Perform full Bayesian estimation of <span class="math inline">\(\gamma\)</span> , plug that (best) estimate into the propensity score model, <span class="math inline">\(b(x_i; \hat{\gamma})\)</span> , and then perform Bayesian analysis of <span class="math inline">\(\tau\)</span> (i.e.&nbsp;propensity score regression)
<ul>
<li>The propensity score model part of the formula is basically a hack and not mimmicking any part of the dgp therefore for bayesians, the regression model is a misspecification.</li>
</ul></li>
</ul></li>
</ul>
</section>
<section id="copulas" class="level2">
<h2 class="anchored" data-anchor-id="copulas">Copulas</h2>
<ul>
<li><p>Todo</p>
<ul>
<li><a href="https://twiecki.io/blog/2018/05/03/copulas/" class="uri">https://twiecki.io/blog/2018/05/03/copulas/</a> - explainer, good starting point</li>
<li><a href="https://copulae.readthedocs.io/en/latest/explainers/introduction.html" class="uri">https://copulae.readthedocs.io/en/latest/explainers/introduction.html</a> - another beginner explainer</li>
<li><a href="https://www.r-bloggers.com/2015/10/modelling-dependence-with-copulas-in-r/" class="uri">https://www.r-bloggers.com/2015/10/modelling-dependence-with-copulas-in-r/</a> - practical example using returns of two stocks</li>
<li><a href="https://arxiv.org/abs/2403.15862">Paper</a>- Non-monotone dependence modeling with copulas: an application to the volume-return relationship - no code but discusses how r pkg is used</li>
</ul></li>
<li><p>Packages</p>
<ul>
<li>{<a href="https://cran.r-project.org/web/packages/copula/index.html">copula</a>} - Multivariate Dependence with Copulas — lots of vignettes</li>
<li>{{<a href="https://github.com/DanielBok/copulae">copulae</a>}} - Multivariate data modelling with Copulas in Python</li>
</ul></li>
<li><p>I think copulas are used for bias correction in post-processing separate forecasts of variables that are related. See <a href="https://www.annualreviews.org/doi/10.1146/annurev-statistics-062713-085831#_i59">paper</a> (section 4.2 and 4.3)</p>
<ul>
<li>stocks of the same sector</li>
<li>ensemble forecasts
<ul>
<li>weather - meteorologists will forecast a variable (e.g.&nbsp;temp) many times but each time the model uses a different set of atmosphereic conditions. These forecasts are put into a regression (i.e.&nbsp;the ensemble) to create the final forecast. But that forecast is biased because the forecasts are related to each other. Post-processing with copula corrects this.</li>
</ul></li>
</ul></li>
<li><p>Ensemble Copula Coupling (ECC) applies the empirical copula of the original ensemble to samples from the postprocessed predictive distributions. (<a href="https://arxiv.org/pdf/1302.7149.pdf">paper</a>)</p>
<ol type="1">
<li>Generate a raw ensemble, consisting of multiple runs of the computer model that differ in the inputs or model parameters in suitable ways.</li>
<li>Apply statistical postprocessing techniques, such as Bayesian model averaging or nonhomogeneous regression, to correct for systematic errors in the raw ensemble, to obtain calibrated and sharp predictive distributions for each univariate output variable individually.</li>
<li>Draw a sample from each postprocessed predictive distribution.</li>
<li>Rearrange the sampled values in the rank order structure of the raw ensemble to obtain the ECC postprocessed ensemble</li>
</ol></li>
<li><p>Depending on the use of Quantiles, Random draws or Transformations at the sampling stage, we distinguish the ECC-Q, ECC-R and ECC-T variants</p></li>
<li><p>ECC is based on empirical copulas aimed at restoring the dependence structure of the forecast and is derived from the rank order of the members in the raw ensemble forecast, under a perfect model assumption, with exchangeable ensemble members. For Schaake shuffle (SSH), on the other hand, the dependence structure is derived from historical observations instead. (Overview of subject - <a href="https://journals.ametsoc.org/view/journals/bams/102/3/BAMS-D-19-0308.1.xml">paper</a>)</p></li>
<li><p>Packages</p>
<ul>
<li><a href="https://cran.r-project.org/web/packages/ensembleBMA/index.html" class="uri">https://cran.r-project.org/web/packages/ensembleBMA/index.html</a></li>
<li><a href="https://cran.r-project.org/web/packages/ensemblepp/index.html" class="uri">https://cran.r-project.org/web/packages/ensemblepp/index.html</a>
<ul>
<li>Data for book, Statistical Postprocessing of Ensemble Forecasts</li>
</ul></li>
<li><a href="https://cran.r-project.org/web/packages/ensembleMOS/index.html" class="uri">https://cran.r-project.org/web/packages/ensembleMOS/index.html</a></li>
</ul></li>
<li><p>Meteorology bias-corrected forecast (<a href="https://chatgpt.com/share/4a17c9f3-e3ec-4ebc-8b9e-fcc38a583dfc">chatgpt 4o</a>)</p>
<div class="sourceCode" id="cb10"><pre class="sourceCode r code-with-copy"><code class="sourceCode r"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Load necessary packages</span></span>
<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a><span class="fu">install.packages</span>(<span class="st">"copula"</span>)</span>
<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a><span class="fu">install.packages</span>(<span class="st">"fitdistrplus"</span>)</span>
<span id="cb10-4"><a href="#cb10-4" aria-hidden="true" tabindex="-1"></a><span class="fu">install.packages</span>(<span class="st">"PerformanceAnalytics"</span>)</span>
<span id="cb10-5"><a href="#cb10-5" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-6"><a href="#cb10-6" aria-hidden="true" tabindex="-1"></a><span class="fu">library</span>(copula)</span>
<span id="cb10-7"><a href="#cb10-7" aria-hidden="true" tabindex="-1"></a><span class="fu">library</span>(fitdistrplus)</span>
<span id="cb10-8"><a href="#cb10-8" aria-hidden="true" tabindex="-1"></a><span class="fu">library</span>(PerformanceAnalytics)</span>
<span id="cb10-9"><a href="#cb10-9" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-10"><a href="#cb10-10" aria-hidden="true" tabindex="-1"></a><span class="co"># Simulated data for demonstration</span></span>
<span id="cb10-11"><a href="#cb10-11" aria-hidden="true" tabindex="-1"></a><span class="fu">set.seed</span>(<span class="dv">123</span>)</span>
<span id="cb10-12"><a href="#cb10-12" aria-hidden="true" tabindex="-1"></a>n <span class="ot">&lt;-</span> <span class="dv">1000</span></span>
<span id="cb10-13"><a href="#cb10-13" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-14"><a href="#cb10-14" aria-hidden="true" tabindex="-1"></a><span class="co"># Simulating ensemble forecasts from three different models</span></span>
<span id="cb10-15"><a href="#cb10-15" aria-hidden="true" tabindex="-1"></a>forecast1 <span class="ot">&lt;-</span> <span class="fu">rnorm</span>(n, <span class="at">mean =</span> <span class="dv">20</span>, <span class="at">sd =</span> <span class="dv">5</span>)</span>
<span id="cb10-16"><a href="#cb10-16" aria-hidden="true" tabindex="-1"></a>forecast2 <span class="ot">&lt;-</span> <span class="fu">rnorm</span>(n, <span class="at">mean =</span> <span class="dv">21</span>, <span class="at">sd =</span> <span class="dv">5</span>)</span>
<span id="cb10-17"><a href="#cb10-17" aria-hidden="true" tabindex="-1"></a>forecast3 <span class="ot">&lt;-</span> <span class="fu">rnorm</span>(n, <span class="at">mean =</span> <span class="dv">19</span>, <span class="at">sd =</span> <span class="dv">5</span>)</span>
<span id="cb10-18"><a href="#cb10-18" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-19"><a href="#cb10-19" aria-hidden="true" tabindex="-1"></a><span class="co"># Simulating observed temperatures</span></span>
<span id="cb10-20"><a href="#cb10-20" aria-hidden="true" tabindex="-1"></a>observed <span class="ot">&lt;-</span> <span class="fl">0.5</span> <span class="sc">*</span> forecast1 <span class="sc">+</span> <span class="fl">0.3</span> <span class="sc">*</span> forecast2 <span class="sc">+</span> <span class="fl">0.2</span> <span class="sc">*</span> forecast3 <span class="sc">+</span> <span class="fu">rnorm</span>(n, <span class="at">mean =</span> <span class="dv">0</span>, <span class="at">sd =</span> <span class="dv">2</span>)</span>
<span id="cb10-21"><a href="#cb10-21" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-22"><a href="#cb10-22" aria-hidden="true" tabindex="-1"></a><span class="co"># Combine forecasts and observations into a data frame</span></span>
<span id="cb10-23"><a href="#cb10-23" aria-hidden="true" tabindex="-1"></a>data <span class="ot">&lt;-</span> <span class="fu">data.frame</span>(forecast1, forecast2, forecast3, observed)</span>
<span id="cb10-24"><a href="#cb10-24" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-25"><a href="#cb10-25" aria-hidden="true" tabindex="-1"></a><span class="co"># Fit normal distributions to each forecast and the observed temperature</span></span>
<span id="cb10-26"><a href="#cb10-26" aria-hidden="true" tabindex="-1"></a>margins <span class="ot">&lt;-</span> <span class="fu">list</span>(</span>
<span id="cb10-27"><a href="#cb10-27" aria-hidden="true" tabindex="-1"></a>  <span class="at">forecast1 =</span> <span class="fu">fitdist</span>(data<span class="sc">$</span>forecast1, <span class="st">"norm"</span>), </span>
<span id="cb10-28"><a href="#cb10-28" aria-hidden="true" tabindex="-1"></a>  <span class="at">forecast2 =</span> <span class="fu">fitdist</span>(data<span class="sc">$</span>forecast2, <span class="st">"norm"</span>), </span>
<span id="cb10-29"><a href="#cb10-29" aria-hidden="true" tabindex="-1"></a>  <span class="at">forecast3 =</span> <span class="fu">fitdist</span>(data<span class="sc">$</span>forecast3, <span class="st">"norm"</span>),</span>
<span id="cb10-30"><a href="#cb10-30" aria-hidden="true" tabindex="-1"></a>  <span class="at">observed =</span> <span class="fu">fitdist</span>(data<span class="sc">$</span>observed, <span class="st">"norm"</span>)</span>
<span id="cb10-31"><a href="#cb10-31" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb10-32"><a href="#cb10-32" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-33"><a href="#cb10-33" aria-hidden="true" tabindex="-1"></a><span class="co"># Transform data to uniform margins</span></span>
<span id="cb10-34"><a href="#cb10-34" aria-hidden="true" tabindex="-1"></a>u1 <span class="ot">&lt;-</span> <span class="fu">pnorm</span>(data<span class="sc">$</span>forecast1, <span class="at">mean =</span> margins<span class="sc">$</span>forecast1<span class="sc">$</span>estimate[<span class="st">"mean"</span>], <span class="at">sd =</span> margins<span class="sc">$</span>forecast1<span class="sc">$</span>estimate[<span class="st">"sd"</span>])</span>
<span id="cb10-35"><a href="#cb10-35" aria-hidden="true" tabindex="-1"></a>u2 <span class="ot">&lt;-</span> <span class="fu">pnorm</span>(data<span class="sc">$</span>forecast2, <span class="at">mean =</span> margins<span class="sc">$</span>forecast2<span class="sc">$</span>estimate[<span class="st">"mean"</span>], <span class="at">sd =</span> margins<span class="sc">$</span>forecast2<span class="sc">$</span>estimate[<span class="st">"sd"</span>])</span>
<span id="cb10-36"><a href="#cb10-36" aria-hidden="true" tabindex="-1"></a>u3 <span class="ot">&lt;-</span> <span class="fu">pnorm</span>(data<span class="sc">$</span>forecast3, <span class="at">mean =</span> margins<span class="sc">$</span>forecast3<span class="sc">$</span>estimate[<span class="st">"mean"</span>], <span class="at">sd =</span> margins<span class="sc">$</span>forecast3<span class="sc">$</span>estimate[<span class="st">"sd"</span>])</span>
<span id="cb10-37"><a href="#cb10-37" aria-hidden="true" tabindex="-1"></a>u_obs <span class="ot">&lt;-</span> <span class="fu">pnorm</span>(data<span class="sc">$</span>observed, <span class="at">mean =</span> margins<span class="sc">$</span>observed<span class="sc">$</span>estimate[<span class="st">"mean"</span>], <span class="at">sd =</span> margins<span class="sc">$</span>observed<span class="sc">$</span>estimate[<span class="st">"sd"</span>])</span>
<span id="cb10-38"><a href="#cb10-38" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-39"><a href="#cb10-39" aria-hidden="true" tabindex="-1"></a><span class="co"># Combine uniform margins into a matrix</span></span>
<span id="cb10-40"><a href="#cb10-40" aria-hidden="true" tabindex="-1"></a>u_matrix <span class="ot">&lt;-</span> <span class="fu">cbind</span>(u1, u2, u3, u_obs)</span>
<span id="cb10-41"><a href="#cb10-41" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-42"><a href="#cb10-42" aria-hidden="true" tabindex="-1"></a><span class="co"># Fit a normal copula to the pseudo-observations</span></span>
<span id="cb10-43"><a href="#cb10-43" aria-hidden="true" tabindex="-1"></a>normal.cop <span class="ot">&lt;-</span> <span class="fu">normalCopula</span>(<span class="at">dim =</span> <span class="dv">4</span>)</span>
<span id="cb10-44"><a href="#cb10-44" aria-hidden="true" tabindex="-1"></a>fit.cop <span class="ot">&lt;-</span> <span class="fu">fitCopula</span>(normal.cop, u_matrix, <span class="at">method =</span> <span class="st">"ml"</span>)</span>
<span id="cb10-45"><a href="#cb10-45" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-46"><a href="#cb10-46" aria-hidden="true" tabindex="-1"></a><span class="co"># Generate new samples from the fitted copula</span></span>
<span id="cb10-47"><a href="#cb10-47" aria-hidden="true" tabindex="-1"></a>copula.samples <span class="ot">&lt;-</span> <span class="fu">rCopula</span>(n, fit.cop<span class="sc">@</span>copula)</span>
<span id="cb10-48"><a href="#cb10-48" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-49"><a href="#cb10-49" aria-hidden="true" tabindex="-1"></a><span class="co"># Transform copula samples back to original scale</span></span>
<span id="cb10-50"><a href="#cb10-50" aria-hidden="true" tabindex="-1"></a>bias_corrected_data <span class="ot">&lt;-</span> <span class="fu">data.frame</span>(</span>
<span id="cb10-51"><a href="#cb10-51" aria-hidden="true" tabindex="-1"></a>  <span class="at">forecast1 =</span> <span class="fu">qnorm</span>(copula.samples[, <span class="dv">1</span>], <span class="at">mean =</span> margins<span class="sc">$</span>forecast1<span class="sc">$</span>estimate[<span class="st">"mean"</span>], <span class="at">sd =</span> margins<span class="sc">$</span>forecast1<span class="sc">$</span>estimate[<span class="st">"sd"</span>]),</span>
<span id="cb10-52"><a href="#cb10-52" aria-hidden="true" tabindex="-1"></a>  <span class="at">forecast2 =</span> <span class="fu">qnorm</span>(copula.samples[, <span class="dv">2</span>], <span class="at">mean =</span> margins<span class="sc">$</span>forecast2<span class="sc">$</span>estimate[<span class="st">"mean"</span>], <span class="at">sd =</span> margins<span class="sc">$</span>forecast2<span class="sc">$</span>estimate[<span class="st">"sd"</span>]),</span>
<span id="cb10-53"><a href="#cb10-53" aria-hidden="true" tabindex="-1"></a>  <span class="at">forecast3 =</span> <span class="fu">qnorm</span>(copula.samples[, <span class="dv">3</span>], <span class="at">mean =</span> margins<span class="sc">$</span>forecast3<span class="sc">$</span>estimate[<span class="st">"mean"</span>], <span class="at">sd =</span> margins<span class="sc">$</span>forecast3<span class="sc">$</span>estimate[<span class="st">"sd"</span>]),</span>
<span id="cb10-54"><a href="#cb10-54" aria-hidden="true" tabindex="-1"></a>  <span class="at">observed =</span> <span class="fu">qnorm</span>(copula.samples[, <span class="dv">4</span>], <span class="at">mean =</span> margins<span class="sc">$</span>observed<span class="sc">$</span>estimate[<span class="st">"mean"</span>], <span class="at">sd =</span> margins<span class="sc">$</span>observed<span class="sc">$</span>estimate[<span class="st">"sd"</span>])</span>
<span id="cb10-55"><a href="#cb10-55" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb10-56"><a href="#cb10-56" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-57"><a href="#cb10-57" aria-hidden="true" tabindex="-1"></a><span class="co"># Compute the mean and standard deviation of the original and bias-corrected forecasts</span></span>
<span id="cb10-58"><a href="#cb10-58" aria-hidden="true" tabindex="-1"></a>original_stats <span class="ot">&lt;-</span> <span class="fu">data.frame</span>(</span>
<span id="cb10-59"><a href="#cb10-59" aria-hidden="true" tabindex="-1"></a>  <span class="at">Mean =</span> <span class="fu">colMeans</span>(data[, <span class="dv">1</span><span class="sc">:</span><span class="dv">3</span>]),</span>
<span id="cb10-60"><a href="#cb10-60" aria-hidden="true" tabindex="-1"></a>  <span class="at">SD =</span> <span class="fu">apply</span>(data[, <span class="dv">1</span><span class="sc">:</span><span class="dv">3</span>], <span class="dv">2</span>, sd)</span>
<span id="cb10-61"><a href="#cb10-61" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb10-62"><a href="#cb10-62" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-63"><a href="#cb10-63" aria-hidden="true" tabindex="-1"></a>bias_corrected_stats <span class="ot">&lt;-</span> <span class="fu">data.frame</span>(</span>
<span id="cb10-64"><a href="#cb10-64" aria-hidden="true" tabindex="-1"></a>  <span class="at">Mean =</span> <span class="fu">colMeans</span>(bias_corrected_data[, <span class="dv">1</span><span class="sc">:</span><span class="dv">3</span>]),</span>
<span id="cb10-65"><a href="#cb10-65" aria-hidden="true" tabindex="-1"></a>  <span class="at">SD =</span> <span class="fu">apply</span>(bias_corrected_data[, <span class="dv">1</span><span class="sc">:</span><span class="dv">3</span>], <span class="dv">2</span>, sd)</span>
<span id="cb10-66"><a href="#cb10-66" aria-hidden="true" tabindex="-1"></a>)</span>
<span id="cb10-67"><a href="#cb10-67" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-68"><a href="#cb10-68" aria-hidden="true" tabindex="-1"></a><span class="co"># Print original and bias-corrected statistics</span></span>
<span id="cb10-69"><a href="#cb10-69" aria-hidden="true" tabindex="-1"></a><span class="fu">cat</span>(<span class="st">"Original Forecast Statistics:</span><span class="sc">\n</span><span class="st">"</span>)</span>
<span id="cb10-70"><a href="#cb10-70" aria-hidden="true" tabindex="-1"></a><span class="fu">print</span>(original_stats)</span>
<span id="cb10-71"><a href="#cb10-71" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-72"><a href="#cb10-72" aria-hidden="true" tabindex="-1"></a><span class="fu">cat</span>(<span class="st">"Bias-Corrected Forecast Statistics:</span><span class="sc">\n</span><span class="st">"</span>)</span>
<span id="cb10-73"><a href="#cb10-73" aria-hidden="true" tabindex="-1"></a><span class="fu">print</span>(bias_corrected_stats)</span>
<span id="cb10-74"><a href="#cb10-74" aria-hidden="true" tabindex="-1"></a></span>
<span id="cb10-75"><a href="#cb10-75" aria-hidden="true" tabindex="-1"></a><span class="co"># Plot the original and bias-corrected forecasts</span></span>
<span id="cb10-76"><a href="#cb10-76" aria-hidden="true" tabindex="-1"></a><span class="fu">par</span>(<span class="at">mfrow =</span> <span class="fu">c</span>(<span class="dv">2</span>, <span class="dv">3</span>))</span>
<span id="cb10-77"><a href="#cb10-77" aria-hidden="true" tabindex="-1"></a><span class="cf">for</span> (i <span class="cf">in</span> <span class="dv">1</span><span class="sc">:</span><span class="dv">3</span>) {</span>
<span id="cb10-78"><a href="#cb10-78" aria-hidden="true" tabindex="-1"></a>  <span class="fu">plot</span>(data[, i], data<span class="sc">$</span>observed, <span class="at">main =</span> <span class="fu">paste</span>(<span class="st">"Original Forecast"</span>, i), <span class="at">xlab =</span> <span class="st">"Forecast"</span>, <span class="at">ylab =</span> <span class="st">"Observed"</span>)</span>
<span id="cb10-79"><a href="#cb10-79" aria-hidden="true" tabindex="-1"></a>  <span class="fu">plot</span>(bias_corrected_data[, i], bias_corrected_data<span class="sc">$</span>observed, <span class="at">main =</span> <span class="fu">paste</span>(<span class="st">"Bias-Corrected Forecast"</span>, i), <span class="at">xlab =</span> <span class="st">"Forecast"</span>, <span class="at">ylab =</span> <span class="st">"Observed"</span>)</span>
<span id="cb10-80"><a href="#cb10-80" aria-hidden="true" tabindex="-1"></a>}</span></code><button title="Copy to Clipboard" class="code-copy-button"><i class="bi"></i></button></pre></div></li>
<li></li>
</ul>
</section>

</main>
<!-- /main column -->
<script id="quarto-html-after-body" type="application/javascript">
window.document.addEventListener("DOMContentLoaded", function (event) {
  const toggleBodyColorMode = (bsSheetEl) => {
    const mode = bsSheetEl.getAttribute("data-mode");
    const bodyEl = window.document.querySelector("body");
    if (mode === "dark") {
      bodyEl.classList.add("quarto-dark");
      bodyEl.classList.remove("quarto-light");
    } else {
      bodyEl.classList.add("quarto-light");
      bodyEl.classList.remove("quarto-dark");
    }
  }
  const toggleBodyColorPrimary = () => {
    const bsSheetEl = window.document.querySelector("link#quarto-bootstrap");
    if (bsSheetEl) {
      toggleBodyColorMode(bsSheetEl);
    }
  }
  toggleBodyColorPrimary();  
  const icon = "";
  const anchorJS = new window.AnchorJS();
  anchorJS.options = {
    placement: 'right',
    icon: icon
  };
  anchorJS.add('.anchored');
  const isCodeAnnotation = (el) => {
    for (const clz of el.classList) {
      if (clz.startsWith('code-annotation-')) {                     
        return true;
      }
    }
    return false;
  }
  const onCopySuccess = function(e) {
    // button target
    const button = e.trigger;
    // don't keep focus
    button.blur();
    // flash "checked"
    button.classList.add('code-copy-button-checked');
    var currentTitle = button.getAttribute("title");
    button.setAttribute("title", "Copied!");
    let tooltip;
    if (window.bootstrap) {
      button.setAttribute("data-bs-toggle", "tooltip");
      button.setAttribute("data-bs-placement", "left");
      button.setAttribute("data-bs-title", "Copied!");
      tooltip = new bootstrap.Tooltip(button, 
        { trigger: "manual", 
          customClass: "code-copy-button-tooltip",
          offset: [0, -8]});
      tooltip.show();    
    }
    setTimeout(function() {
      if (tooltip) {
        tooltip.hide();
        button.removeAttribute("data-bs-title");
        button.removeAttribute("data-bs-toggle");
        button.removeAttribute("data-bs-placement");
      }
      button.setAttribute("title", currentTitle);
      button.classList.remove('code-copy-button-checked');
    }, 1000);
    // clear code selection
    e.clearSelection();
  }
  const getTextToCopy = function(trigger) {
      const codeEl = trigger.previousElementSibling.cloneNode(true);
      for (const childEl of codeEl.children) {
        if (isCodeAnnotation(childEl)) {
          childEl.remove();
        }
      }
      return codeEl.innerText;
  }
  const clipboard = new window.ClipboardJS('.code-copy-button:not([data-in-quarto-modal])', {
    text: getTextToCopy
  });
  clipboard.on('success', onCopySuccess);
  if (window.document.getElementById('quarto-embedded-source-code-modal')) {
    const clipboardModal = new window.ClipboardJS('.code-copy-button[data-in-quarto-modal]', {
      text: getTextToCopy,
      container: window.document.getElementById('quarto-embedded-source-code-modal')
    });
    clipboardModal.on('success', onCopySuccess);
  }
    var localhostRegex = new RegExp(/^(?:http|https):\/\/localhost\:?[0-9]*\//);
    var mailtoRegex = new RegExp(/^mailto:/);
      var filterRegex = new RegExp('/' + window.location.host + '/');
    var isInternal = (href) => {
        return filterRegex.test(href) || localhostRegex.test(href) || mailtoRegex.test(href);
    }
    // Inspect non-navigation links and adorn them if external
 	var links = window.document.querySelectorAll('a[href]:not(.nav-link):not(.navbar-brand):not(.toc-action):not(.sidebar-link):not(.sidebar-item-toggle):not(.pagination-link):not(.no-external):not([aria-hidden]):not(.dropdown-item):not(.quarto-navigation-tool):not(.about-link)');
    for (var i=0; i<links.length; i++) {
      const link = links[i];
      if (!isInternal(link.href)) {
        // undo the damage that might have been done by quarto-nav.js in the case of
        // links that we want to consider external
        if (link.dataset.originalHref !== undefined) {
          link.href = link.dataset.originalHref;
        }
      }
    }
  function tippyHover(el, contentFn, onTriggerFn, onUntriggerFn) {
    const config = {
      allowHTML: true,
      maxWidth: 500,
      delay: 100,
      arrow: false,
      appendTo: function(el) {
          return el.parentElement;
      },
      interactive: true,
      interactiveBorder: 10,
      theme: 'quarto',
      placement: 'bottom-start',
    };
    if (contentFn) {
      config.content = contentFn;
    }
    if (onTriggerFn) {
      config.onTrigger = onTriggerFn;
    }
    if (onUntriggerFn) {
      config.onUntrigger = onUntriggerFn;
    }
    window.tippy(el, config); 
  }
  const noterefs = window.document.querySelectorAll('a[role="doc-noteref"]');
  for (var i=0; i<noterefs.length; i++) {
    const ref = noterefs[i];
    tippyHover(ref, function() {
      // use id or data attribute instead here
      let href = ref.getAttribute('data-footnote-href') || ref.getAttribute('href');
      try { href = new URL(href).hash; } catch {}
      const id = href.replace(/^#\/?/, "");
      const note = window.document.getElementById(id);
      if (note) {
        return note.innerHTML;
      } else {
        return "";
      }
    });
  }
  const xrefs = window.document.querySelectorAll('a.quarto-xref');
  const processXRef = (id, note) => {
    // Strip column container classes
    const stripColumnClz = (el) => {
      el.classList.remove("page-full", "page-columns");
      if (el.children) {
        for (const child of el.children) {
          stripColumnClz(child);
        }
      }
    }
    stripColumnClz(note)
    if (id === null || id.startsWith('sec-')) {
      // Special case sections, only their first couple elements
      const container = document.createElement("div");
      if (note.children && note.children.length > 2) {
        container.appendChild(note.children[0].cloneNode(true));
        for (let i = 1; i < note.children.length; i++) {
          const child = note.children[i];
          if (child.tagName === "P" && child.innerText === "") {
            continue;
          } else {
            container.appendChild(child.cloneNode(true));
            break;
          }
        }
        if (window.Quarto?.typesetMath) {
          window.Quarto.typesetMath(container);
        }
        return container.innerHTML
      } else {
        if (window.Quarto?.typesetMath) {
          window.Quarto.typesetMath(note);
        }
        return note.innerHTML;
      }
    } else {
      // Remove any anchor links if they are present
      const anchorLink = note.querySelector('a.anchorjs-link');
      if (anchorLink) {
        anchorLink.remove();
      }
      if (window.Quarto?.typesetMath) {
        window.Quarto.typesetMath(note);
      }
      if (note.classList.contains("callout")) {
        return note.outerHTML;
      } else {
        return note.innerHTML;
      }
    }
  }
  for (var i=0; i<xrefs.length; i++) {
    const xref = xrefs[i];
    tippyHover(xref, undefined, function(instance) {
      instance.disable();
      let url = xref.getAttribute('href');
      let hash = undefined; 
      if (url.startsWith('#')) {
        hash = url;
      } else {
        try { hash = new URL(url).hash; } catch {}
      }
      if (hash) {
        const id = hash.replace(/^#\/?/, "");
        const note = window.document.getElementById(id);
        if (note !== null) {
          try {
            const html = processXRef(id, note.cloneNode(true));
            instance.setContent(html);
          } finally {
            instance.enable();
            instance.show();
          }
        } else {
          // See if we can fetch this
          fetch(url.split('#')[0])
          .then(res => res.text())
          .then(html => {
            const parser = new DOMParser();
            const htmlDoc = parser.parseFromString(html, "text/html");
            const note = htmlDoc.getElementById(id);
            if (note !== null) {
              const html = processXRef(id, note);
              instance.setContent(html);
            } 
          }).finally(() => {
            instance.enable();
            instance.show();
          });
        }
      } else {
        // See if we can fetch a full url (with no hash to target)
        // This is a special case and we should probably do some content thinning / targeting
        fetch(url)
        .then(res => res.text())
        .then(html => {
          const parser = new DOMParser();
          const htmlDoc = parser.parseFromString(html, "text/html");
          const note = htmlDoc.querySelector('main.content');
          if (note !== null) {
            // This should only happen for chapter cross references
            // (since there is no id in the URL)
            // remove the first header
            if (note.children.length > 0 && note.children[0].tagName === "HEADER") {
              note.children[0].remove();
            }
            const html = processXRef(null, note);
            instance.setContent(html);
          } 
        }).finally(() => {
          instance.enable();
          instance.show();
        });
      }
    }, function(instance) {
    });
  }
      let selectedAnnoteEl;
      const selectorForAnnotation = ( cell, annotation) => {
        let cellAttr = 'data-code-cell="' + cell + '"';
        let lineAttr = 'data-code-annotation="' +  annotation + '"';
        const selector = 'span[' + cellAttr + '][' + lineAttr + ']';
        return selector;
      }
      const selectCodeLines = (annoteEl) => {
        const doc = window.document;
        const targetCell = annoteEl.getAttribute("data-target-cell");
        const targetAnnotation = annoteEl.getAttribute("data-target-annotation");
        const annoteSpan = window.document.querySelector(selectorForAnnotation(targetCell, targetAnnotation));
        const lines = annoteSpan.getAttribute("data-code-lines").split(",");
        const lineIds = lines.map((line) => {
          return targetCell + "-" + line;
        })
        let top = null;
        let height = null;
        let parent = null;
        if (lineIds.length > 0) {
            //compute the position of the single el (top and bottom and make a div)
            const el = window.document.getElementById(lineIds[0]);
            top = el.offsetTop;
            height = el.offsetHeight;
            parent = el.parentElement.parentElement;
          if (lineIds.length > 1) {
            const lastEl = window.document.getElementById(lineIds[lineIds.length - 1]);
            const bottom = lastEl.offsetTop + lastEl.offsetHeight;
            height = bottom - top;
          }
          if (top !== null && height !== null && parent !== null) {
            // cook up a div (if necessary) and position it 
            let div = window.document.getElementById("code-annotation-line-highlight");
            if (div === null) {
              div = window.document.createElement("div");
              div.setAttribute("id", "code-annotation-line-highlight");
              div.style.position = 'absolute';
              parent.appendChild(div);
            }
            div.style.top = top - 2 + "px";
            div.style.height = height + 4 + "px";
            div.style.left = 0;
            let gutterDiv = window.document.getElementById("code-annotation-line-highlight-gutter");
            if (gutterDiv === null) {
              gutterDiv = window.document.createElement("div");
              gutterDiv.setAttribute("id", "code-annotation-line-highlight-gutter");
              gutterDiv.style.position = 'absolute';
              const codeCell = window.document.getElementById(targetCell);
              const gutter = codeCell.querySelector('.code-annotation-gutter');
              gutter.appendChild(gutterDiv);
            }
            gutterDiv.style.top = top - 2 + "px";
            gutterDiv.style.height = height + 4 + "px";
          }
          selectedAnnoteEl = annoteEl;
        }
      };
      const unselectCodeLines = () => {
        const elementsIds = ["code-annotation-line-highlight", "code-annotation-line-highlight-gutter"];
        elementsIds.forEach((elId) => {
          const div = window.document.getElementById(elId);
          if (div) {
            div.remove();
          }
        });
        selectedAnnoteEl = undefined;
      };
        // Handle positioning of the toggle
    window.addEventListener(
      "resize",
      throttle(() => {
        elRect = undefined;
        if (selectedAnnoteEl) {
          selectCodeLines(selectedAnnoteEl);
        }
      }, 10)
    );
    function throttle(fn, ms) {
    let throttle = false;
    let timer;
      return (...args) => {
        if(!throttle) { // first call gets through
            fn.apply(this, args);
            throttle = true;
        } else { // all the others get throttled
            if(timer) clearTimeout(timer); // cancel #2
            timer = setTimeout(() => {
              fn.apply(this, args);
              timer = throttle = false;
            }, ms);
        }
      };
    }
      // Attach click handler to the DT
      const annoteDls = window.document.querySelectorAll('dt[data-target-cell]');
      for (const annoteDlNode of annoteDls) {
        annoteDlNode.addEventListener('click', (event) => {
          const clickedEl = event.target;
          if (clickedEl !== selectedAnnoteEl) {
            unselectCodeLines();
            const activeEl = window.document.querySelector('dt[data-target-cell].code-annotation-active');
            if (activeEl) {
              activeEl.classList.remove('code-annotation-active');
            }
            selectCodeLines(clickedEl);
            clickedEl.classList.add('code-annotation-active');
          } else {
            // Unselect the line
            unselectCodeLines();
            clickedEl.classList.remove('code-annotation-active');
          }
        });
      }
  const findCites = (el) => {
    const parentEl = el.parentElement;
    if (parentEl) {
      const cites = parentEl.dataset.cites;
      if (cites) {
        return {
          el,
          cites: cites.split(' ')
        };
      } else {
        return findCites(el.parentElement)
      }
    } else {
      return undefined;
    }
  };
  var bibliorefs = window.document.querySelectorAll('a[role="doc-biblioref"]');
  for (var i=0; i<bibliorefs.length; i++) {
    const ref = bibliorefs[i];
    const citeInfo = findCites(ref);
    if (citeInfo) {
      tippyHover(citeInfo.el, function() {
        var popup = window.document.createElement('div');
        citeInfo.cites.forEach(function(cite) {
          var citeDiv = window.document.createElement('div');
          citeDiv.classList.add('hanging-indent');
          citeDiv.classList.add('csl-entry');
          var biblioDiv = window.document.getElementById('ref-' + cite);
          if (biblioDiv) {
            citeDiv.innerHTML = biblioDiv.innerHTML;
          }
          popup.appendChild(citeDiv);
        });
        return popup.innerHTML;
      });
    }
  }
});
</script>
</div> <!-- /content -->


</body></html>