fix(stats): threshold not applied along breakdown dimension in plot.effects (#719)

flixopt/statistics_accessor.py

@@ -280,23 +280,51 @@ def _sort_dataset(ds: xr.Dataset) -> xr.Dataset:
     return ds[sorted_vars]
 
 
-def _filter_small_variables(ds: xr.Dataset, threshold: float | None) -> xr.Dataset:
-    """Remove variables where max absolute value is below threshold.
+def _drop_small_data_vars(ds: xr.Dataset, threshold: float) -> xr.Dataset:
+    """Drop data variables whose max absolute value is below threshold."""
+    max_vals = abs(ds).max()
+    keep = [v for v in ds.data_vars if float(max_vals.variables[v].values) >= threshold]
+    return ds[keep] if keep else ds
+
+
+def _drop_small_along_dim(ds: xr.Dataset, dim: str, threshold: float) -> xr.Dataset:
+    """Drop entries along ``dim`` whose max absolute value (over all other axes) is below threshold.
+
+    Needed when a breakdown such as ``by='component'`` lays entities out along a coordinate
+    of a single variable rather than as separate variables, e.g. non-invested components
+    with a near-zero contribution (see #719).
+    """
+    if dim not in ds.dims or not ds.data_vars:
+        return ds
+    arr = abs(ds.to_dataarray())
+    max_along = arr.max(dim=[x for x in arr.dims if x != dim])
+    keep_idx = ds[dim].values[(max_along >= threshold).values]
+    return ds.sel({dim: keep_idx})
 
-    Useful for filtering out solver noise or non-invested components.
+
+def _drop_small(ds: xr.Dataset, threshold: float | None, dim: str | list[str] | None = None) -> xr.Dataset:
+    """Remove entries whose max absolute value is below threshold.
+
+    Useful for filtering out solver noise or non-invested components. Always drops whole
+    data variables that are entirely below threshold; when ``dim`` is given, also drops
+    individual entries along those coordinate dimension(s).
 
     Args:
         ds: Dataset to filter.
         threshold: Minimum max absolute value to keep. If None, no filtering.
+        dim: Optional coordinate dimension(s) to filter along (e.g. 'component',
+            'contributor'). A single name or a list of names.
 
     Returns:
         Filtered dataset.
     """
     if threshold is None or not ds.data_vars:
         return ds
-    max_vals = abs(ds).max()  # Single computation for all variables
-    keep = [v for v in ds.data_vars if float(max_vals.variables[v].values) >= threshold]
-    return ds[keep] if keep else ds
+    ds = _drop_small_data_vars(ds, threshold)
+    dims = [dim] if isinstance(dim, str) else (dim or [])
+    for d in dims:
+        ds = _drop_small_along_dim(ds, d, threshold)
+    return ds
 
 
 def _filter_by_carrier(ds: xr.Dataset, carrier: str | list[str] | None) -> xr.Dataset:
@@ -1557,7 +1585,7 @@ def balance(
             ds = ds.round(round_decimals)
 
         # Filter out variables below threshold
-        ds = _filter_small_variables(ds, threshold)
+        ds = _drop_small(ds, threshold)
 
         # Build color kwargs: bus balance → component colors, component balance → carrier colors
         color_by: Literal['component', 'carrier'] = 'component' if is_bus else 'carrier'
@@ -1713,7 +1741,7 @@ def carrier_balance(
             ds = ds.round(round_decimals)
 
         # Filter out variables below threshold
-        ds = _filter_small_variables(ds, threshold)
+        ds = _drop_small(ds, threshold)
 
         # Build color kwargs with component colors (flows colored by their parent component)
         color_kwargs = self._build_color_kwargs(colors, list(ds.data_vars), color_by='component')
@@ -1793,7 +1821,7 @@ def heatmap(
         # Resolve, select, and stack into single DataArray
         resolved = self._resolve_variable_names(variables, solution)
         ds = _apply_selection(solution[resolved], select)
-        ds = _filter_small_variables(ds, threshold)
+        ds = _drop_small(ds, threshold)
         ds = _sort_dataset(ds)  # Sort for consistent plotting order
         da = xr.concat([ds[v] for v in ds.data_vars], dim=pd.Index(list(ds.data_vars), name='variable'))
 
@@ -1888,7 +1916,7 @@ def flows(
         ds = _apply_selection(ds, select)
 
         # Filter out variables below threshold
-        ds = _filter_small_variables(ds, threshold)
+        ds = _drop_small(ds, threshold)
 
         # Early return for data_only mode (skip figure creation for performance)
         if data_only:
@@ -1957,7 +1985,7 @@ def sizes(
             ds = ds[valid_labels]
 
         # Filter out variables below threshold
-        ds = _filter_small_variables(ds, threshold)
+        ds = _drop_small(ds, threshold)
 
         # Early return for data_only mode (skip figure creation for performance)
         if data_only:
@@ -2052,7 +2080,7 @@ def duration_curve(
         result_ds = ds.fxstats.to_duration_curve(normalize=normalize)
 
         # Filter out variables below threshold
-        result_ds = _filter_small_variables(result_ds, threshold)
+        result_ds = _drop_small(result_ds, threshold)
 
         # Early return for data_only mode (skip figure creation for performance)
         if data_only:
@@ -2180,8 +2208,9 @@ def effects(
         else:
             raise ValueError(f"'by' must be one of 'component', 'contributor', 'time', or None, got {by!r}")
 
-        # Filter out variables below threshold
-        ds = _filter_small_variables(ds, threshold)
+        # Filter out entries below threshold, including along the breakdown dimension
+        breakdown_dim = by if by in ('component', 'contributor') else None
+        ds = _drop_small(ds, threshold, dim=breakdown_dim)
 
         # Early return for data_only mode (skip figure creation for performance)
         if data_only:
@@ -2263,7 +2292,7 @@ def charge_states(
         ds = _apply_selection(ds, select)
 
         # Filter out variables below threshold
-        ds = _filter_small_variables(ds, threshold)
+        ds = _drop_small(ds, threshold)
 
         # Early return for data_only mode (skip figure creation for performance)
         if data_only:
@@ -2377,7 +2406,7 @@ def storage(
             flow_ds = flow_ds.round(round_decimals)
 
         # Filter out flow variables below threshold
-        flow_ds = _filter_small_variables(flow_ds, threshold)
+        flow_ds = _drop_small(flow_ds, threshold)
 
         # Early return for data_only mode (skip figure creation for performance)
         if data_only:

tests/plotting/test_solution_and_plotting.py

@@ -217,6 +217,69 @@ def test_statistics_flow_hours(self, simple_flow_system, highs_solver):
         assert isinstance(flow_hours, xr.Dataset)
         assert len(flow_hours.data_vars) > 0
 
+    @pytest.mark.parametrize('by', ['component', 'contributor'])
+    def test_effects_threshold_drops_uninvested_component(self, highs_solver, by):
+        """threshold must drop non-invested components in effects breakdown (issue #719).
+
+        When broken down by component/contributor, entities live along a coordinate of a
+        single variable rather than as separate variables, so a per-variable threshold
+        alone leaves the ~0 non-invested entry visible.
+        """
+        timesteps = pd.date_range('2024-01-15 08:00', periods=2, freq='h')
+        fs = fx.FlowSystem(timesteps)
+        fs.add_elements(
+            fx.Bus('Heat', carrier='heat'),
+            fx.Effect('costs', '€', 'Total Costs', is_standard=True, is_objective=True),
+            fx.Source(
+                'S1',
+                outputs=[
+                    fx.Flow(
+                        'S1',
+                        bus='Heat',
+                        size=fx.InvestParameters(
+                            minimum_size=10, maximum_size=500, effects_of_investment_per_size=10, mandatory=False
+                        ),
+                        effects_per_flow_hour=10,
+                    )
+                ],
+            ),
+            fx.Source(
+                'S2',
+                outputs=[
+                    fx.Flow(
+                        'S2',
+                        bus='Heat',
+                        size=fx.InvestParameters(
+                            minimum_size=10, maximum_size=500, effects_of_investment_per_size=100, mandatory=False
+                        ),
+                        effects_per_flow_hour=100,
+                    )
+                ],
+            ),
+            fx.Sink('ABC', inputs=[fx.Flow('abc', bus='Heat', size=1, fixed_relative_profile=222)]),
+        )
+        fs.optimize(highs_solver)
+
+        # S2 is the expensive source and stays uninvested -> zero cost contribution.
+        filtered = fs.stats.plot.effects('periodic', effect='costs', by=by, threshold=1.0, show=False, data_only=True)
+        kept = list(filtered.data.coords[by].values)
+        assert all('S2' not in str(label) for label in kept), f'Uninvested S2 should be dropped, got {kept}'
+        assert any('S1' in str(label) for label in kept), f'Invested S1 should remain, got {kept}'
+
+        # threshold=None keeps everything, including the zero-cost S2.
+        unfiltered = fs.stats.plot.effects(
+            'periodic', effect='costs', by=by, threshold=None, show=False, data_only=True
+        )
+        assert any('S2' in str(label) for label in unfiltered.data.coords[by].values)
+
+        # All entries below threshold -> empty breakdown, not a fallback to showing everything,
+        # and the full render must not crash on the empty dataset.
+        empty = fs.stats.plot.effects('periodic', effect='costs', by=by, threshold=1e12, show=False)
+        assert len(empty.data.coords[by].values) == 0, (
+            f'All-below-threshold should drop everything, got {list(empty.data.coords[by].values)}'
+        )
+        assert len(empty.figure.data) == 0
+
 
 # ============================================================================
 # PLOTTING WITH OPTIMIZED DATA TESTS