Skip to content

Refactored Water Data Pipeline #336

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 12 commits into
base: main
Choose a base branch
from
Open

Refactored Water Data Pipeline #336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
2ef5267
feat: add DSL parsing + water-data operations
Wegatriespython Apr 23, 2025
a6169f8
feat(dsl-engine/demands): migrate demands DSL
Wegatriespython Apr 23, 2025
85f1b68
feat(dsl-engine/supply): migrate supply to DSL
Wegatriespython Apr 23, 2025
cba6f2e
feat(dsl-engine/infra): migrate infra to DSL
Wegatriespython Apr 23, 2025
414c6b7
feat(citation_wrapper): utility to wrap citations
Wegatriespython Apr 23, 2025
1e76103
feat(dsl-engine/irrigation): irrigation to DSL
Wegatriespython Apr 23, 2025
5bbbabf
feat(dsl-engine/water_for_ppl): migrate to DSL
Wegatriespython Apr 23, 2025
0231bba
fix: ruff changes
Wegatriespython Apr 23, 2025
e7f1e96
fix: Resolved GDX diffs
Wegatriespython May 2, 2025
3a255cc
style: disucssion related to #336
May 5, 2025
2e250f4
fix: added ssp_user_data decorator
May 5, 2025
4143a18
fix: added ssp_user_data (ignore prev)
May 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
420 changes: 420 additions & 0 deletions message_ix_models/model/water/data/demand_rules.py
View file
Open in desktop

Large diffs are not rendered by default.

1,323 changes: 546 additions & 777 deletions message_ix_models/model/water/data/demands.py
View file
Open in desktop

Large diffs are not rendered by default.

916 changes: 916 additions & 0 deletions message_ix_models/model/water/data/demands_legacy.py
View file
Open in desktop

Large diffs are not rendered by default.

1,311 changes: 527 additions & 784 deletions message_ix_models/model/water/data/infrastructure.py
View file
Open in desktop

Large diffs are not rendered by default.

996 changes: 996 additions & 0 deletions message_ix_models/model/water/data/infrastructure_legacy.py
View file
Open in desktop

Large diffs are not rendered by default.

569 changes: 569 additions & 0 deletions message_ix_models/model/water/data/infrastructure_rules.py
View file
Open in desktop

Large diffs are not rendered by default.

193 changes: 21 additions & 172 deletions message_ix_models/model/water/data/irrigation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
"""Prepare data for water use for cooling & energy technologies."""

import pandas as pd
from message_ix import make_df

from message_ix_models import Context
from message_ix_models.util import broadcast, package_data_path
from message_ix_models.model.water.data.irrigation_rules import (
INPUT_IRRIGATION_RULES,
OUTPUT_IRRIGATION_RULES,
)
from message_ix_models.model.water.dsl_engine import build_standard
from message_ix_models.model.water.utils import safe_concat
from message_ix_models.util import package_data_path


# water & electricity for irrigation
Expand Down Expand Up @@ -40,189 +45,33 @@ def add_irr_structure(context: "Context") -> dict[str, pd.DataFrame]:
if context.type_reg == "country"
else f"{context.regions}_" + df_node["REGION"].astype(str)
)

# Reference to the water configuration
info = context["water build info"]

# probably can be removed
year_wat = [2010, 2015]
year_wat.extend(info.Y)

inp = make_df(
"input",
technology="irrigation_cereal",
value=1,
unit="-",
level="water_supply",
commodity="freshwater",
mode="M1",
time="year",
time_origin="year",
node_origin=df_node["region"],
node_loc=df_node["region"],
).pipe(broadcast, year_vtg=info.Y)

inp = pd.concat(
[
inp,
make_df(
"input",
technology="irrigation_oilcrops",
value=1,
unit="-",
level="water_supply",
commodity="freshwater",
mode="M1",
time="year",
time_origin="year",
node_origin=df_node["region"],
node_loc=df_node["region"],
).pipe(broadcast, year_vtg=info.Y),
]
)
inp_list = []
extra_args = {"year_vtg": info.Y}
current_args = {"rule_dfs": df_node}
for rule in INPUT_IRRIGATION_RULES.get_rule():
inp_list.append(
build_standard(r=rule, base_args=current_args, extra_args=extra_args)
)

inp = pd.concat(
[
inp,
make_df(
"input",
technology="irrigation_sugarcrops",
value=1,
unit="-",
level="water_supply",
commodity="freshwater",
mode="M1",
time="year",
time_origin="year",
node_origin=df_node["region"],
node_loc=df_node["region"],
).pipe(broadcast, year_vtg=info.Y),
]
)
# year_act = year_vts for tecs with 1 time-step lifetime
inp = safe_concat(inp_list)
inp["year_act"] = inp["year_vtg"]

# Electricity values per unit of irrigation water supply
# Reference: Evaluation of Water and Energy Use in
# Pressurized Irrigation Networks in Southern Spain
# Diaz et al. 2011 https://ascelibrary.org/
# doi/10.1061/%28ASCE%29IR.1943-4774.0000338
# Low Value :0.04690743
# Average Value :0.101598174
# High Value : 0.017123288

# inp = pd.concat([inp,
# make_df(
# "input",
# technology="irrigation_sugarcrops",
# value=0.04690743,
# unit="-",
# level="final",
# commodity="electr",
# mode="M1",
# time="year",
# time_origin="year",
# node_origin=df_node["region"],
# node_loc=df_node["region"],
# ).pipe(broadcast, year_vtg=year_wat, year_act=year_wat)
# ])
#
# inp = pd.concat([inp,
# make_df(
# "input",
# technology="irrigation_oilcrops",
# value=0.04690743,
# unit="-",
# level="final",
# commodity="electr",
# mode="M1",
# time="year",
# time_origin="year",
# node_origin=df_node["region"],
# node_loc=df_node["region"],
# ).pipe(broadcast, year_vtg=year_wat, year_act=year_wat)
# ])
#
# inp = pd.concat([inp,
# make_df(
# "input",
# technology="irrigation_cereal",
# value=0.04690743,
# unit="-",
# level="final",
# commodity="electr",
# mode="M1",
# time="year",
# time_origin="year",
# node_origin=df_node["region"],
# node_loc=df_node["region"],
# ).pipe(broadcast, year_vtg=year_wat, year_act=year_wat)
# ])
# inp.loc[(inp['node_loc'] == 'R11_SAS') &
# (inp['commodity'] == 'electr'),
# "value",
# ] *= 0.00004690743

results["input"] = inp

irr_out = make_df(
"output",
technology="irrigation_cereal",
value=1,
unit="km3/year",
level="irr_cereal",
commodity="freshwater",
mode="M1",
time="year",
time_dest="year",
node_loc=df_node["region"],
node_dest=df_node["region"],
).pipe(broadcast, year_vtg=info.Y)

irr_out = pd.concat(
[
irr_out,
make_df(
"output",
technology="irrigation_sugarcrops",
value=1,
unit="km3/year",
level="irr_sugarcrops",
commodity="freshwater",
mode="M1",
time="year",
time_dest="year",
node_loc=df_node["region"],
node_dest=df_node["region"],
).pipe(
broadcast,
year_vtg=info.Y,
),
]
)

irr_out = pd.concat(
[
irr_out,
make_df(
"output",
technology="irrigation_oilcrops",
value=1,
unit="km3/year",
level="irr_oilcrops",
commodity="freshwater",
mode="M1",
time="year",
time_dest="year",
node_loc=df_node["region"],
node_dest=df_node["region"],
).pipe(
broadcast,
year_vtg=info.Y,
),
]
)
irr_out_list = []
for rule in OUTPUT_IRRIGATION_RULES.get_rule():
irr_out_list.append(
build_standard(r=rule, base_args=current_args, extra_args=extra_args)
)

irr_out = safe_concat(irr_out_list)
irr_out["year_act"] = irr_out["year_vtg"]

results["output"] = irr_out
Expand Down
Loading
Loading