From b675472f55ce0aeed14b956e27880bd20a0e3aee Mon Sep 17 00:00:00 2001 From: awsbuild Date: Wed, 15 Jan 2025 19:10:45 +0000 Subject: [PATCH 1/3] Set package to version 4.0.0 --- ods_tools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ods_tools/__init__.py b/ods_tools/__init__.py index cadd191..6c45223 100644 --- a/ods_tools/__init__.py +++ b/ods_tools/__init__.py @@ -1,4 +1,4 @@ -__version__ = '3.2.9' +__version__ = '4.0.0' import logging From 07e06d464584a34032d0d68f0fe27f61733da633 Mon Sep 17 00:00:00 2001 From: Stephane Struzik <34556337+sstruzik@users.noreply.github.com> Date: Thu, 30 Jan 2025 17:18:29 +0000 Subject: [PATCH 2/3] Feature/v4 integration (#173) * add class of business detection * Point package builder to OED4 pre-release spec file * Set package to version 4.0.0 * Fix default for do_disaggregation to true (#169) * OED v4 release (#170) * branches renamed in OED repo (#172) * test oed_v4 * pep8 * ep8 * fixup test * pep8 --------- Co-authored-by: Sam Gamble Co-authored-by: awsbuild Co-authored-by: sambles --- ods_tools/data/model_settings_schema.json | 590 +++++++++++++++++++++- ods_tools/main.py | 2 +- ods_tools/oed/__init__.py | 4 +- ods_tools/oed/common.py | 55 +- ods_tools/oed/exposure.py | 61 ++- ods_tools/oed/oed_schema.py | 5 +- ods_tools/oed/validator.py | 38 +- tests/test_ods_package.py | 55 +- 8 files changed, 783 insertions(+), 27 deletions(-) diff --git a/ods_tools/data/model_settings_schema.json b/ods_tools/data/model_settings_schema.json index b6b82b5..3d26c59 100644 --- a/ods_tools/data/model_settings_schema.json +++ b/ods_tools/data/model_settings_schema.json @@ -1388,8 +1388,594 @@ "type": "boolean", "title": "Apply Oasis Disaggregation", "description": "Set whether to split terms and conditions for aggregate exposure (optional)", - "default": false - } + "default": true + }, + "string_parameters":{ + "title":"Single string paramters", + "type":"array", + "uniqueItems":true, + "items":{ + "type":"object", + "uniqueItems":false, + "title":"String options", + "description":"User selected string value", + "additionalProperties":false, + "properties":{ + "name":{ + "type":"string", + "title":"UI Option", + "description":"UI name for selection", + "minLength":1 + }, + "desc":{ + "type":"string", + "title":"Short description", + "description":"UI description for selection", + "minLength":1 + }, + "used_for":{ + "type":"string", + "title":"Where the setting is applied", + "description":"Set if this parameter is ONLY used at input 'generation' or for output 'losses'", + "enum":[ + "all", + "generation", + "losses" + ] + }, + "tooltip":{ + "type":"string", + "title":"UI tooltip", + "description":"Long description (optional)" + }, + "default":{ + "type":"string", + "title":"Initial string", + "description":"Default 'string' for variable" + } + }, + "required":[ + "name", + "desc", + "default" + ] + } + }, + "list_parameters":{ + "title":"List of strings parameters", + "type":"array", + "uniqueItems":true, + "items":{ + "type":"object", + "uniqueItems":false, + "title":"List options", + "description":"User selected list values", + "additionalProperties":false, + "properties":{ + "name":{ + "type":"string", + "title":"UI Option", + "description":"UI name for selection", + "minLength":1 + }, + "desc":{ + "type":"string", + "title":"Short description", + "description":"UI description for selection", + "minLength":1 + }, + "used_for":{ + "type":"string", + "title":"Where the setting is applied", + "description":"Set if this parameter is ONLY used at input 'generation' or for output 'losses'", + "enum":[ + "all", + "generation", + "losses" + ] + }, + "tooltip":{ + "type":"string", + "title":"UI tooltip", + "description":"Long description (optional)" + }, + "default":{ + "type":"array", + "title":"Default List value", + "description":"Default 'list' set for variable", + "items":{ + "type":"string" + } + } + }, + "required":[ + "name", + "desc", + "default" + ] + } + }, + "dictionary_parameters":{ + "title":"Generic dictionary parameters", + "type":"array", + "uniqueItems":true, + "items":{ + "type":"object", + "uniqueItems":false, + "title":"Dictionary option", + "description":"User selected dictionarys", + "additionalProperties":false, + "properties":{ + "name":{ + "type":"string", + "title":"UI Option", + "description":"UI name for selection", + "minLength":1 + }, + "desc":{ + "type":"string", + "title":"Short description", + "description":"UI description for selection", + "minLength":1 + }, + "used_for":{ + "type":"string", + "title":"Where the setting is applied", + "description":"Set if this parameter is ONLY used at input 'generation' or for output 'losses'", + "enum":[ + "all", + "generation", + "losses" + ] + }, + "tooltip":{ + "type":"string", + "title":"UI tooltip", + "description":"Long description (optional)" + }, + "default":{ + "type":"object", + "title":"Default dictionary", + "description":"Defaults set for variable" + } + }, + "required":[ + "name", + "desc", + "default" + ] + } + }, + "boolean_parameters":{ + "title":"Boolean parameters", + "type":"array", + "uniqueItems":true, + "items":{ + "type":"object", + "uniqueItems":false, + "title":"Boolean option", + "description":"User selected boolean option", + "additionalProperties":false, + "properties":{ + "name":{ + "type":"string", + "title":"UI Option", + "description":"UI name for selection", + "minLength":1 + }, + "desc":{ + "type":"string", + "title":"Short description", + "description":"UI description for selection", + "minLength":1 + }, + "used_for":{ + "type":"string", + "title":"Where the setting is applied", + "description":"Set if this parameter is ONLY used at input 'generation' or for output 'losses'", + "enum":[ + "all", + "generation", + "losses" + ] + }, + "tooltip":{ + "type":"string", + "title":"UI tooltip", + "description":"Long description (optional)" + }, + "default":{ + "type":"boolean", + "title":"Initial value", + "description":"Default 'value' set for variable" + } + }, + "required":[ + "name", + "desc", + "default" + ] + } + }, + "float_parameters":{ + "title":"Bounded float paramters", + "type":"array", + "uniqueItems":true, + "items":{ + "type":"object", + "uniqueItems":false, + "title":"Float option", + "description":"Select float value", + "additionalProperties":false, + "properties":{ + "name":{ + "type":"string", + "title":"UI Option", + "description":"UI name for selection", + "minLength":1 + }, + "desc":{ + "type":"string", + "title":"Short description", + "description":"UI description for selection", + "minLength":1 + }, + "used_for":{ + "type":"string", + "title":"Where the setting is applied", + "description":"Set if this parameter is ONLY used at input 'generation' or for output 'losses'", + "enum":[ + "all", + "generation", + "losses" + ] + }, + "tooltip":{ + "type":"string", + "title":"UI tooltip", + "description":"Long description (optional)" + }, + "default":{ + "type":"number", + "title":"Initial value", + "description":"Default 'value' set for float variable" + }, + "max":{ + "type":"number", + "title":"Maximum value", + "description":"Maximum Value for float variable" + }, + "min":{ + "type":"number", + "title":"Minimum value", + "description":"Minimum Value for float variable" + }, + "stepsize":{ + "type":"number", + "title":"Interval step size", + "description":"The slider widget's step interval for adjusting the float parameter." + } + }, + "required":[ + "name", + "desc", + "default", + "max", + "min" + ] + } + }, + "numeric_parameters":{ + "title":"unbounded numeric paramters", + "type":"array", + "description":"WARNING: option flagged for removal, superseded by `integer_parameters` and `float_parameters`", + "uniqueItems":true, + "items":{ + "type":"object", + "uniqueItems":false, + "title":"Numeric option", + "description":"Select float value", + "additionalProperties":false, + "properties":{ + "name":{ + "type":"string", + "title":"UI Option", + "description":"UI name for selection", + "minLength":1 + }, + "desc":{ + "type":"string", + "title":"Short description", + "description":"UI description for selection", + "minLength":1 + }, + "used_for":{ + "type":"string", + "title":"Where the setting is applied", + "description":"Set if this parameter is ONLY used at input 'generation' or for output 'losses'", + "enum":[ + "all", + "generation", + "losses" + ] + }, + "tooltip":{ + "type":"string", + "title":"UI tooltip", + "description":"Long description (optional)" + }, + "default":{ + "type":"number", + "title":"Initial value, integer or float", + "description":"Default integer or float 'value' set for variable" + } + }, + "required":[ + "name", + "desc", + "default" + ] + } + }, + "integer_parameters":{ + "title":"Integer paramters", + "type":"array", + "uniqueItems":true, + "items":{ + "type":"object", + "uniqueItems":false, + "title":"Integer option", + "description":"Select float value", + "additionalProperties":false, + "properties":{ + "name":{ + "type":"string", + "title":"UI Option", + "description":"UI name for selection", + "minLength":1 + }, + "desc":{ + "type":"string", + "title":"Short description", + "description":"UI description for selection", + "minLength":1 + }, + "used_for":{ + "type":"string", + "title":"Where the setting is applied", + "description":"Set if this parameter is ONLY used at input 'generation' or for output 'losses'", + "enum":[ + "all", + "generation", + "losses" + ] + }, + "tooltip":{ + "type":"string", + "title":"UI tooltip", + "description":"Long description (optional)" + }, + "default":{ + "type":"integer", + "title":"Initial integer value", + "description":"Default integer 'value' set for variable" + } + }, + "required":[ + "name", + "desc", + "default" + ] + } + }, + "dropdown_parameters":{ + "title":"Generic dropdown paramters", + "type":"array", + "uniqueItems":true, + "items":{ + "title":"dropdown option selector", + "description":"The 'id' field is mapped into the analysis settings as 'paramter_name': ''", + "type":"object", + "uniqueItems":false, + "additionalProperties":false, + "properties":{ + "name":{ + "type":"string", + "title":"UI Option", + "description":"UI name for selection", + "minLength":1 + }, + "desc":{ + "type":"string", + "title":"Short description", + "description":"UI description for selection", + "minLength":1 + }, + "used_for":{ + "type":"string", + "title":"Where the setting is applied", + "description":"Set if this parameter is ONLY used at input 'generation' or for output 'losses'", + "enum":[ + "all", + "generation", + "losses" + ] + }, + "tooltip":{ + "type":"string", + "title":"UI tooltip", + "description":"Long description (optional)" + }, + "default":{ + "type":"string", + "title":"Default Event set", + "description":"Initial setting for dropdown option" + }, + "options":{ + "type":"array", + "title":"Selection options", + "description":"Array of possible event sets", + "items":{ + "type":"object", + "title":"Option element", + "description":"Dropdown option", + "additionalProperties":false, + "properties":{ + "id":{ + "type":"string", + "title":"event set suffix", + "description":"String value used to select an event set", + "minLength":1 + }, + "desc":{ + "type":"string", + "title":"Short description", + "description":"UI description for selection", + "minLength":1 + }, + "tooltip":{ + "type":"string", + "title":"UI tooltip", + "description":"Long description (optional)" + } + }, + "required":[ + "id", + "desc" + ] + } + } + }, + "required":[ + "name", + "desc", + "default", + "options" + ] + } + }, + "multi_parameter_options":{ + "title":"Multiple Parameter option", + "description":"Sets of parameters with pre-assigned values", + "type":"array", + "uniqueItems":true, + "items":{ + "type":"object", + "uniqueItems":false, + "title":"Parameter group option", + "additionalProperties":false, + "properties":{ + "name":{ + "type":"string", + "title":"UI Option", + "description":"UI name for selection", + "minLength":1 + }, + "desc":{ + "type":"string", + "title":"Short group description", + "description":"UI description for selection", + "minLength":1 + }, + "used_for":{ + "type":"string", + "title":"Where the setting is applied", + "description":"Set if this parameter is ONLY used at input 'generation' or for output 'losses'", + "enum":[ + "all", + "generation", + "losses" + ] + }, + "tooltip":{ + "type":"string", + "title":"UI tooltip", + "description":"Long description (optional)" + }, + "config":{ + "type":"object", + "title":"Parameter Group configuration", + "description":"JSON object holding : pairs" + } + }, + "required":[ + "name", + "desc", + "config" + ] + } + }, + "parameter_groups":{ + "title":"Parameter Groups", + "type":"array", + "uniqueItems":true, + "items":{ + "title":"Grouping element", + "description":"Defines which parameter are related", + "type":"object", + "uniqueItems":false, + "additionalProperties":false, + "properties":{ + "name":{ + "type":"string", + "title":"Group name", + "description":"Reference for the group element", + "minLength":1 + }, + "desc":{ + "type":"string", + "title":"Short description", + "description":"UI description for selection", + "minLength":1 + }, + "used_for":{ + "type":"string", + "title":"Where the setting is applied", + "description":"Set if this parameter is ONLY used at input 'generation' or for output 'losses'", + "enum":[ + "all", + "generation", + "losses" + ] + }, + "tooltip":{ + "type":"string", + "title":"UI tooltip", + "description":"Long description (optional)" + }, + "priority_id":{ + "type":"integer", + "title":"Display priority", + "description":"Set which parameter groups to display first" + }, + "presentation_order":{ + "type":"array", + "title":"presentation of grouped parameters", + "description":"List of parameters reference by their 'name' property", + "items":{ + "type":"string", + "minItems":1 + } + }, + "collapsible":{ + "title":"Collapsible option for UI", + "description":"Boolean to mark if this parameter group is collapsible", + "type":"boolean" + }, + "default_collapsed":{ + "title":"Default Collapsed State", + "description":"Boolean to mark if parameter group starts collapsed", + "type":"boolean" + } + }, + "required":[ + "name", + "desc", + "priority_id", + "presentation_order" + ] + } + } } }, "lookup_settings":{ diff --git a/ods_tools/main.py b/ods_tools/main.py index f76096e..55aebef 100644 --- a/ods_tools/main.py +++ b/ods_tools/main.py @@ -46,7 +46,7 @@ def check(**kwargs): """run the check command on Exposure""" logger = logging.getLogger(__name__) args_set = {k for k, v in kwargs.items() if v is not None} - args_exp = set(['location', 'account', 'ri_info', 'ri_scope']) + args_exp = set(['location', 'account', 'ri_info', 'ri_scope', 'oed_dir']) try: if args_exp.intersection(set(args_set)): diff --git a/ods_tools/oed/__init__.py b/ods_tools/oed/__init__.py index 0a57778..f4be54e 100644 --- a/ods_tools/oed/__init__.py +++ b/ods_tools/oed/__init__.py @@ -6,7 +6,7 @@ from .common import ( OdsException, PANDAS_COMPRESSION_MAP, PANDAS_DEFAULT_NULL_VALUES, USUAL_FILE_NAME, OED_TYPE_TO_NAME, OED_NAME_TO_TYPE, OED_IDENTIFIER_FIELDS, VALIDATOR_ON_ERROR_ACTION, DEFAULT_VALIDATION_CONFIG, OED_PERIL_COLUMNS, fill_empty, - UnknownColumnSaveOption, BLANK_VALUES, is_empty + UnknownColumnSaveOption, BLANK_VALUES, is_empty, ClassOfBusiness ) @@ -15,5 +15,5 @@ 'AnalysisSettingHandler', 'ModelSettingHandler', 'ModelSettingSchema', 'AnalysisSettingSchema', 'OdsException', 'PANDAS_COMPRESSION_MAP', 'PANDAS_DEFAULT_NULL_VALUES', 'USUAL_FILE_NAME', 'OED_TYPE_TO_NAME', 'OED_NAME_TO_TYPE', 'OED_IDENTIFIER_FIELDS', 'VALIDATOR_ON_ERROR_ACTION', 'DEFAULT_VALIDATION_CONFIG', 'OED_PERIL_COLUMNS', 'fill_empty', - 'UnknownColumnSaveOption', 'BLANK_VALUES', 'is_empty', + 'UnknownColumnSaveOption', 'BLANK_VALUES', 'is_empty', 'ClassOfBusiness' ] # this is necessary for flake8 to pass, otherwise you get an unused import error diff --git a/ods_tools/oed/common.py b/ods_tools/oed/common.py index e733e3d..cbc6bd5 100644 --- a/ods_tools/oed/common.py +++ b/ods_tools/oed/common.py @@ -1,6 +1,8 @@ """ common static variable and ods_tools exceptions """ +import enum + from urllib.parse import urlparse from pathlib import Path import numpy as np @@ -125,9 +127,60 @@ def __get__(self, obj, type=None): 'ReinsScope': ['ReinsNumber', 'PortNumber', 'AccNumber', 'LocNumber'] } + +class ClassOfBusiness(enum.Enum): + prop = 'PROP' + mar = 'MAR' + cyb = 'CYB' + liabs = 'LIABS' + + +CLASS_OF_BUSINESSES = { + ClassOfBusiness.prop: { + 'name': 'Property', + 'field_status_name': 'Property field status', + 'subject_at_risk_source': 'location', + 'subject_at_risk_id_fields': ['PortNumber', 'AccNumber', 'LocNumber'], + 'coherence_rules': [ + {"name": "location", "type": "R", "r_sources": ["location"], }, + {"name": "reinsurance", "type": "CR", "c_sources": ["ri_info", "ri_scope"], "r_sources": ["account"]} + ], + }, + ClassOfBusiness.mar: { + 'name': 'Marine Cargo', + 'field_status_name': 'Marine Cargo field status', + 'subject_at_risk_source': 'location', + 'subject_at_risk_id_fields': ['PortNumber', 'AccNumber', 'LocNumber'], + 'coherence_rules': [ + {"name": "location", "type": "R", "r_sources": ["location"], }, + {"name": "reinsurance", "type": "CR", "c_sources": ["ri_info", "ri_scope"], "r_sources": ["account"]} + ], + }, + ClassOfBusiness.cyb: { + 'name': 'Cyber', + 'field_status_name': 'Cyber field status', + 'subject_at_risk_source': 'account', + 'subject_at_risk_id_fields': ['PortNumber', 'AccNumber'], + 'coherence_rules': [ + {"name": "account", "type": "R", "r_sources": ["account"]}, + {"name": "reinsurance", "type": "CR", "c_sources": ["ri_info", "ri_scope"]} + ] + }, + ClassOfBusiness.liabs: { + 'name': 'Liability', + 'field_status_name': 'Liability field status', + 'subject_at_risk_source': 'account', + 'subject_at_risk_id_fields': ['PortNumber', 'AccNumber'], + 'coherence_rules': [ + {"name": "account", "type": "R", "r_sources": ["account"]}, + {"name": "reinsurance", "type": "CR", "c_sources": ["ri_info", "ri_scope"]} + ] + }, +} + VALIDATOR_ON_ERROR_ACTION = {'raise', 'log', 'ignore', 'return'} DEFAULT_VALIDATION_CONFIG = [ - {'name': 'source_coherence', 'on_error': 'raise'}, + {'name': 'source_coherence', 'on_error': 'log'}, {'name': 'required_fields', 'on_error': 'raise'}, {'name': 'unknown_column', 'on_error': 'raise'}, {'name': 'valid_values', 'on_error': 'raise'}, diff --git a/ods_tools/oed/exposure.py b/ods_tools/oed/exposure.py index 05789a4..da81c96 100644 --- a/ods_tools/oed/exposure.py +++ b/ods_tools/oed/exposure.py @@ -14,7 +14,8 @@ from .common import (PANDAS_COMPRESSION_MAP, USUAL_FILE_NAME, OED_TYPE_TO_NAME, - UnknownColumnSaveOption) + UnknownColumnSaveOption, CLASS_OF_BUSINESSES, OdsException, + ClassOfBusiness) from .oed_schema import OedSchema from .source import OedSource from .validator import Validator @@ -30,7 +31,7 @@ class OedExposure: """ DEFAULT_EXPOSURE_CONFIG_NAME = 'exposure_info.json' - def __init__(self, + def __init__(self, *, location=None, account=None, ri_info=None, @@ -38,6 +39,7 @@ def __init__(self, oed_schema_info=None, currency_conversion=None, reporting_currency=None, + class_of_business=None, check_oed=False, use_field=False, validation_config=None, @@ -129,6 +131,8 @@ def fn(df): self.reporting_currency = reporting_currency + self.class_of_business = class_of_business + self.validation_config = validation_config if not working_dir: @@ -139,6 +143,49 @@ def fn(df): if check_oed: self.check() + def get_class_of_business(self): + any_field_info = next(iter(self.get_input_fields('null').values())) + if 'Required Field' in any_field_info: + logger.debug(f"OED schema version < 4.0.0, only support {ClassOfBusiness.prop}") + return ClassOfBusiness.prop + class_of_businesses = set(CLASS_OF_BUSINESSES) + exclusion_messages = {} + + for oed_source in self.get_oed_sources(): + present_field = set(field_info['Input Field Name'] for field_info in oed_source.get_column_to_field().values()) + for field_info in self.get_input_fields(oed_source.oed_type).values(): + for class_of_business in class_of_businesses: + cob_field_status = CLASS_OF_BUSINESSES[class_of_business]['field_status_name'] + if field_info.get(cob_field_status) == 'R': + if field_info['Input Field Name'] not in present_field: + exclusion_messages.setdefault(class_of_business, {}).setdefault('missing', []).append(field_info['Input Field Name']) + elif field_info.get(cob_field_status) == 'n/a': + if field_info['Input Field Name'] in present_field: + exclusion_messages.setdefault(class_of_business, {}).setdefault('present', []).append(field_info['Input Field Name']) + + final_cobs = class_of_businesses.difference(exclusion_messages) + if len(final_cobs) == 1: + final_cobs = final_cobs.pop() + logger.info(f"detected class of business is {final_cobs}") + return final_cobs + elif len(final_cobs) == 0: + error_msg = "\n".join(f"{class_of_business}:" + + ("\n " + ", ".join(messages['missing']) + " missing" if messages.get('missing') else "") + + ("\n " + ", ".join(messages['present']) + " present" if messages.get('present') else "") + for class_of_business, messages in exclusion_messages.items()) + raise OdsException(error_msg) + elif len(final_cobs) == 2 and len(final_cobs.difference({ClassOfBusiness.prop, ClassOfBusiness.mar})) == 0: + # Marine and Property have mostly the same column, default to Property if undetermined + return ClassOfBusiness.prop + else: + raise OdsException(f"could not determine the COB of the exposure between those {final_cobs}") + + @property + def class_of_business_info(self): + if self.class_of_business is None: + self.class_of_business = self.get_class_of_business() + return CLASS_OF_BUSINESSES[self.class_of_business] + @classmethod def resolve_oed_info(cls, oed_info, df_engine): if isinstance(oed_info, (str, Path)): @@ -208,7 +255,7 @@ def find_fp(names): kwargs['working_dir'] = oed_dir missing_files = [file for file, found in files_found.items() if not found] - if missing_files: + if missing_files and False: raise FileNotFoundError(f"Files not found in current path ({oed_dir}): {', '.join(missing_files)}") return cls(**{**config, **kwargs}) @@ -256,6 +303,11 @@ def get_oed_sources(self): if oed_source: yield oed_source + def get_subject_at_risk_source(self) -> OedSource: + if self.class_of_business is None: + self.class_of_business = self.get_class_of_business() + return getattr(self, CLASS_OF_BUSINESSES[self.class_of_business]['subject_at_risk_source']) + def save_config(self, filepath): """ save data to directory, loadable later on @@ -327,6 +379,9 @@ def check(self, validation_config=None): OdsException if some invalid data is found """ + if self.class_of_business is None: + self.class_of_business = self.get_class_of_business() + if validation_config is None: validation_config = self.validation_config validator = Validator(self) diff --git a/ods_tools/oed/oed_schema.py b/ods_tools/oed/oed_schema.py index 748750b..738d0cc 100644 --- a/ods_tools/oed/oed_schema.py +++ b/ods_tools/oed/oed_schema.py @@ -1,4 +1,5 @@ import json +import logging import os from pathlib import Path import numba as nb @@ -6,6 +7,8 @@ from .common import OdsException, BLANK_VALUES, cached_property, dtype_to_python +logger = logging.getLogger(__name__) + ENV_ODS_SCHEMA_PATH = os.getenv('ODS_SCHEMA_PATH') @@ -74,6 +77,7 @@ def from_oed_schema_info(cls, oed_schema_info): elif isinstance(oed_schema_info, cls): return oed_schema_info elif oed_schema_info is None: + logger.debug(f"loading default schema {cls.DEFAULT_ODS_SCHEMA_PATH}") return cls.from_json(cls.DEFAULT_ODS_SCHEMA_PATH) else: raise OdsException(f"{oed_schema_info} is not a supported format to create {cls} object") @@ -245,7 +249,6 @@ def is_valid_value(value, valid_ranges, allow_blanks): Returns: True if value is in one of the range """ - if value in BLANK_VALUES: return allow_blanks for valid_range in valid_ranges: diff --git a/ods_tools/oed/validator.py b/ods_tools/oed/validator.py index ba26221..e87cd63 100644 --- a/ods_tools/oed/validator.py +++ b/ods_tools/oed/validator.py @@ -6,7 +6,7 @@ from pathlib import Path from collections.abc import Iterable -from .common import (OdsException, OED_PERIL_COLUMNS, OED_IDENTIFIER_FIELDS, DEFAULT_VALIDATION_CONFIG, +from .common import (OdsException, OED_PERIL_COLUMNS, OED_IDENTIFIER_FIELDS, DEFAULT_VALIDATION_CONFIG, CLASS_OF_BUSINESSES, VALIDATOR_ON_ERROR_ACTION, BLANK_VALUES, is_empty) from .oed_schema import OedSchema @@ -92,19 +92,24 @@ def invalid_data_to_str(_data): def check_source_coherence(self): """""" invalid_data = [] - if not self.exposure.location: - invalid_data.append({'name': 'location', 'source': None, - 'msg': f"Exposure needs a Location file, location={self.exposure.location}"}) - - if self.exposure.ri_info or self.exposure.ri_scope: - if not self.exposure.account: - invalid_data.append({'name': 'account', 'source': None, - 'msg': f"Exposure needs account if reinsurance is provided account={self.exposure.account}"}) - - if not self.exposure.ri_info and self.exposure.ri_scope: - invalid_data.append({'name': 'reinsurance', 'source': None, - 'msg': f"Exposure needs both ri_scope and ri_scope for reinsurance" - f"ri_info={self.exposure.ri_info} ri_scope={self.exposure.ri_scope}"}) + coherence_rules = CLASS_OF_BUSINESSES[self.exposure.class_of_business]['coherence_rules'] + for coherence_rule in coherence_rules: + r_sources = [] + if coherence_rule["type"] == "CR": + c_sources = [getattr(self.exposure, source) for source in coherence_rule["c_sources"]] + if any(c_sources): + if not all(c_sources): + invalid_data.append( + {'name': coherence_rule['name'], 'source': None, + 'msg': f"Exposure needs all {coherence_rule['c_sources']} for {coherence_rule['name']}" + f" got {c_sources}"}) + r_sources = [getattr(self.exposure, source) for source in coherence_rule.get("r_sources", [])] + elif coherence_rule["type"] == "R": + r_sources = [getattr(self.exposure, source) for source in coherence_rule["r_sources"]] + + if not all(r_sources): + invalid_data.append({'name': coherence_rule['name'], 'source': None, + 'msg': f"Exposure needs {coherence_rule['r_sources']}, got={r_sources}"}) return invalid_data @@ -126,8 +131,9 @@ def check_required_fields(self): for field_info in input_fields.values(): if field_info['Input Field Name'] not in field_to_columns: - # OED v4 = 'Property field status' and OED v3 = 'Required Field' - requ_field_ref = 'Property field status' if 'Property field status' in field_info else 'Required Field' + requ_field_ref = CLASS_OF_BUSINESSES[self.exposure.class_of_business]['field_status_name'] + if requ_field_ref not in field_info: # OED v3 only support PROP and used 'Required Field' + requ_field_ref = 'Required Field' if field_info.get(requ_field_ref) == 'R': invalid_data.append({'name': oed_source.oed_name, 'source': oed_source.current_source, 'msg': f"missing required column {field_info['Input Field Name']}"}) diff --git a/tests/test_ods_package.py b/tests/test_ods_package.py index 27949c6..a383ad6 100644 --- a/tests/test_ods_package.py +++ b/tests/test_ods_package.py @@ -19,7 +19,8 @@ sys.path.append(sys.path.pop(0)) from ods_tools.main import convert -from ods_tools.oed import OedExposure, OedSchema, OdsException, ModelSettingSchema, AnalysisSettingSchema, OED_TYPE_TO_NAME, UnknownColumnSaveOption +from ods_tools.oed import (OedExposure, OedSchema, OdsException, ModelSettingSchema, AnalysisSettingSchema, OED_TYPE_TO_NAME, UnknownColumnSaveOption, + ClassOfBusiness) from ods_tools.odtf.controller import transform_format logger = logging.getLogger(__file__) @@ -115,6 +116,58 @@ def test_load_oed_from_config(self): exposure2 = OedExposure(**config) self.assertTrue(exposure.location.dataframe.equals(exposure2.location.dataframe)) + def test_oed_V3(self): + with tempfile.TemporaryDirectory() as tmp_run_dir: + with open(os.path.join(tmp_run_dir, 'OpenExposureData_Spec.json'), 'wb') as schema_file: + schema_file.write( + urllib.request.urlopen('https://github.com/OasisLMF/ODS_OpenExposureData/releases/download/3.4.0/OpenExposureData_Spec.json') + .read()) + + config = { + 'location': base_url + '/SourceLocOEDPiWind.csv', + 'account': base_url + '/SourceAccOEDPiWind.csv', + 'ri_info': base_url + '/SourceReinsInfoOEDPiWind.csv', + 'ri_scope': base_url + '/SourceReinsScopeOEDPiWind.csv', + 'oed_schema_info': os.path.join(tmp_run_dir, 'OpenExposureData_Spec.json'), + 'check_oed': True, + 'use_field': True, + } + assert OedExposure(**config).class_of_business == ClassOfBusiness.prop + + def test_oed_cyber_example(self): + oed_example_url = "https://raw.githubusercontent.com/OasisLMF/ODS_OpenExposureData/refs/heads/main/Examples" + config = { + 'account': oed_example_url + '/cyber_account.csv', + 'check_oed': True, # issue with current marine exemple set to true and remove the correction when fixed + 'use_field': True, + } + assert OedExposure(**config).class_of_business == ClassOfBusiness.cyb + + def test_oed_marinecargo_example(self): + oed_example_url = "https://raw.githubusercontent.com/OasisLMF/ODS_OpenExposureData/refs/heads/main/Examples" + config = { + 'location': oed_example_url + '/marinecargo_location.csv', + 'account': oed_example_url + '/marinecargo_account.csv', + 'check_oed': False, # issue with current marine exemple set to true and remove the correction when fixed + 'use_field': True, + } + ## marine example manual fixup ### + exposure = OedExposure(**config) + exposure.account.dataframe["PolDedType6All"] = 1 + config['account'] = exposure.account.dataframe + config['check_oed'] = True + ##### + assert OedExposure(**config).class_of_business in [ClassOfBusiness.prop, ClassOfBusiness.mar] + + def test_oed_liability_example(self): + oed_example_url = "https://raw.githubusercontent.com/OasisLMF/ODS_OpenExposureData/refs/heads/main/Examples" + config = { + 'account': oed_example_url + '/liability_account.csv', + 'check_oed': True, + 'use_field': True, + } + assert OedExposure(**config).class_of_business == ClassOfBusiness.liabs + def test_categorical_with_default(self): # UseReinsDates is a string column with a non null default, check default setting works with tempfile.TemporaryDirectory() as tmp_run_dir: From 8727b24767e65b136495294c30e09e2a5abf159c Mon Sep 17 00:00:00 2001 From: awsbuild Date: Fri, 31 Jan 2025 14:09:11 +0000 Subject: [PATCH 3/3] Update changelog --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f01f1b0..71a9f35 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,8 @@ ODS_Tools Changelog =================== +.. _`4.0.0`: https://github.com/OasisLMF/ODS_Tools/compare/3.2.10...4.0.0 + `3.2.9`_ --------- * [#148](https://github.com/OasisLMF/ODS_Tools/pull/162) - PLA options to be added into MS/AS schemas explicitly