Add nested_model_default_partial_update flag and DefaultSettingsSource (#348)

Co-authored-by: Hasan Ramezani <[email protected]>

Author: kschwab

docs/index.md

@@ -371,6 +371,39 @@ print(Settings().model_dump())
 #> {'numbers': [1, 2, 3]}
 ```
 
+## Nested model default partial updates
+
+By default, Pydantic settings does not allow partial updates to nested model default objects. This behavior can be
+overriden by setting the `nested_model_default_partial_update` flag to `True`, which will allow partial updates on
+nested model default object fields.
+
+```py
+import os
+
+from pydantic import BaseModel
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class SubModel(BaseModel):
+    val: int = 0
+    flag: bool = False
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_nested_delimiter='__', nested_model_default_partial_update=True
+    )
+
+    nested_model: SubModel = SubModel()
+
+
+# Apply a partial update to the default object using environment variables
+os.environ['NESTED_MODEL__FLAG'] = 'True'
+
+assert Settings().model_dump() == {'nested_model': {'val': 0, 'flag': True}}
+```
+
 ## Dotenv (.env) support
 
 Dotenv files (generally named `.env`) are a common pattern that make it easy to use environment variables in a
@@ -474,7 +507,8 @@ models. There are two primary use cases for Pydantic settings CLI:
 
 By default, the experience is tailored towards use case #1 and builds on the foundations established in [parsing
 environment variables](#parsing-environment-variable-values). If your use case primarily falls into #2, you will likely
-want to enable [enforcing required arguments at the CLI](#enforce-required-arguments-at-cli).
+want to enable [enforcing required arguments at the CLI](#enforce-required-arguments-at-cli) and [nested model default
+partial updates](#nested-model-default-partial-updates).
 
 ### The Basics
 

pydantic_settings/main.py

@@ -11,6 +11,7 @@
 from .sources import (
     ENV_FILE_SENTINEL,
     CliSettingsSource,
+    DefaultSettingsSource,
     DotEnvSettingsSource,
     DotenvType,
     EnvSettingsSource,
@@ -23,6 +24,7 @@
 
 class SettingsConfigDict(ConfigDict, total=False):
     case_sensitive: bool
+    nested_model_default_partial_update: bool | None
     env_prefix: str
     env_file: DotenvType | None
     env_file_encoding: str | None
@@ -89,6 +91,8 @@ class BaseSettings(BaseModel):
 
     Args:
         _case_sensitive: Whether environment variables names should be read with case-sensitivity. Defaults to `None`.
+        _nested_model_default_partial_update: Whether to allow partial updates on nested model default object fields.
+            Defaults to `False`.
         _env_prefix: Prefix for all environment variables. Defaults to `None`.
         _env_file: The env file(s) to load settings values from. Defaults to `Path('')`, which
             means that the value from `model_config['env_file']` should be used. You can also pass
@@ -123,6 +127,7 @@ class BaseSettings(BaseModel):
     def __init__(
         __pydantic_self__,
         _case_sensitive: bool | None = None,
+        _nested_model_default_partial_update: bool | None = None,
         _env_prefix: str | None = None,
         _env_file: DotenvType | None = ENV_FILE_SENTINEL,
         _env_file_encoding: str | None = None,
@@ -149,6 +154,7 @@ def __init__(
             **__pydantic_self__._settings_build_values(
                 values,
                 _case_sensitive=_case_sensitive,
+                _nested_model_default_partial_update=_nested_model_default_partial_update,
                 _env_prefix=_env_prefix,
                 _env_file=_env_file,
                 _env_file_encoding=_env_file_encoding,
@@ -199,6 +205,7 @@ def _settings_build_values(
         self,
         init_kwargs: dict[str, Any],
         _case_sensitive: bool | None = None,
+        _nested_model_default_partial_update: bool | None = None,
         _env_prefix: str | None = None,
         _env_file: DotenvType | None = None,
         _env_file_encoding: str | None = None,
@@ -222,6 +229,11 @@ def _settings_build_values(
         # Determine settings config values
         case_sensitive = _case_sensitive if _case_sensitive is not None else self.model_config.get('case_sensitive')
         env_prefix = _env_prefix if _env_prefix is not None else self.model_config.get('env_prefix')
+        nested_model_default_partial_update = (
+            _nested_model_default_partial_update
+            if _nested_model_default_partial_update is not None
+            else self.model_config.get('nested_model_default_partial_update')
+        )
         env_file = _env_file if _env_file != ENV_FILE_SENTINEL else self.model_config.get('env_file')
         env_file_encoding = (
             _env_file_encoding if _env_file_encoding is not None else self.model_config.get('env_file_encoding')
@@ -273,7 +285,14 @@ def _settings_build_values(
         secrets_dir = _secrets_dir if _secrets_dir is not None else self.model_config.get('secrets_dir')
 
         # Configure built-in sources
-        init_settings = InitSettingsSource(self.__class__, init_kwargs=init_kwargs)
+        default_settings = DefaultSettingsSource(
+            self.__class__, nested_model_default_partial_update=nested_model_default_partial_update
+        )
+        init_settings = InitSettingsSource(
+            self.__class__,
+            init_kwargs=init_kwargs,
+            nested_model_default_partial_update=nested_model_default_partial_update,
+        )
         env_settings = EnvSettingsSource(
             self.__class__,
             case_sensitive=case_sensitive,
@@ -305,7 +324,7 @@ def _settings_build_values(
             env_settings=env_settings,
             dotenv_settings=dotenv_settings,
             file_secret_settings=file_secret_settings,
-        )
+        ) + (default_settings,)
         if not any([source for source in sources if isinstance(source, CliSettingsSource)]):
             if cli_parse_args is not None or cli_settings_source is not None:
                 cli_settings = (
@@ -352,6 +371,7 @@ def _settings_build_values(
         validate_default=True,
         case_sensitive=False,
         env_prefix='',
+        nested_model_default_partial_update=False,
         env_file=None,
         env_file_encoding=None,
         env_ignore_empty=False,

pydantic_settings/sources.py

@@ -13,7 +13,7 @@
     from argparse import BooleanOptionalAction
 from argparse import SUPPRESS, ArgumentParser, Namespace, RawDescriptionHelpFormatter, _SubParsersAction
 from collections import deque
-from dataclasses import is_dataclass
+from dataclasses import asdict, is_dataclass
 from enum import Enum
 from pathlib import Path
 from textwrap import dedent
@@ -22,6 +22,7 @@
     TYPE_CHECKING,
     Any,
     Callable,
+    Dict,
     Generic,
     Iterator,
     List,
@@ -38,8 +39,9 @@
 
 import typing_extensions
 from dotenv import dotenv_values
-from pydantic import AliasChoices, AliasPath, BaseModel, Json, RootModel
+from pydantic import AliasChoices, AliasPath, BaseModel, Json, RootModel, TypeAdapter
 from pydantic._internal._repr import Representation
+from pydantic._internal._signature import _field_name_for_signature
 from pydantic._internal._typing_extra import WithArgsTypes, origin_is_union, typing_base
 from pydantic._internal._utils import deep_update, is_model_class, lenient_issubclass
 from pydantic.dataclasses import is_pydantic_dataclass
@@ -261,21 +263,71 @@ def __call__(self) -> dict[str, Any]:
         pass
 
 
+class DefaultSettingsSource(PydanticBaseSettingsSource):
+    """
+    Source class for loading default object values.
+
+    Args:
+        settings_cls: The Settings class.
+        nested_model_default_partial_update: Whether to allow partial updates on nested model default object fields.
+            Defaults to `False`.
+    """
+
+    def __init__(self, settings_cls: type[BaseSettings], nested_model_default_partial_update: bool | None = None):
+        super().__init__(settings_cls)
+        self.defaults: dict[str, Any] = {}
+        self.nested_model_default_partial_update = (
+            nested_model_default_partial_update
+            if nested_model_default_partial_update is not None
+            else self.config.get('nested_model_default_partial_update', False)
+        )
+        if self.nested_model_default_partial_update:
+            for field_name, field_info in settings_cls.model_fields.items():
+                if is_dataclass(type(field_info.default)):
+                    self.defaults[_field_name_for_signature(field_name, field_info)] = asdict(field_info.default)
+                elif is_model_class(type(field_info.default)):
+                    self.defaults[_field_name_for_signature(field_name, field_info)] = field_info.default.model_dump()
+
+    def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
+        # Nothing to do here. Only implement the return statement to make mypy happy
+        return None, '', False
+
+    def __call__(self) -> dict[str, Any]:
+        return self.defaults
+
+    def __repr__(self) -> str:
+        return f'DefaultSettingsSource(nested_model_default_partial_update={self.nested_model_default_partial_update})'
+
+
 class InitSettingsSource(PydanticBaseSettingsSource):
     """
     Source class for loading values provided during settings class initialization.
     """
 
-    def __init__(self, settings_cls: type[BaseSettings], init_kwargs: dict[str, Any]):
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        init_kwargs: dict[str, Any],
+        nested_model_default_partial_update: bool | None = None,
+    ):
         self.init_kwargs = init_kwargs
         super().__init__(settings_cls)
+        self.nested_model_default_partial_update = (
+            nested_model_default_partial_update
+            if nested_model_default_partial_update is not None
+            else self.config.get('nested_model_default_partial_update', False)
+        )
 
     def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
         # Nothing to do here. Only implement the return statement to make mypy happy
         return None, '', False
 
     def __call__(self) -> dict[str, Any]:
-        return self.init_kwargs
+        return (
+            TypeAdapter(Dict[str, Any]).dump_python(self.init_kwargs)
+            if self.nested_model_default_partial_update
+            else self.init_kwargs
+        )
 
     def __repr__(self) -> str:
         return f'InitSettingsSource(init_kwargs={self.init_kwargs!r})'
@@ -1581,7 +1633,7 @@ def _add_parser_submodels(
         if self.cli_use_class_docs_for_groups and len(sub_models) == 1:
             model_group_kwargs['description'] = None if sub_models[0].__doc__ is None else dedent(sub_models[0].__doc__)
 
-        if model_default not in (PydanticUndefined, None):
+        if model_default is not PydanticUndefined:
             if is_model_class(type(model_default)) or is_pydantic_dataclass(type(model_default)):
                 model_default = getattr(model_default, field_name)
         else:

tests/test_settings.py

@@ -56,6 +56,7 @@
     CliPositionalArg,
     CliSettingsSource,
     CliSubCommand,
+    DefaultSettingsSource,
     SettingsError,
 )
 
@@ -499,6 +500,80 @@ class ComplexSettings(BaseSettings):
     ]
 
 
+def test_class_nested_model_default_partial_update(env):
+    class NestedA(BaseModel):
+        v0: bool
+        v1: bool
+
+    @pydantic_dataclasses.dataclass
+    class NestedB:
+        v0: bool
+        v1: bool
+
+    @dataclasses.dataclass
+    class NestedC:
+        v0: bool
+        v1: bool
+
+    class NestedD(BaseModel):
+        v0: bool = False
+        v1: bool = True
+
+    class SettingsDefaultsA(BaseSettings, env_nested_delimiter='__', nested_model_default_partial_update=True):
+        nested_a: NestedA = NestedA(v0=False, v1=True)
+        nested_b: NestedB = NestedB(v0=False, v1=True)
+        nested_d: NestedC = NestedC(v0=False, v1=True)
+        nested_c: NestedD = NestedD()
+
+    env.set('NESTED_A__V0', 'True')
+    env.set('NESTED_B__V0', 'True')
+    env.set('NESTED_C__V0', 'True')
+    env.set('NESTED_D__V0', 'True')
+    assert SettingsDefaultsA().model_dump() == {
+        'nested_a': {'v0': True, 'v1': True},
+        'nested_b': {'v0': True, 'v1': True},
+        'nested_c': {'v0': True, 'v1': True},
+        'nested_d': {'v0': True, 'v1': True},
+    }
+
+
+def test_init_kwargs_nested_model_default_partial_update(env):
+    class DeepSubModel(BaseModel):
+        v4: str
+
+    class SubModel(BaseModel):
+        v1: str
+        v2: bytes
+        v3: int
+        deep: DeepSubModel
+
+    class Settings(BaseSettings, env_nested_delimiter='__', nested_model_default_partial_update=True):
+        v0: str
+        sub_model: SubModel
+
+        @classmethod
+        def settings_customise_sources(
+            cls, settings_cls, init_settings, env_settings, dotenv_settings, file_secret_settings
+        ):
+            return env_settings, dotenv_settings, init_settings, file_secret_settings
+
+    env.set('SUB_MODEL__DEEP__V4', 'override-v4')
+
+    s_final = {'v0': '0', 'sub_model': {'v1': 'init-v1', 'v2': b'init-v2', 'v3': 3, 'deep': {'v4': 'override-v4'}}}
+
+    s = Settings(v0='0', sub_model={'v1': 'init-v1', 'v2': b'init-v2', 'v3': 3, 'deep': {'v4': 'init-v4'}})
+    assert s.model_dump() == s_final
+
+    s = Settings(v0='0', sub_model=SubModel(v1='init-v1', v2=b'init-v2', v3=3, deep=DeepSubModel(v4='init-v4')))
+    assert s.model_dump() == s_final
+
+    s = Settings(v0='0', sub_model=SubModel(v1='init-v1', v2=b'init-v2', v3=3, deep={'v4': 'init-v4'}))
+    assert s.model_dump() == s_final
+
+    s = Settings(v0='0', sub_model={'v1': 'init-v1', 'v2': b'init-v2', 'v3': 3, 'deep': DeepSubModel(v4='init-v4')})
+    assert s.model_dump() == s_final
+
+
 def test_env_str(env):
     class Settings(BaseSettings):
         apple: str = Field(None, validation_alias='BOOM')
@@ -1575,6 +1650,10 @@ def settings_customise_sources(cls, *args, **kwargs):
 
 
 def test_builtins_settings_source_repr():
+    assert (
+        repr(DefaultSettingsSource(BaseSettings, nested_model_default_partial_update=True))
+        == 'DefaultSettingsSource(nested_model_default_partial_update=True)'
+    )
     assert (
         repr(InitSettingsSource(BaseSettings, init_kwargs={'apple': 'value 0', 'banana': 'value 1'}))
         == "InitSettingsSource(init_kwargs={'apple': 'value 0', 'banana': 'value 1'})"