Merge pull request #1933 from fetchai/develop

Release v0.7.3

HISTORY.md

@@ -1,5 +1,18 @@
 # Release History
 
+## 0.7.3 (2020-11-12)
+
+- Extends AW AEAs
+- Fixes overwriting of private key files on startup
+- Fixes behaviour bugs
+- Adds tests for tac participation skill
+- Adds development setup guide
+- Improves exception logging for easier debugging
+- Fixes mixed mode in upgrade command
+- Reduces verbosity of some CLI commands
+- Multiple docs updates based on user feedback
+- Multiple additional tests and test stability fixes
+
 ## 0.7.2 (2020-11-09)
 
 - Fixes some AW2 AEAs

Makefile

@@ -78,17 +78,17 @@ common_checks: security misc_checks lint static docs
 
 .PHONY: test
 test:
-	pytest -rfE --doctest-modules aea packages/fetchai/protocols packages/fetchai/connections packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller tests/ --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term --cov=aea --cov=packages/fetchai/protocols --cov=packages/fetchai/connections  --cov=packages/fetchai/skills/generic_buyer --cov-config=.coveragerc
+	pytest -rfE --doctest-modules aea packages/fetchai/protocols packages/fetchai/connections packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation tests/ --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term --cov=aea --cov=packages/fetchai/protocols --cov=packages/fetchai/connections --cov=packages/fetchai/skills/generic_buyer --cov=packages/fetchai/skills/generic_seller --cov=packages/fetchai/skills/tac_control --cov=packages/fetchai/skills/tac_control_contract --cov=packages/fetchai/skills/tac_participation --cov-config=.coveragerc
 	find . -name ".coverage*" -not -name ".coveragerc" -exec rm -fr "{}" \;
 
 .PHONY: test-sub
 test-sub:
-	pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller  tests/test_$(tdir) --cov=aea.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term  --cov-config=.coveragerc
+	pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation  tests/test_$(tdir) --cov=aea.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term  --cov-config=.coveragerc
 	find . -name ".coverage*" -not -name ".coveragerc" -exec rm -fr "{}" \;
 
 .PHONY: test-sub-p
 test-sub-p:
-	pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller tests/test_packages/test_$(tdir) --cov=packages.fetchai.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term  --cov-config=.coveragerc
+	pytest -rfE --doctest-modules aea packages/fetchai/connections packages/fetchai/protocols packages/fetchai/skills/generic_buyer packages/fetchai/skills/generic_seller packages/fetchai/skills/tac_control packages/fetchai/skills/tac_control_contract packages/fetchai/skills/tac_participation tests/test_packages/test_$(tdir) --cov=packages.fetchai.$(dir) --cov-report=html --cov-report=xml --cov-report=term-missing --cov-report=term  --cov-config=.coveragerc
 	find . -name ".coverage*" -not -name ".coveragerc" -exec rm -fr "{}" \;
 
 

aea/__version__.py

@@ -22,7 +22,7 @@
 __title__ = "aea"
 __description__ = "Autonomous Economic Agent framework"
 __url__ = "https://github.com/fetchai/agents-aea.git"
-__version__ = "0.7.2"
+__version__ = "0.7.3"
 __author__ = "Fetch.AI Limited"
 __license__ = "Apache-2.0"
 __copyright__ = "2019 Fetch.AI Limited"

aea/aea.py

@@ -45,15 +45,14 @@
 from aea.crypto.ledger_apis import DEFAULT_CURRENCY_DENOMINATIONS
 from aea.crypto.wallet import Wallet
 from aea.decision_maker.base import DecisionMakerHandler
-from aea.exceptions import AEAException
+from aea.exceptions import AEAException, _StopRuntime
 from aea.helpers.exception_policy import ExceptionPolicyEnum
 from aea.helpers.logging import AgentLoggerAdapter, get_logger
 from aea.identity.base import Identity
 from aea.mail.base import Envelope
 from aea.protocols.base import Message, Protocol
 from aea.registries.filter import Filter
 from aea.registries.resources import Resources
-from aea.runtime import _StopRuntime
 from aea.skills.base import Behaviour, Handler
 
 
@@ -338,7 +337,7 @@ def handle_envelope(self, envelope: Envelope) -> None:
             return
 
         for handler in handlers:
-            handler.handle(msg)
+            handler.handle_wrapper(msg)
 
     def _setup_loggers(self):
         """Set up logger with agent name."""
@@ -402,10 +401,14 @@ def exception_handler(self, exception: Exception, function: Callable) -> bool:
         :return: bool, propagate exception if True otherwise skip it.
         """
         # docstyle: ignore # noqa: E800
-        def log_exception(e, fn):
-            self.logger.exception(f"<{e}> raised during `{fn}`")
+        def log_exception(e, fn, is_debug: bool = False):
+            if is_debug:
+                self.logger.debug(f"<{e}> raised during `{fn}`")
+            else:
+                self.logger.exception(f"<{e}> raised during `{fn}`")
 
         if self._skills_exception_policy == ExceptionPolicyEnum.propagate:
+            log_exception(exception, function, is_debug=True)
             return True
 
         if self._skills_exception_policy == ExceptionPolicyEnum.stop_and_exit:
@@ -434,7 +437,6 @@ def teardown(self) -> None:
 
         :return: None
         """
-        self.logger.debug("Calling teardown method...")
         self.resources.teardown()
 
     def get_task_result(self, task_id: int) -> AsyncResult:

aea/agent_loop.py

@@ -168,7 +168,7 @@ def __init__(
 
         self._periodic_tasks: Dict[Callable, PeriodicCaller] = {}
 
-    def _periodic_task_exception_callback(
+    def _periodic_task_exception_callback(  # pylint: disable=unused-argument
         self, task_callable: Callable, exc: Exception
     ) -> None:
         """
@@ -179,9 +179,6 @@ def _periodic_task_exception_callback(
 
         :return: None
         """
-        self.logger.exception(
-            f"Loop: Exception: `{exc}` occured during `{task_callable}` processing"
-        )
         self._exceptions.append(exc)
 
     def _execution_control(

aea/cli/add.py

@@ -28,6 +28,7 @@
 from aea.cli.utils.config import load_item_config
 from aea.cli.utils.context import Context
 from aea.cli.utils.decorators import check_aea_project, clean_after, pass_ctx
+from aea.cli.utils.loggers import logger
 from aea.cli.utils.package_utils import (
     copy_package_directory,
     find_item_in_distribution,
@@ -206,7 +207,7 @@ def fetch_item_mixed(
             ctx, item_type, item_public_id, dest_path
         )
     except click.ClickException:
-        click.echo("Fetch from local registry failed, trying remote registry...")
+        logger.debug("Fetch from local registry failed, trying remote registry...")
         # the following might raise exception, but we don't catch it this time
         package_path = fetch_package(
             item_type, public_id=item_public_id, cwd=ctx.cwd, dest=dest_path

aea/cli/fetch.py

@@ -31,6 +31,7 @@
 from aea.cli.utils.config import try_to_load_agent_config
 from aea.cli.utils.context import Context
 from aea.cli.utils.decorators import clean_after
+from aea.cli.utils.loggers import logger
 from aea.cli.utils.package_utils import try_get_item_source_path
 from aea.configurations.base import DEFAULT_AEA_CONFIG_FILE, PublicId
 from aea.configurations.constants import DEFAULT_REGISTRY_PATH
@@ -166,5 +167,5 @@ def fetch_mixed(
     try:
         fetch_agent_locally(ctx, public_id, alias=alias, target_dir=target_dir)
     except click.ClickException:
-        click.echo("Fetch from local registry failed, trying on remote registry...")
+        logger.debug("Fetch from local registry failed, trying on remote registry...")
         fetch_agent(ctx, public_id, alias=alias, target_dir=target_dir)

aea/cli/registry/utils.py

@@ -227,20 +227,52 @@ def get_package_meta(obj_type: str, public_id: PublicId) -> dict:
     return resp
 
 
+def get_latest_public_id_mixed(
+    ctx: Context, item_type: str, item_public_id: PublicId,
+) -> PublicId:
+    """
+    Get latest public id of the message, mixed mode.
+
+    That is, give priority to local registry, and fall back to remote registry
+    in case of failure.
+
+    :param ctx: the CLI context.
+    :param item_type: the item type.
+    :param item_public_id: the item public id.
+    :return: the path to the found package.
+    """
+    try:
+        _, item_config = find_item_locally(ctx, item_type, item_public_id)
+        latest_item_public_id = item_config.public_id
+    except click.ClickException:
+        logger.debug(
+            "Get latest public id from local registry failed, trying remote registry..."
+        )
+        # the following might raise exception, but we don't catch it this time
+        package_meta = get_package_meta(item_type, item_public_id)
+        latest_item_public_id = PublicId.from_str(package_meta["public_id"])
+    return latest_item_public_id
+
+
 def get_latest_version_available_in_registry(
     ctx: Context, item_type: str, item_public_id: PublicId
 ) -> PublicId:
     """
-    Get latest avalable package version public id.
+    Get latest available package version public id.
 
     :param ctx: Context object.
     :param item_type: the item type.
     :param item_public_id: the item public id.
-    :return: PublicId
+    :return: the latest public id.
     """
     is_local = ctx.config.get("is_local")
+    is_mixed = ctx.config.get("is_mixed")
     try:
-        if is_local:
+        if is_mixed:
+            latest_item_public_id = get_latest_public_id_mixed(
+                ctx, item_type, item_public_id
+            )
+        elif is_local:
             _, item_config = find_item_locally(ctx, item_type, item_public_id)
             latest_item_public_id = item_config.public_id
         else:

aea/cli/run.py

@@ -33,7 +33,6 @@
 from aea.cli.utils.context import Context
 from aea.cli.utils.decorators import check_aea_project
 from aea.configurations.base import PublicId
-from aea.exceptions import AEAPackageLoadingError
 from aea.helpers.base import load_env_file
 
 
@@ -129,7 +128,5 @@ def _build_aea(
         )
         aea = builder.build(connection_ids=connection_ids)
         return aea
-    except AEAPackageLoadingError as e:  # pragma: nocover
-        raise click.ClickException("Package loading error: {}".format(str(e)))
     except Exception as e:
         raise click.ClickException(str(e))

aea/components/loader.py

@@ -25,7 +25,7 @@
 from aea.configurations.base import ComponentConfiguration, ComponentType
 from aea.connections.base import Connection
 from aea.contracts.base import Contract
-from aea.exceptions import AEAPackageLoadingError, enforce
+from aea.exceptions import AEAInstantiationException, AEAPackageLoadingError, enforce
 from aea.protocols.base import Protocol
 from aea.skills.base import Skill
 
@@ -59,6 +59,8 @@ def load_component_from_config(  # type: ignore
     component_class = component_type_to_class(component_type)
     try:
         return component_class.from_config(*args, configuration=configuration, **kwargs)  # type: ignore
+    except AEAInstantiationException as e:
+        raise e  # pramga: nocover
     except ModuleNotFoundError as e:
         _handle_error_while_loading_component_module_not_found(configuration, e)
     except Exception as e:  # pylint: disable=broad-except
@@ -133,7 +135,7 @@ def get_new_error_message_with_package_found() -> str:
         new_message = get_new_error_message_with_package_found()
 
     raise AEAPackageLoadingError(
-        "An error occurred while loading {} {}: No module named {}; {}".format(
+        "Package loading error: An error occurred while loading {} {}: No module named {}; {}".format(
             str(configuration.component_type),
             configuration.public_id,
             import_path,
@@ -152,6 +154,6 @@ def _handle_error_while_loading_component_generic_error(
     """
     raise Exception(
         "An error occurred while loading {} {}: {}".format(
-            str(configuration.component_type), configuration.public_id, str(e)
+            str(configuration.component_type), configuration.public_id, e
         )
-    ) from e
+    )

aea/connections/base.py

@@ -31,7 +31,7 @@
 from aea.configurations.base import ComponentType, ConnectionConfig, PublicId
 from aea.configurations.loader import load_component_configuration
 from aea.crypto.wallet import CryptoStore
-from aea.exceptions import enforce
+from aea.exceptions import AEAInstantiationException, enforce, parse_exception
 from aea.helpers.async_utils import AsyncState
 from aea.helpers.base import load_module
 from aea.helpers.logging import get_logger
@@ -268,12 +268,19 @@ def from_config(
             connection_class is not None,
             "Connection class '{}' not found.".format(connection_class_name),
         )
-        return connection_class(
-            configuration=configuration,
-            identity=identity,
-            crypto_store=crypto_store,
-            **kwargs,
-        )
+        try:
+            connection = connection_class(
+                configuration=configuration,
+                identity=identity,
+                crypto_store=crypto_store,
+                **kwargs,
+            )
+        except Exception as e:  # pragma: nocover
+            e_str = parse_exception(e)
+            raise AEAInstantiationException(
+                f"An error occured during instantiation of connection {configuration.public_id}/{configuration.class_name}:\n{e_str}"
+            )
+        return connection
 
     @property
     def is_connected(self) -> bool:  # pragma: nocover

aea/crypto/cosmos.py

@@ -704,14 +704,14 @@ def _get_transaction(
     )
     def _try_get_account_number_and_sequence(
         self, address: Address
-    ) -> Optional[Tuple[int, int]]:
+    ) -> Tuple[Optional[int], Optional[int]]:
         """
         Try get account number and sequence for an address.
 
         :param address: the address
         :return: a tuple of account number and sequence
         """
-        result = None  # type: Optional[Tuple[int, int]]
+        result: Tuple[Optional[int], Optional[int]] = (None, None)
         url = self.network_address + f"/auth/accounts/{address}"
         response = requests.get(url=url)
         if response.status_code == 200:

aea/crypto/helpers.py

@@ -20,6 +20,7 @@
 """Module wrapping the helpers of public and private key cryptography."""
 
 import logging
+import os
 import sys
 from pathlib import Path
 
@@ -61,6 +62,12 @@ def verify_or_create_private_keys(
         if config_private_key_path is None:
             private_key_path = PRIVATE_KEY_PATH_SCHEMA.format(identifier)
             if identifier == aea_conf.default_ledger:  # pragma: nocover
+                if os.path.exists(private_key_path):
+                    raise ValueError(
+                        "File {} for private key {} already exists. Add to aea-config.yaml.".format(
+                            repr(config_private_key_path), identifier
+                        )
+                    )
                 create_private_key(
                     identifier,
                     private_key_file=str(aea_project_path / private_key_path),