Merge pull request #2157 from fetchai/develop

Release v0.9.1

Author: DavidMinarsch

.spelling

@@ -162,6 +162,8 @@ unregistering
 pipx
 BibTex
 Kubernetes
+Golang
+web3
  - docs/language-agnostic-definition.md
 fetchai
 protocol_id

HISTORY.md

@@ -1,5 +1,18 @@
 # Release History
 
+## 0.9.1 (2020-01-14)
+
+- Fixes multiple issues with `MultiAgentManager` including overrides not being correctly applied
+- Restructures docs navigation
+- Updates `MultiAgentManager` documentation
+- Extends functionality of `aea upgrade` command to cover more cases
+- Fixes a bug in the `aea upgrade` command which prevented upgrading across version minors
+- Fixes a bug in `aea fetch` where the console output was inconsistent with the actual error
+- Fixes scaffold connection constructor
+- Multiple additional tests to improve stability
+- Multiple docs updates based on user feedback
+- Multiple additional tests and test stability fixes
+
 ## 0.9.0 (2020-01-06)
 
 - Adds multiple bug fixes on `MultiAgentManager`

README.md

@@ -187,7 +187,7 @@ The `fetchai/p2p_libp2p` package is partially developed in Go.
 
 - To install Go visit the [Golang site](https://golang.org/doc/install).
 
-- We use [golines](https://github.com/segmentio/golines) and [golangci-lint](https://golangci-lint.run) for linting.
+- We use [`golines`](https://github.com/segmentio/golines) and [`golangci-lint`](https://golangci-lint.run) for linting.
 
 - To run tests, use `go test -p 1 -timeout 0 -count 1 -v ./...` from the root directory of the package.
 

aea/__version__.py

@@ -22,7 +22,7 @@
 __title__ = "aea"
 __description__ = "Autonomous Economic Agent framework"
 __url__ = "https://github.com/fetchai/agents-aea.git"
-__version__ = "0.9.0"
+__version__ = "0.9.1"
 __author__ = "Fetch.AI Limited"
 __license__ = "Apache-2.0"
 __copyright__ = "2019 Fetch.AI Limited"

aea/aea_builder.py

@@ -311,6 +311,7 @@ def __init__(
         self,
         with_default_packages: bool = True,
         registry_dir: str = DEFAULT_REGISTRY_NAME,
+        build_dir_root: Optional[str] = None,
     ):
         """
         Initialize the builder.
@@ -320,6 +321,7 @@ def __init__(
         WithLogger.__init__(self, logger=_default_logger)
         self.registry_dir = os.path.join(os.getcwd(), registry_dir)
         self._with_default_packages = with_default_packages
+        self.build_dir_root = build_dir_root
         self._reset(is_full_reset=True)
 
     def reset(self, is_full_reset: bool = False) -> None:
@@ -773,7 +775,7 @@ def _set_component_build_directory(
         :return: None
         """
         configuration.build_directory = os.path.join(
-            self.AEA_CLASS.get_build_dir(),
+            self.get_build_root_directory(),
             configuration.component_type.value,
             configuration.author,
             configuration.name,
@@ -900,14 +902,12 @@ def remove_contract(self, public_id: PublicId) -> "AEABuilder":
         self.remove_component(ComponentId(ComponentType.CONTRACT, public_id))
         return self
 
-    def call_all_build_entrypoints(self, root_dir: str = "."):
+    def call_all_build_entrypoints(self):
         """Call all the build entrypoints."""
         for config in self._package_dependency_manager._dependencies.values():  # type: ignore # pylint: disable=protected-access
             self.run_build_for_component_configuration(config, logger=self.logger)
 
-        target_directory = os.path.abspath(
-            os.path.join(root_dir, self.AEA_CLASS.get_build_dir())
-        )
+        target_directory = self.get_build_root_directory()
 
         if self._build_entrypoint:
             self.logger.info("Building AEA package...")
@@ -917,9 +917,13 @@ def call_all_build_entrypoints(self, root_dir: str = "."):
                 build_entrypoint, source_directory, target_directory, logger=self.logger
             )
 
+    def get_build_root_directory(self) -> str:
+        """Get build directory root."""
+        return os.path.join(self.build_dir_root or ".", self.AEA_CLASS.get_build_dir())
+
     @classmethod
     def run_build_for_component_configuration(
-        cls, config: ComponentConfiguration, logger: Optional[logging.Logger] = None
+        cls, config: ComponentConfiguration, logger: Optional[logging.Logger] = None,
     ) -> None:
         """Run a build entrypoint script for component configuration."""
         if not config.build_entrypoint:

aea/cli/add.py

@@ -98,15 +98,17 @@ def add_item(ctx: Context, item_type: str, item_public_id: PublicId) -> None:
     :return: None
     """
     click.echo(f"Adding {item_type} '{item_public_id}'...")
-    if is_item_present(ctx, item_type, item_public_id):
-        present_item_id = get_item_id_present(ctx, item_type, item_public_id)
+    if is_item_present(ctx.cwd, ctx.agent_config, item_type, item_public_id):
+        present_item_id = get_item_id_present(
+            ctx.agent_config, item_type, item_public_id
+        )
         raise click.ClickException(
             "A {} with id '{}' already exists. Aborting...".format(
                 item_type, present_item_id
             )
         )
 
-    dest_path = get_package_path(ctx, item_type, item_public_id)
+    dest_path = get_package_path(ctx.cwd, item_type, item_public_id)
     is_local = ctx.config.get("is_local")
     is_mixed = ctx.config.get("is_mixed")
 

aea/cli/core.py

@@ -18,6 +18,7 @@
 #
 # ------------------------------------------------------------------------------
 """Core definitions for the AEA command-line tool."""
+from typing import Optional
 
 import click
 
@@ -58,6 +59,7 @@
 from aea.cli.search import search
 from aea.cli.transfer import transfer
 from aea.cli.upgrade import upgrade
+from aea.cli.utils.click_utils import registry_path_option
 from aea.cli.utils.config import get_or_create_cli_config
 from aea.cli.utils.constants import AUTHOR_KEY
 from aea.cli.utils.context import Context
@@ -77,11 +79,15 @@
     default=False,
     help="Skip consistency checks of agent during command execution.",
 )
+@registry_path_option
 @click.pass_context
-def cli(click_context, skip_consistency_check: bool) -> None:
+def cli(
+    click_context, skip_consistency_check: bool, registry_path: Optional[str]
+) -> None:
     """Command-line tool for setting up an Autonomous Economic Agent (AEA)."""
     verbosity_option = click_context.meta.pop("verbosity")
     click_context.obj = Context(cwd=".", verbosity=verbosity_option)
+    click_context.obj.registry_path = registry_path
     click_context.obj.set_config("skip_consistency_check", skip_consistency_check)
 
     # enables CTRL+C support on windows!

aea/cli/eject.py

@@ -23,10 +23,13 @@
 from typing import cast
 
 import click
+from packaging.version import Version
 
+import aea
 from aea.cli.remove import ItemRemoveHelper
 from aea.cli.utils.click_utils import PublicIdParameter
 from aea.cli.utils.config import (
+    load_item_config,
     set_cli_author,
     try_to_load_agent_config,
     update_item_config,
@@ -59,7 +62,11 @@
     SKILL,
 )
 from aea.configurations.utils import get_latest_component_id_from_prefix
-from aea.helpers.base import find_topological_order, reachable_nodes
+from aea.helpers.base import (
+    compute_specifier_from_version,
+    find_topological_order,
+    reachable_nodes,
+)
 
 
 @click.group()
@@ -149,17 +156,25 @@ def _eject_item(
     cli_author: str = cast(str, ctx.config.get("cli_author"))
     item_type_plural = item_type + "s"
     if not is_item_present(
-        ctx, item_type, public_id, is_vendor=True, with_version=True
+        ctx.cwd,
+        ctx.agent_config,
+        item_type,
+        public_id,
+        is_vendor=True,
+        with_version=True,
     ):  # pragma: no cover
         raise click.ClickException(
             f"{item_type.title()} {public_id} not found in agent's vendor items."
         )
-    src = get_package_path(ctx, item_type, public_id)
-    dst = get_package_path(ctx, item_type, public_id, is_vendor=False)
-    if is_item_present(ctx, item_type, public_id, is_vendor=False):  # pragma: no cover
+    src = get_package_path(ctx.cwd, item_type, public_id)
+    dst = get_package_path(ctx.cwd, item_type, public_id, is_vendor=False)
+    if is_item_present(
+        ctx.cwd, ctx.agent_config, item_type, public_id, is_vendor=False
+    ):  # pragma: no cover
         raise click.ClickException(
             f"{item_type.title()} {public_id} is already a non-vendor package."
         )
+    configuration = load_item_config(item_type, Path(src))
 
     if public_id.package_version.is_latest:
         # get 'concrete' public id, in case it is 'latest'
@@ -208,8 +223,18 @@ def _eject_item(
     copy_package_directory(Path(src), dst)
 
     new_public_id = PublicId(cli_author, public_id.name, DEFAULT_VERSION)
+    current_version = Version(aea.__version__)
+    new_aea_range = (
+        configuration.aea_version
+        if configuration.aea_version_specifiers.contains(current_version)
+        else compute_specifier_from_version(current_version)
+    )
     update_item_config(
-        item_type, Path(dst), author=new_public_id.author, version=new_public_id.version
+        item_type,
+        Path(dst),
+        author=new_public_id.author,
+        version=new_public_id.version,
+        aea_version=new_aea_range,
     )
     update_item_public_id_in_init(item_type, Path(dst), new_public_id)
     shutil.rmtree(src)

aea/cli/get_multiaddress.py

@@ -225,7 +225,9 @@ def _try_get_connection_multiaddress(
     if connection_id not in ctx.agent_config.connections:
         raise ValueError(f"Cannot find connection with the public id {connection_id}.")
 
-    package_path = Path(get_package_path_unified(ctx, CONNECTION, connection_id))
+    package_path = Path(
+        get_package_path_unified(ctx.cwd, ctx.agent_config, CONNECTION, connection_id)
+    )
     connection_config = cast(
         ConnectionConfig, load_item_config(CONNECTION, package_path)
     )

aea/cli/issue_certificates.py

@@ -16,7 +16,6 @@
 #   limitations under the License.
 #
 # ------------------------------------------------------------------------------
-
 """Implementation of the 'aea issue_certificates' subcommand."""
 import os
 from typing import Dict, List, cast
@@ -28,7 +27,7 @@
 from aea.cli.utils.decorators import check_aea_project
 from aea.cli.utils.loggers import logger
 from aea.cli.utils.package_utils import get_dotted_package_path_unified
-from aea.configurations.base import PublicId
+from aea.configurations.base import AgentConfig, PublicId
 from aea.configurations.constants import CONNECTION
 from aea.configurations.manager import AgentConfigManager, VariableDoesNotExist
 from aea.crypto.helpers import make_certificate
@@ -43,29 +42,66 @@
 def issue_certificates(click_context):
     """Issue certificates for connections that require them."""
     ctx = cast(Context, click_context.obj)
-    issue_certificates_(ctx)
+    agent_config_manager = AgentConfigManager.load(ctx.cwd)
+    issue_certificates_(ctx.cwd, agent_config_manager)
+    click.echo("All certificates have been issued.")
 
 
-def issue_certificates_(ctx: Context):
+def issue_certificates_(
+    project_directory: str, agent_config_manager: AgentConfigManager
+):
     """Issue certificates for connections that require them."""
-    agent_config_manager = AgentConfigManager.load(ctx.cwd)
-
-    # agent_config_manager.
     for connection_id in agent_config_manager.agent_config.connections:
-        _process_connection(ctx, agent_config_manager, connection_id)
+        cert_requests = _get_cert_requests(
+            project_directory, agent_config_manager, connection_id
+        )
+        _process_connection(
+            project_directory, agent_config_manager, cert_requests, connection_id
+        )
 
     click.echo("All certificates have been issued.")
 
 
+def _get_cert_requests(
+    project_directory: str, manager: AgentConfigManager, connection_id: PublicId
+) -> List[CertRequest]:
+    """
+    Get certificate requests, taking the overrides into account.
+
+    :param project_directory: aea project directory.
+    :param manager: AgentConfigManager
+    :param connection_id: the connection id.
+
+    :return: the list of cert requests.
+    """
+    path = get_dotted_package_path_unified(
+        project_directory, manager.agent_config, CONNECTION, connection_id
+    )
+    path_to_cert_requests = f"{path}.cert_requests"
+
+    try:
+        cert_requests = manager.get_variable(path_to_cert_requests)
+    except VariableDoesNotExist:
+        return []
+
+    cert_requests = cast(List[Dict], cert_requests)
+    return [
+        CertRequest.from_json(cert_request_json) for cert_request_json in cert_requests
+    ]
+
+
 def _process_certificate(
-    ctx: Context, cert_request: CertRequest, connection_id: PublicId
+    project_directory: str,
+    agent_config: AgentConfig,
+    cert_request: CertRequest,
+    connection_id: PublicId,
 ):
     """Process a single certificate request."""
     ledger_id = cert_request.ledger_id
     output_path = cert_request.save_path
     if cert_request.key_identifier is not None:
         key_identifier = cert_request.key_identifier
-        connection_private_key_path = ctx.agent_config.connection_private_key_paths.read(
+        connection_private_key_path = agent_config.connection_private_key_paths.read(
             key_identifier
         )
         if connection_private_key_path is None:
@@ -82,38 +118,43 @@ def _process_certificate(
             public_key is not None,
             "Internal error - one of key_identifier or public_key must be not None.",
         )
-    crypto_private_key_path = ctx.agent_config.private_key_paths.read(ledger_id)
+    crypto_private_key_path = agent_config.private_key_paths.read(ledger_id)
     if crypto_private_key_path is None:
         raise ClickException(
             f"Cannot find private key with id '{ledger_id}'. Please use `aea generate-key {key_identifier}` and `aea add-key {key_identifier}` to add a private key with id '{key_identifier}'."
         )
     message = cert_request.get_message(public_key)
     cert = make_certificate(
-        ledger_id, crypto_private_key_path, message, os.path.join(ctx.cwd, output_path)
+        ledger_id,
+        crypto_private_key_path,
+        message,
+        os.path.join(project_directory, output_path),
     )
     click.echo(f"Generated signature: '{cert}'")
 
 
 def _process_connection(
-    ctx: Context, agent_config_manager: AgentConfigManager, connection_id: PublicId
+    project_directory: str,
+    agent_config_manager: AgentConfigManager,
+    cert_requests: List[CertRequest],
+    connection_id: PublicId,
 ):
-    path = get_dotted_package_path_unified(ctx, CONNECTION, connection_id)
-    path_to_cert_requests = f"{path}.cert_requests"
 
-    try:
-        cert_requests = agent_config_manager.get_variable(path_to_cert_requests)
-    except VariableDoesNotExist:
+    if len(cert_requests) == 0:
         logger.debug("No certificates to process.")
         return
 
     logger.debug(f"Processing connection '{connection_id}'...")
-    cert_requests = cast(List[Dict], cert_requests)
-    for cert_request_json in cert_requests:
-        cert_request = CertRequest.from_json(cert_request_json)
+    for cert_request in cert_requests:
         click.echo(
             f"Issuing certificate '{cert_request.identifier}' for connection {connection_id}..."
         )
-        _process_certificate(ctx, cert_request, connection_id)
+        _process_certificate(
+            project_directory,
+            agent_config_manager.agent_config,
+            cert_request,
+            connection_id,
+        )
         click.echo(
             f"Dumped certificate '{cert_request.identifier}' in '{cert_request.save_path}' for connection {connection_id}."
         )