From 59af32fc75f2c08addde370f6bb4ca217d37cb35 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Sat, 1 May 2021 17:56:46 +0100 Subject: [PATCH 01/25] feat: update tac deployment fix: exception handling improvement in soef --- examples/tac_deploy/Dockerfile | 4 +- examples/tac_deploy/README.md | 65 +++++++++++-------- examples/tac_deploy/data/.gitignore | 1 - examples/tac_deploy/entrypoint.sh | 17 ++--- examples/tac_deploy/tac-deployment.yaml | 6 +- .../fetchai/connections/soef/connection.py | 4 +- .../fetchai/connections/soef/connection.yaml | 2 +- packages/hashes.csv | 2 +- 8 files changed, 52 insertions(+), 49 deletions(-) delete mode 100644 examples/tac_deploy/data/.gitignore diff --git a/examples/tac_deploy/Dockerfile b/examples/tac_deploy/Dockerfile index a40e06773e..adf8632d15 100644 --- a/examples/tac_deploy/Dockerfile +++ b/examples/tac_deploy/Dockerfile @@ -9,13 +9,13 @@ RUN apk add --no-cache gcc musl-dev python3-dev libffi-dev openssl-dev # https://stackoverflow.com/a/57485724 RUN apk add --update --no-cache py3-numpy py3-scipy py3-pillow -ENV PYTHONPATH "$PYTHONPATH:/usr/lib/python3.8/site-packages" +ENV PYTHONPATH "${PYTHONPATH}:/usr/lib/python3.8/site-packages" # golang RUN apk add --no-cache go # aea installation -RUN pip install --upgrade pip +RUN python -m pip install --upgrade pip RUN pip install --upgrade --force-reinstall aea[all]==1.0.0 # directories and aea cli config diff --git a/examples/tac_deploy/README.md b/examples/tac_deploy/README.md index 7df4c41109..0f6434636f 100644 --- a/examples/tac_deploy/README.md +++ b/examples/tac_deploy/README.md @@ -1,77 +1,88 @@ # Tac Deployment image +The TAC deployment deploys one controller and `n` tac participants. ### Build the image +To build the image: ``` bash -docker build -t tac-deploy -f Dockerfile . +docker build -t tac-deploy -f Dockerfile . --no-cache ``` -### Configure -add preferred amount of tac participants agents + +## Run locally + +Add preferred amount of tac participants agents to `.env` file: ``` PARTICIPANTS_AMOUNT=5 ``` -to `.env` file - -### Run locally +Run: ``` bash docker run --env-file .env -v "$(pwd)/data:/data" -ti tac-deploy ``` +## Run in the cloud -### Run in the cloud GCloud should be configured first! -tag it first +### Push image + +Tag the image first with the latest tag: ``` bash -docker image tag tac-deploy gcr.io/fetch-ai-sandbox/tac_deploy:0.0.3 +docker image tag tac-deploy gcr.io/fetch-ai-sandbox/tac_deploy:0.0.6 ``` - -push it to remote repo +Push it to remote repo: ``` bash -docker push gcr.io/fetch-ai-sandbox/tac_deploy:0.0.3 +docker push gcr.io/fetch-ai-sandbox/tac_deploy:0.0.6 ``` -run it +### Run it manually + +Run it ``` bash -kubectl run tac-deploy-test --image=gcr.io/fetch-ai-sandbox/tac_deploy:0.0.3 --env="PARTICIPANTS_AMOUNT=5" --attach +kubectl run tac-deploy-{SOMETHING} --image=gcr.io/fetch-ai-sandbox/tac_deploy:0.0.6 --env="PARTICIPANTS_AMOUNT=5" --attach ``` -access the container -run it +Or simply restart existing deployment and latest image will be used with default configs (see below): ``` bash -kubectl exec tac-deploy-test -ti -- /bin/sh -cd /data -ls -1 +kubectl delete pod tac-deploy-{SOMETHING} ``` +### Manipulate container + +To access the container run: +``` bash +kubectl exec tac-deploy-{SOMETHING} -ti -- /bin/sh +``` -stop it +To remove all logs and all keys: ``` bash -kubectl delete pod tac-deploy-test +cd ../../data +find . -name \*.log -type f -delete +find . -name \*.txt -type f -delete ``` +### Full deployment: +First, push the latest image, as per above. -deploy: -push image first! +Second, update the `tac-deployment.yaml` file and then run: ``` bash kubectl apply -f ./tac-deployment.yaml ``` -get pods list +Check for pods list: ``` bash kubectl get pods ``` -fetch logs +To fetch logs: ``` bash -kubectl cp tac-deploy-SOMETHING:/data ./output_dir +kubectl cp tac-deploy-{SOMETHING}:/data ./output_dir ``` -delete deployment +To delete deployment: ``` bash kubectl delete deployment tac-deploy ``` \ No newline at end of file diff --git a/examples/tac_deploy/data/.gitignore b/examples/tac_deploy/data/.gitignore deleted file mode 100644 index f59ec20aab..0000000000 --- a/examples/tac_deploy/data/.gitignore +++ /dev/null @@ -1 +0,0 @@ -* \ No newline at end of file diff --git a/examples/tac_deploy/entrypoint.sh b/examples/tac_deploy/entrypoint.sh index cf901f1388..b5b78ba72f 100755 --- a/examples/tac_deploy/entrypoint.sh +++ b/examples/tac_deploy/entrypoint.sh @@ -3,11 +3,8 @@ set -e LEDGER=fetchai PEER="/dns4/acn.fetch.ai/tcp/9001/p2p/16Uiu2HAmVWnopQAqq4pniYLw44VRvYxBUoRHqjz1Hh2SoCyjbyRW" - TAC_NAME='some_tac_id' BASE_PORT=10000 - - BASE_DIR=/data if [ -z "$COMPETITION_TIMEOUT" ]; @@ -37,13 +34,13 @@ echo MINUTES_TILL_START $MINUTES_TILL_START if [ -z "$SEARCH_INTERVAL" ]; then - SEARCH_INTERVAL=300 + SEARCH_INTERVAL=600 fi echo SEARCH_INTERVAL $SEARCH_INTERVAL if [ -z "$CLEANUP_INTERVAL" ]; then - CLEANUP_INTERVAL=600 + CLEANUP_INTERVAL=1800 fi echo CLEANUP_INTERVAL $CLEANUP_INTERVAL @@ -53,8 +50,6 @@ then fi echo LOG_LEVEL $LOG_LEVEL - - function generate_key (){ ledger=$1 prefix=$2 @@ -91,12 +86,12 @@ function set_agent(){ json=$(printf '{"log_file": "%s", "delegate_uri": null, "entry_peers": ["%s"], "local_uri": "127.0.0.1:%s", "public_uri": null}' "$agent_data_dir/libp2p_node.log" "$PEER" "$port") aea config set --type dict vendor.fetchai.connections.p2p_libp2p.config "$json" log_file=$agent_data_dir/$name.log - json=$(printf '{"version": 1, "formatters": {"standard": {"format": '%(asctime)s [%(levelname)s] %(name)s: %(message)s'}} "handlers": {"console": {"class": "logging.StreamHandler", "formatter": "standard", "level": "%s"}, "file": {"class": "logging.FileHandler", "filename": "%s", "mode": "w", "level": "%s"}}, "loggers": {"aea": {"level": "%s", "handlers": ["console", "file"]}}}' "$LOG_LEVEL" "$log_file" "$LOG_LEVEL" "$LOG_LEVEL") + json=$(printf '{"version": 1, "formatters": {"standard": {"format": ""}}, "handlers": {"console": {"class": "logging.StreamHandler", "formatter": "standard", "level": "%s"}, "file": {"class": "logging.FileHandler", "filename": "%s", "mode": "w", "level": "%s", "formatter": "standard"}}, "loggers": {"aea": {"level": "%s", "handlers": ["file"]}}}' "$LOG_LEVEL" "$log_file" "$LOG_LEVEL" "$LOG_LEVEL") aea config set --type dict agent.logging_config "$json" + aea config set agent.logging_config.formatters.standard.format '%(asctime)s [%(levelname)s] %(name)s: %(message)s' aea config set vendor.fetchai.connections.soef.config.token_storage_path $agent_data_dir/soef_token.txt aea config set agent.skill_exception_policy just_log aea config set agent.connection_exception_policy just_log - } function set_tac_name (){ @@ -119,7 +114,6 @@ function set_participant(){ cd .. } - agents_list='' for i in $(seq $PARTICIPANTS_AMOUNT); do @@ -128,8 +122,6 @@ do agents_list="$agent_name $agents_list" done - - cd tac_controller set_agent tac_controller $BASE_PORT set_tac_name @@ -139,4 +131,5 @@ aea config set vendor.fetchai.skills.tac_control.models.parameters.args.competit aea config set vendor.fetchai.skills.tac_control.models.parameters.args.inactivity_timeout $INACTIVITY_TIMEOUT cd .. +echo "Launching agents..." aea launch tac_controller $agents_list diff --git a/examples/tac_deploy/tac-deployment.yaml b/examples/tac_deploy/tac-deployment.yaml index 224969a333..88951e2061 100644 --- a/examples/tac_deploy/tac-deployment.yaml +++ b/examples/tac_deploy/tac-deployment.yaml @@ -18,7 +18,7 @@ spec: kubernetes.io/os: linux containers: - name: tac-deploy-container - image: gcr.io/fetch-ai-sandbox/tac_deploy:0.0.3 + image: gcr.io/fetch-ai-sandbox/tac_deploy:0.0.6 env: - name: PARTICIPANTS_AMOUNT value: "70" @@ -29,9 +29,9 @@ spec: - name: INACTIVITY_TIMEOUT value: "3600" - name: SEARCH_INTERVAL - value: "300" + value: "600" - name: CLEANUP_INTERVAL - value: "300" + value: "1800" - name: LOG_LEVEL value: "INFO" volumeMounts: diff --git a/packages/fetchai/connections/soef/connection.py b/packages/fetchai/connections/soef/connection.py index e9e469bdd9..ff19b2bfe7 100644 --- a/packages/fetchai/connections/soef/connection.py +++ b/packages/fetchai/connections/soef/connection.py @@ -1090,7 +1090,7 @@ async def connect(self) -> None: # make sure we first unregister, in case of improper previous termination try: await self._unregister_agent() - except SOEFException as e: # pragma: nocover + except Exception as e: # pylint: disable=broad-except # pragma: nocover if "Bad Requestagent lookup failed" not in str(e): # i.e. we're not trying to unregister and already unregistered agent raise e @@ -1115,7 +1115,7 @@ async def disconnect(self) -> None: try: await self._unregister_agent() - except SOEFException as e: # pragma: nocover + except Exception as e: # pylint: disable=broad-except # pragma: nocover self.logger.exception(str(e)) await self.in_queue.put(None) diff --git a/packages/fetchai/connections/soef/connection.yaml b/packages/fetchai/connections/soef/connection.yaml index cddc19c46e..c1d2f1ceb3 100644 --- a/packages/fetchai/connections/soef/connection.yaml +++ b/packages/fetchai/connections/soef/connection.yaml @@ -8,7 +8,7 @@ aea_version: '>=1.0.0, <2.0.0' fingerprint: README.md: QmVBMZaexeLaFt3ArNay7EnV2yjxejNwzraVLCrkEH6MsS __init__.py: Qmd5VBGFJHXFe1H45XoUh5mMSYBwvLSViJuGFeMgbPdQts - connection.py: QmSPeFc6HbASvTRbjTw8ke6NtiGG4PhSB36sFfjULeBbnr + connection.py: QmWyHBkHyqBq1WJ8bDcB1B9oaUnbEK6tCz9dM3V74w1LU2 fingerprint_ignore_patterns: [] connections: [] protocols: diff --git a/packages/hashes.csv b/packages/hashes.csv index f55cea0731..7dd1118570 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -46,7 +46,7 @@ fetchai/connections/p2p_libp2p_client,QmYq2U83xTTnv1WDy1quvEz47QaV2uw1ZmB6sFHoCL fetchai/connections/p2p_stub,QmToCExj3ZpdxUu3vYSMo4jZRJiMTkXMbyhbh18Hq6Nc4b fetchai/connections/prometheus,QmeRmgrccxRh8UG4AfTMct6D5wSuK9BXYoLwnohut2LJaB fetchai/connections/scaffold,QmSrZ99ccW1VDxo6kR8TWENzNXcFWmV7aje6RNcSEwqRyd -fetchai/connections/soef,QmU5qNdzipEDJFYd5nMxzXbXwy7ee4B8znJUhhSvUk7hmt +fetchai/connections/soef,QmNRiG5rz4UT2LugF1sjdXmkX43Srq6ZxpQ6KpcoKyfjJN fetchai/connections/stub,QmQjSUgExNU6Wgks9rwBa1zYsjdPkzs7FZy3SS2Lo3bcqz fetchai/connections/tcp,QmceuewTDJ8eKeCkcHH1enwF7EEocajkmuHi7QptJB7r5j fetchai/connections/webhook,QmQn8vSouUJrjzH7SNj148jRRDK3snRDMHMkB5GDHWBbMP From 1e7e5b6fd9754bc6d7d02e21c4c615ee0b3299ff Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Sat, 1 May 2021 21:11:22 +0100 Subject: [PATCH 02/25] chore: ignore dev only security warnings feat: extend tac deployment feat: improve logging and default config in multiple packages --- Makefile | 2 +- examples/tac_deploy/README.md | 20 +++++++-- examples/tac_deploy/data/.gitignore | 0 examples/tac_deploy/entrypoint.sh | 43 ++++++++++++++++--- examples/tac_deploy/tac-deployment.yaml | 12 +++++- .../connections/p2p_libp2p/connection.py | 2 +- .../connections/p2p_libp2p/connection.yaml | 5 ++- .../fetchai/skills/tac_control/behaviours.py | 5 +++ .../fetchai/skills/tac_control/handlers.py | 4 +- .../fetchai/skills/tac_control/skill.yaml | 4 +- .../skills/tac_participation/behaviours.py | 10 +++-- .../skills/tac_participation/handlers.py | 1 + .../skills/tac_participation/skill.yaml | 4 +- packages/hashes.csv | 6 +-- tox.ini | 2 +- 15 files changed, 93 insertions(+), 27 deletions(-) create mode 100644 examples/tac_deploy/data/.gitignore diff --git a/Makefile b/Makefile index 70ab85b23c..16be07f074 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ security: plugins/aea-ledger-cosmos/aea_ledger_cosmos \ plugins/aea-cli-ipfs/aea_cli_ipfs bandit -s B101 -r tests scripts - safety check -i 37524 -i 38038 -i 37776 -i 38039 -i 39621 + safety check -i 37524 -i 38038 -i 37776 -i 38039 -i 39621 -i 40291 -i 39706 .PHONY: static static: diff --git a/examples/tac_deploy/README.md b/examples/tac_deploy/README.md index 0f6434636f..74ea71478b 100644 --- a/examples/tac_deploy/README.md +++ b/examples/tac_deploy/README.md @@ -29,19 +29,19 @@ GCloud should be configured first! Tag the image first with the latest tag: ``` bash -docker image tag tac-deploy gcr.io/fetch-ai-sandbox/tac_deploy:0.0.6 +docker image tag tac-deploy gcr.io/fetch-ai-sandbox/tac_deploy:0.0.7 ``` Push it to remote repo: ``` bash -docker push gcr.io/fetch-ai-sandbox/tac_deploy:0.0.6 +docker push gcr.io/fetch-ai-sandbox/tac_deploy:0.0.7 ``` ### Run it manually Run it ``` bash -kubectl run tac-deploy-{SOMETHING} --image=gcr.io/fetch-ai-sandbox/tac_deploy:0.0.6 --env="PARTICIPANTS_AMOUNT=5" --attach +kubectl run tac-deploy-{SOMETHING} --image=gcr.io/fetch-ai-sandbox/tac_deploy:0.0.7 --env="PARTICIPANTS_AMOUNT=5" --attach ``` Or simply restart existing deployment and latest image will be used with default configs (see below): @@ -85,4 +85,16 @@ kubectl cp tac-deploy-{SOMETHING}:/data ./output_dir To delete deployment: ``` bash kubectl delete deployment tac-deploy -``` \ No newline at end of file +``` + +### Analysing Logs: + +Handy commands to analyse logs: +``` bash +grep -rl 'TAKE CARE! Circumventing controller identity check!' output_dir/ | sort +grep -rl 'TAKE CARE! Circumventing controller identity check!' output_dir/ | wc -l +grep -rnw 'SOEF Network Connection Error' output_dir/ | wc -l +grep -rnw 'connect to libp2p process within timeout' output_dir/ | wc -l +grep -rnw 'handling valid transaction' output_dir/tac_controller/ | wc -l +grep -rnw 'received invalid tac message' output_dir/tac_controller/ | wc -l +``` diff --git a/examples/tac_deploy/data/.gitignore b/examples/tac_deploy/data/.gitignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/tac_deploy/entrypoint.sh b/examples/tac_deploy/entrypoint.sh index b5b78ba72f..4028903bff 100755 --- a/examples/tac_deploy/entrypoint.sh +++ b/examples/tac_deploy/entrypoint.sh @@ -32,11 +32,17 @@ then fi echo MINUTES_TILL_START $MINUTES_TILL_START -if [ -z "$SEARCH_INTERVAL" ]; +if [ -z "$SEARCH_INTERVAL_GAME" ]; then - SEARCH_INTERVAL=600 + SEARCH_INTERVAL_GAME=20 fi -echo SEARCH_INTERVAL $SEARCH_INTERVAL +echo SEARCH_INTERVAL_GAME $SEARCH_INTERVAL_GAME + +if [ -z "$SEARCH_INTERVAL_TRADING" ]; +then + SEARCH_INTERVAL_TRADING=600 +fi +echo SEARCH_INTERVAL_TRADING $SEARCH_INTERVAL_TRADING if [ -z "$CLEANUP_INTERVAL" ]; then @@ -44,12 +50,38 @@ then fi echo CLEANUP_INTERVAL $CLEANUP_INTERVAL +if [ -z "$NODE_CONNECTION_TIMEOUT" ]; +then + NODE_CONNECTION_TIMEOUT=20 +fi +echo NODE_CONNECTION_TIMEOUT $NODE_CONNECTION_TIMEOUT + if [ -z "$LOG_LEVEL" ]; then LOG_LEVEL=INFO fi echo LOG_LEVEL $LOG_LEVEL +if [ -z "$CLEAR_LOG_DATA_ON_LAUNCH" ]; +then + CLEAR_LOG_DATA_ON_LAUNCH=true +fi +echo CLEAR_LOG_DATA_ON_LAUNCH $CLEAR_LOG_DATA_ON_LAUNCH + +if [ -z "$CLEAR_KEY_DATA_ON_LAUNCH" ]; +then + CLEAR_KEY_DATA_ON_LAUNCH=false +fi +echo CLEAR_KEY_DATA_ON_LAUNCH $CLEAR_KEY_DATA_ON_LAUNCH + +if [ "$CLEAR_LOG_DATA_ON_LAUNCH" == true ]; then + find "$BASE_DIR" -name \*.log -type f -delete +fi + +if [ "$CLEAR_KEY_DATA_ON_LAUNCH" == true ]; then + find "$BASE_DIR" -name \*.txt -type f -delete +fi + function generate_key (){ ledger=$1 prefix=$2 @@ -88,7 +120,7 @@ function set_agent(){ log_file=$agent_data_dir/$name.log json=$(printf '{"version": 1, "formatters": {"standard": {"format": ""}}, "handlers": {"console": {"class": "logging.StreamHandler", "formatter": "standard", "level": "%s"}, "file": {"class": "logging.FileHandler", "filename": "%s", "mode": "w", "level": "%s", "formatter": "standard"}}, "loggers": {"aea": {"level": "%s", "handlers": ["file"]}}}' "$LOG_LEVEL" "$log_file" "$LOG_LEVEL" "$LOG_LEVEL") aea config set --type dict agent.logging_config "$json" - aea config set agent.logging_config.formatters.standard.format '%(asctime)s [%(levelname)s] %(name)s: %(message)s' + aea config set agent.logging_config.formatters.standard.format '%(asctime)s [%(levelname)s]: %(message)s' aea config set vendor.fetchai.connections.soef.config.token_storage_path $agent_data_dir/soef_token.txt aea config set agent.skill_exception_policy just_log aea config set agent.connection_exception_policy just_log @@ -109,7 +141,8 @@ function set_participant(){ sed -e "s/tac_participant_template/$agent_name/" -i ./aea-config.yaml set_agent $agent_name $(expr $BASE_PORT + $agent_id) aea config set vendor.fetchai.skills.tac_negotiation.behaviours.clean_up.args.tick_interval $CLEANUP_INTERVAL - aea config set vendor.fetchai.skills.tac_negotiation.behaviours.tac_negotiation.args.search_interval $SEARCH_INTERVAL + aea config set vendor.fetchai.skills.tac_negotiation.behaviours.tac_negotiation.args.search_interval $SEARCH_INTERVAL_TRADING + aea config set vendor.fetchai.skills.tac_participation.behaviours.tac_search.args.tick_interval $SEARCH_INTERVAL_GAME aea config set vendor.fetchai.skills.tac_participation.models.game.args.search_query.search_value $TAC_NAME cd .. } diff --git a/examples/tac_deploy/tac-deployment.yaml b/examples/tac_deploy/tac-deployment.yaml index 88951e2061..3a2bc579bf 100644 --- a/examples/tac_deploy/tac-deployment.yaml +++ b/examples/tac_deploy/tac-deployment.yaml @@ -18,7 +18,7 @@ spec: kubernetes.io/os: linux containers: - name: tac-deploy-container - image: gcr.io/fetch-ai-sandbox/tac_deploy:0.0.6 + image: gcr.io/fetch-ai-sandbox/tac_deploy:0.0.7 env: - name: PARTICIPANTS_AMOUNT value: "70" @@ -28,12 +28,20 @@ spec: value: "86400" - name: INACTIVITY_TIMEOUT value: "3600" - - name: SEARCH_INTERVAL + - name: SEARCH_INTERVAL_GAME + value: "20" + - name: SEARCH_INTERVAL_TRADING value: "600" - name: CLEANUP_INTERVAL value: "1800" + - name: NODE_CONNECTION_TIMEOUT + value: "20" - name: LOG_LEVEL value: "INFO" + - name: CLEAR_LOG_DATA_ON_LAUNCH + value: "true" + - name: CLEAR_KEY_DATA_ON_LAUNCH + value: "false" volumeMounts: - name: tac-deploy-data-vol mountPath: /data diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index 0d95fb8146..83ea39917a 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -541,7 +541,7 @@ def __init__(self, **kwargs: Any) -> None: "storage_path" ) node_connection_timeout: Optional[float] = self.configuration.config.get( - "node_connection_timeout" + "node_connection_timeout", PIPE_CONN_TIMEOUT ) if ( self.has_crypto_store diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 7eaa722b82..1009ac6ce4 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -11,7 +11,7 @@ fingerprint: README.md: QmVRxZcTquf2m7j5ZpvCogKRuGWiHnLxBPx55CpyTxWdey __init__.py: QmYQuLNyQ8WTjgRYAoKAzoJEb7ocKXvM2hTyK4hsGch5D6 check_dependencies.py: QmP14nkQ8senwzdPdrZJLsA6EQ7zaKKEaLGDELhT42gp1P - connection.py: QmQ7zWZELuzwpYfbm6kJDvSFXa23h46cwsH5DRnFnC7Uyx + connection.py: Qmb1o2qD6jgpqarBwuMYDqvySETUyc1MyxHHcRv7SPHuwc libp2p_node/README.md: QmSNJEQQwh25QSHgQPM4C84xKU6FczRpmJdN7HkCQyDPuC libp2p_node/aea/api.go: QmdFR5Rmkk2FGVDwzgHtjobjAKyLejqk2CAYBCvcF23AG7 libp2p_node/aea/envelope.pb.go: QmRfUNGpCeVJfsW3H1MzCN4pwDWgumfyWufVFp6xvUjjug @@ -50,7 +50,10 @@ config: local_uri: 127.0.0.1:9000 log_file: libp2p_node.log max_node_restarts: 5 + monitoring_uri: null + node_connection_timeout: 10 public_uri: 127.0.0.1:9000 + storage_path: null cert_requests: - identifier: acn ledger_id: fetchai diff --git a/packages/fetchai/skills/tac_control/behaviours.py b/packages/fetchai/skills/tac_control/behaviours.py index f452e283ca..87577bd14a 100644 --- a/packages/fetchai/skills/tac_control/behaviours.py +++ b/packages/fetchai/skills/tac_control/behaviours.py @@ -239,6 +239,11 @@ def _unregister_agent(self) -> None: def _start_tac(self, game: Game) -> None: """Create a game and send the game configuration to every registered agent.""" + count = len(game.conf.agent_addr_to_name) + participant_names = sorted(list(game.conf.agent_addr_to_name.values())) + self.context.logger.info( + f"starting competition with {count} participants and list of participants: {participant_names}" + ) self.context.logger.info( "started competition:\n{}".format(game.holdings_summary) ) diff --git a/packages/fetchai/skills/tac_control/handlers.py b/packages/fetchai/skills/tac_control/handlers.py index 8552f3eb12..350d0c188e 100644 --- a/packages/fetchai/skills/tac_control/handlers.py +++ b/packages/fetchai/skills/tac_control/handlers.py @@ -99,7 +99,9 @@ def _handle_unidentified_dialogue(self, tac_msg: TacMessage) -> None: :param tac_msg: the message """ self.context.logger.info( - "received invalid tac message={}, unidentified dialogue.".format(tac_msg) + "received invalid tac message={}, unidentified dialogue (reference={}).".format( + tac_msg, tac_msg.dialogue_reference + ) ) default_dialogues = cast(DefaultDialogues, self.context.default_dialogues) default_msg, _ = default_dialogues.create( diff --git a/packages/fetchai/skills/tac_control/skill.yaml b/packages/fetchai/skills/tac_control/skill.yaml index dedefc8d25..200ccc9d4f 100644 --- a/packages/fetchai/skills/tac_control/skill.yaml +++ b/packages/fetchai/skills/tac_control/skill.yaml @@ -9,10 +9,10 @@ aea_version: '>=1.0.0, <2.0.0' fingerprint: README.md: QmPakq2qYHCenyWKfqQHRLWdF7uo6eG24GTprsBfrkyny2 __init__.py: QmbjPVuLk3Va17gPzVvXuj3LWGNTpeRxymnPtkHSBZR6pz - behaviours.py: QmSAqcSS8obnivk9A3Xmu4rnARLW2xNgGRCAynRX1d7hmG + behaviours.py: QmY12kkAT7zpaKxiuh1htku7PhYxMikVX7aaBeBJCQtQWy dialogues.py: QmbQt96MAo8f4xjZKHLwRpiu1jDkyRV6k6FfK16zGkcMid game.py: QmTEpteGoMD6eQvfrW762DEfr3S3gP86LwWg9uPpiC2eCd - handlers.py: QmQUCiruXh9fzPijE2B67m2oiFEtaWGLjjA2zu3o9q53pG + handlers.py: QmZWcktpzb6hHaHpsDEz4h4RTjeVi9wBTxkhCxmYWt38Uz helpers.py: QmdgSxjFPGr19Ptw4gLKLD4mvgZurzitwF9MzTNy1cZF2P parameters.py: QmcvNYRk7ED2LqedLkFfVwFsRF9iuXpXQWMdsVXqEsKzMo fingerprint_ignore_patterns: [] diff --git a/packages/fetchai/skills/tac_participation/behaviours.py b/packages/fetchai/skills/tac_participation/behaviours.py index 7d577540ec..458ba2ad2c 100644 --- a/packages/fetchai/skills/tac_participation/behaviours.py +++ b/packages/fetchai/skills/tac_participation/behaviours.py @@ -121,10 +121,9 @@ def _process_transactions(self) -> None: ) tx_ids = list(transactions.keys()) for tx_id in tx_ids: - self.context.logger.info( - "sending transaction {} to controller.".format(tx_id) - ) - last_msg = tac_dialogue.last_message + last_msg = ( + tac_dialogue.last_message + ) # could be a problem if messages are delivered out of order if last_msg is None: raise ValueError("No last message available.") tx_content = transactions.pop(tx_id, None) @@ -147,4 +146,7 @@ def _process_transactions(self) -> None: counterparty_signature=counterparty_signature, nonce=terms.nonce, ) + self.context.logger.info( + "sending transaction {} to controller, message={}.".format(tx_id, msg) + ) self.context.outbox.put_message(message=msg) diff --git a/packages/fetchai/skills/tac_participation/handlers.py b/packages/fetchai/skills/tac_participation/handlers.py index 9cc30c695c..ff81c62a76 100644 --- a/packages/fetchai/skills/tac_participation/handlers.py +++ b/packages/fetchai/skills/tac_participation/handlers.py @@ -338,6 +338,7 @@ def _update_ownership_and_preferences(self, tac_msg: TacMessage) -> None: :param tac_msg: the game data :return: None """ + self.context.logger.info("processing game data, message={}".format(tac_msg)) state_update_dialogues = cast( StateUpdateDialogues, self.context.state_update_dialogues ) diff --git a/packages/fetchai/skills/tac_participation/skill.yaml b/packages/fetchai/skills/tac_participation/skill.yaml index 6a1b81e507..e4b5b9ff4f 100644 --- a/packages/fetchai/skills/tac_participation/skill.yaml +++ b/packages/fetchai/skills/tac_participation/skill.yaml @@ -9,10 +9,10 @@ aea_version: '>=1.0.0, <2.0.0' fingerprint: README.md: QmXMADH6TxdWKhQ5EUxq7XjzHN4ECK3rD6Zj8RV2Pt9DYF __init__.py: Qma4J7Z7yW95JKyoeuSte5wpk9cXa9sYyvjV7Vwn5fWC6Y - behaviours.py: QmVBTgWJXLiow5L5zP7W2CJL8NqnPJ2D6gAUv517W9ahx7 + behaviours.py: QmbGGjMFH4ijGo2VYjXbYUxHQTSaDJLwswzEmwCXXzSP4H dialogues.py: QmU2ydtDjQUAiU5gyCQNCSBSR6RVCpwbVVTvyaArPzqxuQ game.py: QmaWS1wv8eGeMC4z8Jb28PxhQ5Y7CjNd7viigsns2AoGfL - handlers.py: QmY7WpXh9DiAs2yjXiqbres9vkUmuUEBePdk1AxcZak8nw + handlers.py: QmUkghPgkE6EpwdcwgJWuAZJuNcvHSghKHsjb5qFTfLikD fingerprint_ignore_patterns: [] connections: [] contracts: diff --git a/packages/hashes.csv b/packages/hashes.csv index 7dd1118570..74059eb33e 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -41,7 +41,7 @@ fetchai/connections/http_server,QmZqiszQJuG7XA6LFWsMqXYSmViTrUkDfpkHwgYtDMbyXy fetchai/connections/ledger,QmT7ffwPzJ3isCMhN2qoj6NRyqinE2RkpSpUKNRFRXxpes fetchai/connections/local,QmUxLhmeE98S8BcuRDB7W7pRsJzpC3wVJV5ELLxVeEkoKC fetchai/connections/oef,QmaHQhxryQLaBk5TvC4iJFZtFvkPp4CoHxHg1iLnh2PAdm -fetchai/connections/p2p_libp2p,QmPTDRUe9oMKoNVkvVB7XC3mkKmVoBoL7ziYzKzqZPtFz2 +fetchai/connections/p2p_libp2p,QmPkAszJ5DmVVGJjaLu6VkH3jLfDN7PuL4NH8UTq4TKz2y fetchai/connections/p2p_libp2p_client,QmYq2U83xTTnv1WDy1quvEz47QaV2uw1ZmB6sFHoCL31GC fetchai/connections/p2p_stub,QmToCExj3ZpdxUu3vYSMo4jZRJiMTkXMbyhbh18Hq6Nc4b fetchai/connections/prometheus,QmeRmgrccxRh8UG4AfTMct6D5wSuK9BXYoLwnohut2LJaB @@ -103,10 +103,10 @@ fetchai/skills/simple_oracle_client,QmUNDGJEczurkhpEgNbuiQLjkyNpM1S4g3Kc5BNRf6MD fetchai/skills/simple_seller,QmcLVFyejLjJRZdR7PkpKGz3uFUBXTPfCAa3N3HR2NQu9Y fetchai/skills/simple_service_registration,QmRqH46RtqU693NZFqsjU9QS9RrRQNYhv6vEKZ7XhU7gVb fetchai/skills/simple_service_search,QmargbxeoFVqCxqxNpbWTr2byDKH32Tyw6iutLUpa63UF6 -fetchai/skills/tac_control,QmTn2HAFmrfDXk6h1egMx4pv9BQoS7N8seWvuDjb8eWJZe +fetchai/skills/tac_control,QmSFqmv5uuKdkaGixqk2xssajBoaU8LKAFTm1EyrfZt5sB fetchai/skills/tac_control_contract,Qmd9vMc6EXXfM4s3iZzgZxN2HTc1p3zqiRbuUEaHHoXNnB fetchai/skills/tac_negotiation,QmakLsqLdpZCzRJXxxkk15Fj9QhLjfnHqjDd8TeXis86X7 -fetchai/skills/tac_participation,QmPWeHSrDEQdb9UWqoPu2PmJRf3pgtF73ZLZ5GDeQnF7fH +fetchai/skills/tac_participation,QmQt62chmPz5DHdoyxAoVDafGY2XapeaabvA67Mq9tqxYJ fetchai/skills/thermometer,QmaVJhnDBGufjwn4WtV6515mAUCsDc8eWJN14ugQ5pnPhQ fetchai/skills/thermometer_client,QmVDoVuHf44i7ZaXmBGgNHczdqiE7o3q9SjKZsZeWUsnLt fetchai/skills/weather_client,QmR2BAWxLgSxNW7Tugz2rgBLdzMwWRxCWocKxRXV9Zqts6 diff --git a/tox.ini b/tox.ini index a87002cced..4bd94462a6 100644 --- a/tox.ini +++ b/tox.ini @@ -265,7 +265,7 @@ skipsdist = True skip_install = True deps = safety==1.10.3 -commands = safety check -i 37524 -i 38038 -i 37776 -i 38039 -i 39621 +commands = safety check -i 37524 -i 38038 -i 37776 -i 38039 -i 39621 -i 40291 -i 39706 [testenv:vulture] skipsdist = True From 0a1157eec01e8e7979fbe93c9f7a7bcbccc2b942 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Sat, 1 May 2021 21:55:19 +0100 Subject: [PATCH 03/25] fix: broken tests --- examples/tac_deploy/README.md | 3 +++ .../test_tac_control/test_handlers.py | 2 +- .../test_tac_participation/test_behaviours.py | 22 ++++--------------- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/examples/tac_deploy/README.md b/examples/tac_deploy/README.md index 74ea71478b..af091c1ed6 100644 --- a/examples/tac_deploy/README.md +++ b/examples/tac_deploy/README.md @@ -94,6 +94,9 @@ Handy commands to analyse logs: grep -rl 'TAKE CARE! Circumventing controller identity check!' output_dir/ | sort grep -rl 'TAKE CARE! Circumventing controller identity check!' output_dir/ | wc -l grep -rnw 'SOEF Network Connection Error' output_dir/ | wc -l +grep -rnw 'SOEF Server Bad Response Error' output_dir/ | wc -l +grep -rnw 'Failure during pipe closing.' output_dir/ | wc -l +grep -rnw 'Exception' output_dir/ | wc -l grep -rnw 'connect to libp2p process within timeout' output_dir/ | wc -l grep -rnw 'handling valid transaction' output_dir/tac_controller/ | wc -l grep -rnw 'received invalid tac message' output_dir/tac_controller/ | wc -l diff --git a/tests/test_packages/test_skills/test_tac_control/test_handlers.py b/tests/test_packages/test_skills/test_tac_control/test_handlers.py index a7c3840044..2403c1edea 100644 --- a/tests/test_packages/test_skills/test_tac_control/test_handlers.py +++ b/tests/test_packages/test_skills/test_tac_control/test_handlers.py @@ -106,7 +106,7 @@ def test_handle_unidentified_dialogue(self): # after mock_logger.assert_any_call( logging.INFO, - f"received invalid tac message={incoming_message}, unidentified dialogue.", + f"received invalid tac message={incoming_message}, unidentified dialogue (reference={incoming_message.dialogue_reference}).", ) self.assert_quantity_in_outbox(1) has_attributes, error_str = self.message_has_attributes( diff --git a/tests/test_packages/test_skills/test_tac_participation/test_behaviours.py b/tests/test_packages/test_skills/test_tac_participation/test_behaviours.py index 3a3ab55ddd..bea68e0f96 100644 --- a/tests/test_packages/test_skills/test_tac_participation/test_behaviours.py +++ b/tests/test_packages/test_skills/test_tac_participation/test_behaviours.py @@ -274,14 +274,8 @@ def test_process_transactions_tac_dialogue_is_empty(self): self.game._tac_dialogue = tac_dialogue # operation - with patch.object(self.logger, "log") as mock_logger: - with pytest.raises(ValueError, match="No last message available."): - self.transaction_process_behaviour.act() - - # after - mock_logger.assert_any_call( - logging.INFO, f"sending transaction {self.tx_ids[0]} to controller." - ) + with pytest.raises(ValueError, match="No last message available."): + self.transaction_process_behaviour.act() def test_process_transactions_invalid_tx(self): """Test the _process_transactions method of the transaction_process behaviour where transactions are None.""" @@ -297,16 +291,8 @@ def test_process_transactions_invalid_tx(self): self.game._tac_dialogue = tac_dialogue # operation - with patch.object(self.logger, "log") as mock_logger: - with pytest.raises( - ValueError, match=f"Tx for id={self.tx_ids[0]} not found." - ): - self.transaction_process_behaviour.act() - - # after - mock_logger.assert_any_call( - logging.INFO, f"sending transaction {self.tx_ids[0]} to controller." - ) + with pytest.raises(ValueError, match=f"Tx for id={self.tx_ids[0]} not found."): + self.transaction_process_behaviour.act() def test_teardown(self): """Test the teardown method of the transaction_process behaviour.""" From 5a90dd55f531ee8f8699a6f9b1e79e71e728f101 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Sun, 2 May 2021 20:20:51 +0100 Subject: [PATCH 04/25] fix: transaction ordering in tac --- .../skills/tac_negotiation/handlers.py | 3 +- .../fetchai/skills/tac_negotiation/skill.yaml | 2 +- .../skills/tac_participation/behaviours.py | 4 ++- .../skills/tac_participation/skill.yaml | 2 +- packages/hashes.csv | 4 +-- .../test_tac_participation/test_behaviours.py | 33 ++++++++++--------- 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/packages/fetchai/skills/tac_negotiation/handlers.py b/packages/fetchai/skills/tac_negotiation/handlers.py index a281ab33aa..3e29a0f393 100644 --- a/packages/fetchai/skills/tac_negotiation/handlers.py +++ b/packages/fetchai/skills/tac_negotiation/handlers.py @@ -19,6 +19,7 @@ """This package contains a scaffold of a handler.""" +from collections import OrderedDict from typing import Optional, Tuple, cast from aea.configurations.base import PublicId @@ -516,7 +517,7 @@ def _handle_signed_message( counterparty_signature = fipa_dialogue.counterparty_signature tx_id = fipa_dialogue.terms.sender_hash if "transactions" not in self.context.shared_state.keys(): - self.context.shared_state["transactions"] = {} + self.context.shared_state["transactions"] = OrderedDict() tx = { "terms": fipa_dialogue.terms, "sender_signature": signing_msg.signed_message.body, diff --git a/packages/fetchai/skills/tac_negotiation/skill.yaml b/packages/fetchai/skills/tac_negotiation/skill.yaml index e29e3725ab..9003d426b1 100644 --- a/packages/fetchai/skills/tac_negotiation/skill.yaml +++ b/packages/fetchai/skills/tac_negotiation/skill.yaml @@ -11,7 +11,7 @@ fingerprint: __init__.py: QmUz2i53GeSpQVmuQSrZQABdT83k39tSPi4qTUiFKZTvJw behaviours.py: QmQrguMfyMXUnSgGBhNE4D6ep1or3TxPQ2zXj7EVFwtwcT dialogues.py: Qmes8sm5yw7pYsKbGbfZNvihJnmtqDdzbdWmrZdx8p8P18 - handlers.py: QmNXd9d5aBVmPK1bPsp28voaN4becNHKnpAvABFcHfdB38 + handlers.py: QmSAevkXv9ZCnYVAwB3J3o1FkyFYaNTnDSsPmqy26m7cMg helpers.py: QmTJbGL8V6CLhbVhLekqKkHbu7cJMfBcv8DtWLSpkKP5tk strategy.py: QmfYTikXbLjEsjNUt6ayr61pPhUYVbMSVqhQt7L1E53NT8 transactions.py: QmPZoTBt1otvnkdMDwWdPExLKA7kk6fyUS1prVZqipVUVM diff --git a/packages/fetchai/skills/tac_participation/behaviours.py b/packages/fetchai/skills/tac_participation/behaviours.py index 458ba2ad2c..2ad3e22f9a 100644 --- a/packages/fetchai/skills/tac_participation/behaviours.py +++ b/packages/fetchai/skills/tac_participation/behaviours.py @@ -19,6 +19,7 @@ """This package contains a tac search behaviour.""" +from collections import OrderedDict from typing import Any, Dict, cast from aea.skills.behaviours import TickerBehaviour @@ -117,7 +118,8 @@ def _process_transactions(self) -> None: game = cast(Game, self.context.game) tac_dialogue = game.tac_dialogue transactions = cast( - Dict[str, Dict[str, Any]], self.context.shared_state.get("transactions", {}) + Dict[str, Dict[str, Any]], + self.context.shared_state.get("transactions", OrderedDict()), ) tx_ids = list(transactions.keys()) for tx_id in tx_ids: diff --git a/packages/fetchai/skills/tac_participation/skill.yaml b/packages/fetchai/skills/tac_participation/skill.yaml index e4b5b9ff4f..cf183208a7 100644 --- a/packages/fetchai/skills/tac_participation/skill.yaml +++ b/packages/fetchai/skills/tac_participation/skill.yaml @@ -9,7 +9,7 @@ aea_version: '>=1.0.0, <2.0.0' fingerprint: README.md: QmXMADH6TxdWKhQ5EUxq7XjzHN4ECK3rD6Zj8RV2Pt9DYF __init__.py: Qma4J7Z7yW95JKyoeuSte5wpk9cXa9sYyvjV7Vwn5fWC6Y - behaviours.py: QmbGGjMFH4ijGo2VYjXbYUxHQTSaDJLwswzEmwCXXzSP4H + behaviours.py: QmRkpYvYX7v81Hi9MgqMQNBJxHxHtFnWCVKVjNUbsuxjfj dialogues.py: QmU2ydtDjQUAiU5gyCQNCSBSR6RVCpwbVVTvyaArPzqxuQ game.py: QmaWS1wv8eGeMC4z8Jb28PxhQ5Y7CjNd7viigsns2AoGfL handlers.py: QmUkghPgkE6EpwdcwgJWuAZJuNcvHSghKHsjb5qFTfLikD diff --git a/packages/hashes.csv b/packages/hashes.csv index 74059eb33e..3e51a49df7 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -105,8 +105,8 @@ fetchai/skills/simple_service_registration,QmRqH46RtqU693NZFqsjU9QS9RrRQNYhv6vEK fetchai/skills/simple_service_search,QmargbxeoFVqCxqxNpbWTr2byDKH32Tyw6iutLUpa63UF6 fetchai/skills/tac_control,QmSFqmv5uuKdkaGixqk2xssajBoaU8LKAFTm1EyrfZt5sB fetchai/skills/tac_control_contract,Qmd9vMc6EXXfM4s3iZzgZxN2HTc1p3zqiRbuUEaHHoXNnB -fetchai/skills/tac_negotiation,QmakLsqLdpZCzRJXxxkk15Fj9QhLjfnHqjDd8TeXis86X7 -fetchai/skills/tac_participation,QmQt62chmPz5DHdoyxAoVDafGY2XapeaabvA67Mq9tqxYJ +fetchai/skills/tac_negotiation,QmVQ27pXA3DZsiCaSma4TWdsQYaCvNThAbPkVQYEMoAcS3 +fetchai/skills/tac_participation,QmbdmUxFgiaoV8MdZkoMBTcmsWCKazbeKMuD8j25VLxrPF fetchai/skills/thermometer,QmaVJhnDBGufjwn4WtV6515mAUCsDc8eWJN14ugQ5pnPhQ fetchai/skills/thermometer_client,QmVDoVuHf44i7ZaXmBGgNHczdqiE7o3q9SjKZsZeWUsnLt fetchai/skills/weather_client,QmR2BAWxLgSxNW7Tugz2rgBLdzMwWRxCWocKxRXV9Zqts6 diff --git a/tests/test_packages/test_skills/test_tac_participation/test_behaviours.py b/tests/test_packages/test_skills/test_tac_participation/test_behaviours.py index bea68e0f96..efeac41a57 100644 --- a/tests/test_packages/test_skills/test_tac_participation/test_behaviours.py +++ b/tests/test_packages/test_skills/test_tac_participation/test_behaviours.py @@ -19,6 +19,7 @@ """This module contains the tests of the behaviour classes of the tac participation skill.""" import logging +from collections import OrderedDict from pathlib import Path from typing import cast from unittest.mock import patch @@ -177,18 +178,20 @@ def setup(cls): "counterparty_signature_2", ] - cls.txs = { - cls.tx_ids[0]: { - "terms": cls.terms[0], - "sender_signature": cls.sender_signatures[0], - "counterparty_signature": cls.counterparty_signatures[0], - }, - cls.tx_ids[1]: { - "terms": cls.terms[1], - "sender_signature": cls.sender_signatures[1], - "counterparty_signature": cls.counterparty_signatures[1], - }, - } + cls.txs = OrderedDict( + { + cls.tx_ids[0]: { + "terms": cls.terms[0], + "sender_signature": cls.sender_signatures[0], + "counterparty_signature": cls.counterparty_signatures[0], + }, + cls.tx_ids[1]: { + "terms": cls.terms[1], + "sender_signature": cls.sender_signatures[1], + "counterparty_signature": cls.counterparty_signatures[1], + }, + } + ) def test_setup(self): """Test the setup method of the transaction_process behaviour.""" @@ -216,7 +219,6 @@ def test_act_ii(self): self.skill.skill_context._agent_context._shared_state = { "transactions": self.txs } - tac_dialogue = self.prepare_skill_dialogue( self.tac_dialogues, self.list_of_tac_messages, ) @@ -232,10 +234,11 @@ def test_act_ii(self): # _process_transactions count = 0 while count != no_tx: + message = self.get_message_from_outbox() mock_logger.assert_any_call( - logging.INFO, f"sending transaction {self.tx_ids[count]} to controller." + logging.INFO, + f"sending transaction {self.tx_ids[count]} to controller, message={message}.", ) - message = self.get_message_from_outbox() has_attributes, error_str = self.message_has_attributes( actual_message=message, message_type=TacMessage, From 15adb8403cb3b8b0234f840f8573ec67ac0eec91 Mon Sep 17 00:00:00 2001 From: Amit Solanki Date: Mon, 3 May 2021 11:11:23 +0530 Subject: [PATCH 05/25] feat: staging environment deployment automation --- .github/workflows/publish.yaml | 53 +++++++++++++++++++++++++++++++++ scripts/acn/build_upload_img.sh | 24 ++++++--------- 2 files changed, 62 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/publish.yaml diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000000..32e1d497ee --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,53 @@ +name: Build, Publish and Deploy Docker Container + +on: + push: + branches: + - develop + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout + uses: actions/checkout@master + + - name: Setup GCloud - sandbox + uses: GoogleCloudPlatform/github-actions/setup-gcloud@master + if: github.ref == 'refs/heads/develop' + with: + project_id: ${{ secrets.GCLOUD_FETCH_AI_SANDBOX_PROJECT }} + service_account_key: ${{ secrets.GCLOUD_FETCH_AI_SANDBOX_KEY }} + + - name: Configure Docker + run: | + gcloud auth configure-docker + + - name: Set Image Tag + id: vars + run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + + # Push image to Google Container Registry + - name: Build and Push Images + run: | + chmod +x ./scripts/acn/build_upload_img.sh + + if [ ${{ github.ref }} == 'refs/heads/develop' ] + then + ./scripts/acn/build_upload_img.sh + fi + + - name: Repository Dispatch + env: + IMAGE_TAG: ${{ steps.vars.outputs.sha_short }} + run: | + if [ ${{ github.ref }} == 'refs/heads/develop' ] + then + curl -H "Accept: application/vnd.github.everest-preview+json" \ + -H "Authorization: token ${{ secrets.GH_PAT }}" \ + --request POST \ + --data '{"event_type": "agents-dht-testnet", "client_payload": {"image": "gcr.io/fetch-ai-sandbox/acn_node", "tag": "'"$IMAGE_TAG"'"}}' \ + https://api.github.com/repos/fetchai/infra-sandbox-london-b-deployment/dispatches + fi diff --git a/scripts/acn/build_upload_img.sh b/scripts/acn/build_upload_img.sh index 83919a7e53..50f262d041 100755 --- a/scripts/acn/build_upload_img.sh +++ b/scripts/acn/build_upload_img.sh @@ -1,16 +1,10 @@ - #!/usr/bin/env bash -VERSION=$(git describe --always --dirty=-WIP) +set -e -#Check For WIP -if [[ $VERSION == *-WIP ]]; -then - echo "WIP detected - please commit changes" - exit 1 -fi +VERSION=$(git rev-parse --short HEAD) -# read -p 'Where to upload the image (prod, or colearn)?: ' envvar -envvar="colearn" +# read -p 'Where to upload the image (prod, or sandbox)?: ' envvar +envvar=${1:-"sandbox"} shopt -s nocasematch case "$envvar" in "prod" ) @@ -19,9 +13,9 @@ case "$envvar" in DOCKERFILE="Dockerfile.dev" echo "Registry to upload is $REGISTRY" ;; - "colearn" ) - echo "colearn config selected" - REGISTRY="gcr.io/fetch-ai-colearn" + "sandbox" ) + echo "sandbox config selected" + REGISTRY="gcr.io/fetch-ai-sandbox" DOCKERFILE="Dockerfile.dev" echo "Registry to upload is $REGISTRY" ;; @@ -34,5 +28,5 @@ esac sleep 2 -docker build -t ${REGISTRY}/acn_node:${VERSION} -f ${DOCKERFILE} ../../ -docker push ${REGISTRY}/acn_node:${VERSION} \ No newline at end of file +docker build -t ${REGISTRY}/acn_node:${VERSION} -f ./scripts/acn/${DOCKERFILE} ./ +docker push ${REGISTRY}/acn_node:${VERSION} From f0246b0460b74a112c88b775fabf7ec31766fdc5 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 3 May 2021 12:37:40 +0200 Subject: [PATCH 06/25] test: add testson p2p_libp2p_client add test on message order guarantees. The test sets up two p2p_libp2p client connections, and makes one send a burst of envelopes to the other. The test then checks whether the receiver has received all the envelopes and in the right order. --- .../test_communication.py | 57 ++++++++++++++++--- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_communication.py b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_communication.py index eb46ca0751..df497928b6 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p_client/test_communication.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p_client/test_communication.py @@ -29,7 +29,7 @@ from aea_ledger_ethereum import EthereumCrypto from aea_ledger_fetchai import FetchAICrypto -from aea.mail.base import Envelope +from aea.mail.base import Empty, Envelope from aea.multiplexer import Multiplexer from packages.fetchai.connections.p2p_libp2p_client.connection import NodeClient, Uri @@ -627,7 +627,7 @@ def test_libp2pclientconnection_uri(): @libp2p_log_on_failure_all -class BaseTestLibp2pClientReconnection: +class BaseTestLibp2pClientSamePeer: """Base test class for reconnection tests.""" @classmethod @@ -697,12 +697,18 @@ def teardown_class(cls): except (OSError, IOError): pass - def _make_envelope(self, sender_address: str, receiver_address: str): + def _make_envelope( + self, + sender_address: str, + receiver_address: str, + message_id: int = 1, + target: int = 0, + ): """Make an envelope for testing purposes.""" msg = DefaultMessage( dialogue_reference=("", ""), - message_id=1, - target=0, + message_id=message_id, + target=target, performative=DefaultMessage.Performative.BYTES, content=b"hello", ) @@ -715,7 +721,8 @@ def _make_envelope(self, sender_address: str, receiver_address: str): return envelope -class TestLibp2pClientReconnectionSendEnvelope(BaseTestLibp2pClientReconnection): +@libp2p_log_on_failure_all +class TestLibp2pClientReconnectionSendEnvelope(BaseTestLibp2pClientSamePeer): """Test that connection will send envelope with error, and that reconnection fixes it.""" def test_envelope_sent(self): @@ -747,7 +754,7 @@ def test_envelope_sent(self): @libp2p_log_on_failure_all -class TestLibp2pClientReconnectionReceiveEnvelope(BaseTestLibp2pClientReconnection): +class TestLibp2pClientReconnectionReceiveEnvelope(BaseTestLibp2pClientSamePeer): """Test that connection will receive envelope with error, and that reconnection fixes it.""" def test_envelope_received(self): @@ -785,6 +792,42 @@ def test_envelope_received(self): assert delivered_envelope.message == envelope.message +class TestLibp2pClientEnvelopeOrderSamePeer(BaseTestLibp2pClientSamePeer): + """Test that the order of envelope is the guaranteed to be the same.""" + + NB_ENVELOPES = 1000 + + def test_burst_order(self): + """Test order of envelope burst is guaranteed on receiving end.""" + addr_1 = self.connection_client_1.address + addr_2 = self.connection_client_2.address + + sent_envelopes = [ + self._make_envelope(addr_1, addr_2, i, i - 1) + for i in range(1, self.NB_ENVELOPES + 1) + ] + for envelope in sent_envelopes: + self.multiplexer_client_1.put(envelope) + + received_envelopes = [] + for _ in range(1, self.NB_ENVELOPES + 1): + envelope = self.multiplexer_client_2.get(block=True, timeout=20) + received_envelopes.append(envelope) + + # test no new message is "created" + with pytest.raises(Empty): + self.multiplexer_client_2.get(block=True, timeout=1) + + assert len(sent_envelopes) == len( + received_envelopes + ), f"expected number of envelopes {len(sent_envelopes)}, got {len(received_envelopes)}" + for expected, actual in zip(sent_envelopes, received_envelopes): + assert expected.message == actual.message, ( + "message content differ; probably a wrong message " + "ordering on the receiving end" + ) + + @pytest.mark.asyncio async def test_nodeclient_pipe_connect(): """Test pipe.connect called on NodeClient.connect.""" From 7350b7e74eddfe02946a88a9f25238465c6671de Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 3 May 2021 14:44:17 +0200 Subject: [PATCH 07/25] test: add test on p2p_libp2p client communication for message ordering client 1 connected to peer 1 sends a burst of envelopes to client 2 connected to peer 2, and vice-versa. Then, check that clients have received the messages in the same order they have been sent. --- .../connections/p2p_libp2p/connection.yaml | 2 +- .../libp2p_node/dht/dhtpeer/dhtpeer_test.go | 117 ++++++++++++++++++ packages/hashes.csv | 2 +- 3 files changed, 119 insertions(+), 2 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 1009ac6ce4..80109c114b 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -28,7 +28,7 @@ fingerprint: libp2p_node/dht/dhtnode/utils.go: QmcHGWgCXh1mtZCL9S9yQ2qjgKB2AbsYiG5sajrSBjCGuF libp2p_node/dht/dhtpeer/benchmarks_test.go: QmX2KWsaFaVd9KGjvgYNgkLtgnu1CUhBPtTe9abKYndq4C libp2p_node/dht/dhtpeer/dhtpeer.go: QmUrp42Kty9Rt4p2hY3FutRMNKKdRXQGjGBoSd1YiSsAaH - libp2p_node/dht/dhtpeer/dhtpeer_test.go: QmYxZ8dkULMwjGpaUUtqtvQcE8WaUezrDhqtLkRTszQVJg + libp2p_node/dht/dhtpeer/dhtpeer_test.go: QmXySdeKevKC6jWweDX77ncGA1G5nyYNyfTgFB4S1y4kw8 libp2p_node/dht/dhtpeer/options.go: QmRbB1dnA5TEeJZQpKBJQoxFNHSPLRiEtwtkK6ZJDZdjAX libp2p_node/dht/dhttests/dhttests.go: QmWTwAqXy4xPBWx49dvg91pBfaeyh79bgbh4yYc3u6kGhg libp2p_node/dht/monitoring/file.go: QmfSw8prwiuxqcaTrEkWAL4dZYi7EzWGA4ZCgPgQ99ZnGN diff --git a/packages/fetchai/connections/p2p_libp2p/libp2p_node/dht/dhtpeer/dhtpeer_test.go b/packages/fetchai/connections/p2p_libp2p/libp2p_node/dht/dhtpeer/dhtpeer_test.go index ce82c63277..d7686e3e4d 100644 --- a/packages/fetchai/connections/p2p_libp2p/libp2p_node/dht/dhtpeer/dhtpeer_test.go +++ b/packages/fetchai/connections/p2p_libp2p/libp2p_node/dht/dhtpeer/dhtpeer_test.go @@ -856,6 +856,123 @@ func TestMessageOrderingWithDelegateClient(t *testing.T) { } +// TestMessageOrderingWithDelegateClientTwoHops +func TestMessageOrderingWithDelegateClientTwoHops(t *testing.T) { + peer1Index := 0 + peer2Index := 1 + client1Index := 2 + client2Index := 3 + peer1, peerCleanup1, err := SetupLocalDHTPeer( + FetchAITestKeys[peer1Index], + AgentsTestKeys[peer1Index], + DefaultLocalPort, + DefaultDelegatePort, + []string{}, + ) + if err != nil { + t.Fatal("Failed to initialize DHTPeer:", err) + } + defer peerCleanup1() + + peer2, peerCleanup2, err := SetupLocalDHTPeer( + FetchAITestKeys[peer2Index], + AgentsTestKeys[peer2Index], + DefaultLocalPort+1, + DefaultDelegatePort+1, + []string{peer1.MultiAddr()}, + ) + if err != nil { + t.Fatal("Failed to initialize DHTPeer:", err) + } + defer peerCleanup2() + + time.Sleep(1 * time.Second) + + client1, clientCleanup1, err := SetupDelegateClient( + AgentsTestKeys[client1Index], + DefaultLocalHost, + DefaultDelegatePort, + FetchAITestPublicKeys[peer1Index], + ) + if err != nil { + t.Fatal("Failed to initialize DelegateClient:", err) + } + defer clientCleanup1() + + client2, clientCleanup2, err := SetupDelegateClient( + AgentsTestKeys[client2Index], + DefaultLocalHost, + DefaultDelegatePort+1, + FetchAITestPublicKeys[peer2Index], + ) + if err != nil { + t.Fatal("Failed to initialize DelegateClient:", err) + } + defer clientCleanup2() + + rxClient1 := make(chan *aea.Envelope, 20) + client1.ProcessEnvelope(func(envel *aea.Envelope) error { + rxClient1 <- envel + return nil + }) + rxClient2 := make(chan *aea.Envelope, 20) + client2.ProcessEnvelope(func(envel *aea.Envelope) error { + rxClient2 <- envel + return nil + }) + + ensureAddressAnnounced(peer1, peer2) + + max := 100 + i := 0 + for x := 0; x < max; x++ { + envelope := &aea.Envelope{ + To: AgentsTestAddresses[client2Index], + Sender: AgentsTestAddresses[client1Index], + Message: []byte(strconv.Itoa(i)), + } + err = client1.Send(envelope) + if err != nil { + t.Error("Failed to Send envelope from DelegateClient to DHTPeer:", err) + } + i++ + t.Log("Sending Envelope : ", envelope) + // time.Sleep(100 * time.Millisecond) + + envelope1 := &aea.Envelope{ + To: AgentsTestAddresses[client1Index], + Sender: AgentsTestAddresses[client2Index], + Message: []byte(strconv.Itoa(i)), + } + err = client2.Send(envelope1) + if err != nil { + t.Error("Failed to Send envelope from DelegateClient to DHTPeer:", err) + } + i++ + t.Log("Sending Envelope : ", envelope1) + // time.Sleep(100 * time.Millisecond) + } + + // go func() { + ii := 0 + for x := 0; x < max; x++ { + expectEnvelopeOrdered(t, rxClient2, ii) + ii++ + ii++ + } + // }() + + // go func() { + iii := 0 + for x := 0; x < max; x++ { + iii++ + expectEnvelopeOrdered(t, rxClient1, iii) + iii++ + } + // }() + +} + // TestRoutingDelegateClientToDHTPeerIndirectTwoHops func TestRoutingDelegateClientToDHTPeerIndirectTwoHops(t *testing.T) { entryPeer, entryPeerCleanup, err := SetupLocalDHTPeer( diff --git a/packages/hashes.csv b/packages/hashes.csv index 3e51a49df7..d3cade1521 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -41,7 +41,7 @@ fetchai/connections/http_server,QmZqiszQJuG7XA6LFWsMqXYSmViTrUkDfpkHwgYtDMbyXy fetchai/connections/ledger,QmT7ffwPzJ3isCMhN2qoj6NRyqinE2RkpSpUKNRFRXxpes fetchai/connections/local,QmUxLhmeE98S8BcuRDB7W7pRsJzpC3wVJV5ELLxVeEkoKC fetchai/connections/oef,QmaHQhxryQLaBk5TvC4iJFZtFvkPp4CoHxHg1iLnh2PAdm -fetchai/connections/p2p_libp2p,QmPkAszJ5DmVVGJjaLu6VkH3jLfDN7PuL4NH8UTq4TKz2y +fetchai/connections/p2p_libp2p,QmexwMZJE362BsGKySY3Rx3Sph4SUd9wHiLR8evqUjAYbK fetchai/connections/p2p_libp2p_client,QmYq2U83xTTnv1WDy1quvEz47QaV2uw1ZmB6sFHoCL31GC fetchai/connections/p2p_stub,QmToCExj3ZpdxUu3vYSMo4jZRJiMTkXMbyhbh18Hq6Nc4b fetchai/connections/prometheus,QmeRmgrccxRh8UG4AfTMct6D5wSuK9BXYoLwnohut2LJaB From d8a7499adea76ee2b68bf09eeb8b2065ee346062 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 3 May 2021 19:02:24 +0200 Subject: [PATCH 08/25] add (failing) test for peer-to-peer message ordering the test does not currently pass due to a bug in the Go implementation. --- .../test_p2p_libp2p/test_communication.py | 75 +++++++++++++++++-- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_communication.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_communication.py index 82d926d74a..7090828308 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_communication.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_communication.py @@ -29,10 +29,11 @@ from aea_ledger_fetchai import FetchAICrypto from aea.crypto.registries import make_crypto -from aea.mail.base import Envelope +from aea.mail.base import Empty, Envelope from aea.multiplexer import Multiplexer from packages.fetchai.connections.p2p_libp2p.connection import NodeClient, Uri +from packages.fetchai.protocols.default import DefaultSerializer from packages.fetchai.protocols.default.message import DefaultMessage from tests.conftest import ( @@ -671,8 +672,31 @@ def teardown(self): @libp2p_log_on_failure_all -class BaseTestP2PLibp2pReconnection: - """Base test class for reconnection tests.""" +class BaseTestP2PLibp2p: + """Base test class for p2p libp2p tests with two peers.""" + + def _make_envelope( + self, + sender_address: str, + receiver_address: str, + message_id: int = 1, + target: int = 0, + ): + """Make an envelope for testing purposes.""" + msg = DefaultMessage( + dialogue_reference=("", ""), + message_id=message_id, + target=target, + performative=DefaultMessage.Performative.BYTES, + content=b"hello", + ) + envelope = Envelope( + to=receiver_address, + sender=sender_address, + protocol_specification_id=DefaultMessage.protocol_specification_id, + message=DefaultSerializer().encode(msg), + ) + return envelope @classmethod @libp2p_log_on_failure @@ -734,7 +758,7 @@ def teardown_class(cls): @libp2p_log_on_failure_all -class TestP2PLibp2pReconnectionSendEnvelope(BaseTestP2PLibp2pReconnection): +class TestP2PLibp2PSendEnvelope(BaseTestP2PLibp2p): """Test that connection will send envelope with error, and that reconnection fixes it.""" def test_connection_is_established(self): @@ -784,7 +808,7 @@ def test_envelope_routed(self): @libp2p_log_on_failure_all -class TestP2PLibp2pReconnectionReceiveEnvelope(BaseTestP2PLibp2pReconnection): +class TestP2PLibp2PReceiveEnvelope(BaseTestP2PLibp2p): """Test that connection will receive envelope with error, and that reconnection fixes it.""" def test_envelope_routed(self): @@ -827,6 +851,47 @@ def test_envelope_routed(self): assert envelope.message == msg +class TestLibp2pEnvelopeOrder(BaseTestP2PLibp2p): + """ + Test message ordering. + + Test that the order of envelope is the guaranteed to be the same + when communicating between two peers. + """ + + NB_ENVELOPES = 1000 + + def test_burst_order(self): + """Test order of envelope burst is guaranteed on receiving end.""" + addr_1 = self.connection1.address + addr_2 = self.connection2.address + + sent_envelopes = [ + self._make_envelope(addr_1, addr_2, i, i - 1) + for i in range(1, self.NB_ENVELOPES + 1) + ] + for envelope in sent_envelopes: + self.multiplexer1.put(envelope) + + received_envelopes = [] + for _ in range(1, self.NB_ENVELOPES + 1): + envelope = self.multiplexer2.get(block=True, timeout=20) + received_envelopes.append(envelope) + + # test no new message is "created" + with pytest.raises(Empty): + self.multiplexer2.get(block=True, timeout=1) + + assert len(sent_envelopes) == len( + received_envelopes + ), f"expected number of envelopes {len(sent_envelopes)}, got {len(received_envelopes)}" + for expected, actual in zip(sent_envelopes, received_envelopes): + assert expected.message == actual.message, ( + "message content differ; probably a wrong message " + "ordering on the receiving end" + ) + + @pytest.mark.asyncio async def test_nodeclient_pipe_connect(): """Test pipe.connect called on NodeClient.connect.""" From e8236175e2c27aba9ad8f05196d7cca0b8155e42 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 3 May 2021 19:12:58 +0200 Subject: [PATCH 09/25] test: update twin go lib test module --- .../libp2p_node/dht/dhtpeer/dhtpeer_test.go | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/libs/go/libp2p_node/dht/dhtpeer/dhtpeer_test.go b/libs/go/libp2p_node/dht/dhtpeer/dhtpeer_test.go index ce82c63277..d7686e3e4d 100644 --- a/libs/go/libp2p_node/dht/dhtpeer/dhtpeer_test.go +++ b/libs/go/libp2p_node/dht/dhtpeer/dhtpeer_test.go @@ -856,6 +856,123 @@ func TestMessageOrderingWithDelegateClient(t *testing.T) { } +// TestMessageOrderingWithDelegateClientTwoHops +func TestMessageOrderingWithDelegateClientTwoHops(t *testing.T) { + peer1Index := 0 + peer2Index := 1 + client1Index := 2 + client2Index := 3 + peer1, peerCleanup1, err := SetupLocalDHTPeer( + FetchAITestKeys[peer1Index], + AgentsTestKeys[peer1Index], + DefaultLocalPort, + DefaultDelegatePort, + []string{}, + ) + if err != nil { + t.Fatal("Failed to initialize DHTPeer:", err) + } + defer peerCleanup1() + + peer2, peerCleanup2, err := SetupLocalDHTPeer( + FetchAITestKeys[peer2Index], + AgentsTestKeys[peer2Index], + DefaultLocalPort+1, + DefaultDelegatePort+1, + []string{peer1.MultiAddr()}, + ) + if err != nil { + t.Fatal("Failed to initialize DHTPeer:", err) + } + defer peerCleanup2() + + time.Sleep(1 * time.Second) + + client1, clientCleanup1, err := SetupDelegateClient( + AgentsTestKeys[client1Index], + DefaultLocalHost, + DefaultDelegatePort, + FetchAITestPublicKeys[peer1Index], + ) + if err != nil { + t.Fatal("Failed to initialize DelegateClient:", err) + } + defer clientCleanup1() + + client2, clientCleanup2, err := SetupDelegateClient( + AgentsTestKeys[client2Index], + DefaultLocalHost, + DefaultDelegatePort+1, + FetchAITestPublicKeys[peer2Index], + ) + if err != nil { + t.Fatal("Failed to initialize DelegateClient:", err) + } + defer clientCleanup2() + + rxClient1 := make(chan *aea.Envelope, 20) + client1.ProcessEnvelope(func(envel *aea.Envelope) error { + rxClient1 <- envel + return nil + }) + rxClient2 := make(chan *aea.Envelope, 20) + client2.ProcessEnvelope(func(envel *aea.Envelope) error { + rxClient2 <- envel + return nil + }) + + ensureAddressAnnounced(peer1, peer2) + + max := 100 + i := 0 + for x := 0; x < max; x++ { + envelope := &aea.Envelope{ + To: AgentsTestAddresses[client2Index], + Sender: AgentsTestAddresses[client1Index], + Message: []byte(strconv.Itoa(i)), + } + err = client1.Send(envelope) + if err != nil { + t.Error("Failed to Send envelope from DelegateClient to DHTPeer:", err) + } + i++ + t.Log("Sending Envelope : ", envelope) + // time.Sleep(100 * time.Millisecond) + + envelope1 := &aea.Envelope{ + To: AgentsTestAddresses[client1Index], + Sender: AgentsTestAddresses[client2Index], + Message: []byte(strconv.Itoa(i)), + } + err = client2.Send(envelope1) + if err != nil { + t.Error("Failed to Send envelope from DelegateClient to DHTPeer:", err) + } + i++ + t.Log("Sending Envelope : ", envelope1) + // time.Sleep(100 * time.Millisecond) + } + + // go func() { + ii := 0 + for x := 0; x < max; x++ { + expectEnvelopeOrdered(t, rxClient2, ii) + ii++ + ii++ + } + // }() + + // go func() { + iii := 0 + for x := 0; x < max; x++ { + iii++ + expectEnvelopeOrdered(t, rxClient1, iii) + iii++ + } + // }() + +} + // TestRoutingDelegateClientToDHTPeerIndirectTwoHops func TestRoutingDelegateClientToDHTPeerIndirectTwoHops(t *testing.T) { entryPeer, entryPeerCleanup, err := SetupLocalDHTPeer( From 64a402f09c169c49f225a7caeff3c5a5b1495bf4 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 3 May 2021 19:50:27 +0200 Subject: [PATCH 10/25] fix message ordering bug in libp2p_node (all credit goes to @DavidMinarsch !) --- packages/fetchai/connections/p2p_libp2p/connection.yaml | 2 +- .../connections/p2p_libp2p/libp2p_node/libp2p_node.go | 6 ++---- packages/hashes.csv | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 80109c114b..732f157728 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -36,7 +36,7 @@ fingerprint: libp2p_node/dht/monitoring/service.go: QmT47y2LHZECYcoE2uJ9QCGh3Kq8ePhYedo8dQE7X7v6YV libp2p_node/go.mod: QmT2LQXkSen6Th4QLS2RU3tRVxJSLjdZTxRULUcnQFXfEL libp2p_node/go.sum: QmZc6oiDqQhB6ZKVojN26JxoTNH1QetyuYd9zjSksQnTFt - libp2p_node/libp2p_node.go: QmZFR2bdi1EjFmPkjwtYeA19kSQKPtcGoNVafQnZRWotaN + libp2p_node/libp2p_node.go: QmVgVVxGDKsycgfTFu7TPPoXfgwEA2anh7ye89Kz1rgMXq libp2p_node/utils/utils.go: QmawKkyn8nZtmgULM7vYWfHqMD4o49TQZGF4arwVxyYeCL fingerprint_ignore_patterns: [] build_entrypoint: check_dependencies.py diff --git a/packages/fetchai/connections/p2p_libp2p/libp2p_node/libp2p_node.go b/packages/fetchai/connections/p2p_libp2p/libp2p_node/libp2p_node.go index 44abe9d98a..3c0a12b681 100644 --- a/packages/fetchai/connections/p2p_libp2p/libp2p_node/libp2p_node.go +++ b/packages/fetchai/connections/p2p_libp2p/libp2p_node/libp2p_node.go @@ -157,10 +157,8 @@ func main() { for envel := range agent.Queue() { envelope := envel logger.Info().Msgf("received envelope from agent: %s", envelope) - go func() { - err := node.RouteEnvelope(envelope) - ignore(err) - }() + err := node.RouteEnvelope(envelope) + ignore(err) } }() diff --git a/packages/hashes.csv b/packages/hashes.csv index d3cade1521..8129b066e3 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -41,7 +41,7 @@ fetchai/connections/http_server,QmZqiszQJuG7XA6LFWsMqXYSmViTrUkDfpkHwgYtDMbyXy fetchai/connections/ledger,QmT7ffwPzJ3isCMhN2qoj6NRyqinE2RkpSpUKNRFRXxpes fetchai/connections/local,QmUxLhmeE98S8BcuRDB7W7pRsJzpC3wVJV5ELLxVeEkoKC fetchai/connections/oef,QmaHQhxryQLaBk5TvC4iJFZtFvkPp4CoHxHg1iLnh2PAdm -fetchai/connections/p2p_libp2p,QmexwMZJE362BsGKySY3Rx3Sph4SUd9wHiLR8evqUjAYbK +fetchai/connections/p2p_libp2p,QmfJnTnQ6MGqVu8fFBnJ7KfbQ33FbxYHnVEp3wj5pHxqtz fetchai/connections/p2p_libp2p_client,QmYq2U83xTTnv1WDy1quvEz47QaV2uw1ZmB6sFHoCL31GC fetchai/connections/p2p_stub,QmToCExj3ZpdxUu3vYSMo4jZRJiMTkXMbyhbh18Hq6Nc4b fetchai/connections/prometheus,QmeRmgrccxRh8UG4AfTMct6D5wSuK9BXYoLwnohut2LJaB From 5abaaa199cf058e73c269e8c8b89504a18faf3cd Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Mon, 3 May 2021 19:53:27 +0200 Subject: [PATCH 11/25] update twin module 'libp2p_node.go' --- libs/go/libp2p_node/libp2p_node.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/libs/go/libp2p_node/libp2p_node.go b/libs/go/libp2p_node/libp2p_node.go index 44abe9d98a..3c0a12b681 100644 --- a/libs/go/libp2p_node/libp2p_node.go +++ b/libs/go/libp2p_node/libp2p_node.go @@ -157,10 +157,8 @@ func main() { for envel := range agent.Queue() { envelope := envel logger.Info().Msgf("received envelope from agent: %s", envelope) - go func() { - err := node.RouteEnvelope(envelope) - ignore(err) - }() + err := node.RouteEnvelope(envelope) + ignore(err) } }() From 2bac010df231ae44b6e1b2e242d22aa46c693d5d Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 4 May 2021 13:08:37 +0200 Subject: [PATCH 12/25] fix: address codeql warning in p2p_libp2p library The warning ID was 'go/allocation-size-overflow'. We solved it by checking the size of the message before assigning it to an unsigned 32 integer. --- libs/go/libp2p_node/aea/pipe_unix.go | 4 ++++ libs/go/libp2p_node/aea/pipe_windows.go | 5 +++++ packages/fetchai/connections/p2p_libp2p/connection.yaml | 4 ++-- .../connections/p2p_libp2p/libp2p_node/aea/pipe_unix.go | 4 ++++ .../connections/p2p_libp2p/libp2p_node/aea/pipe_windows.go | 5 +++++ packages/hashes.csv | 2 +- 6 files changed, 21 insertions(+), 3 deletions(-) diff --git a/libs/go/libp2p_node/aea/pipe_unix.go b/libs/go/libp2p_node/aea/pipe_unix.go index 8e5082f278..9b03e165da 100644 --- a/libs/go/libp2p_node/aea/pipe_unix.go +++ b/libs/go/libp2p_node/aea/pipe_unix.go @@ -25,6 +25,7 @@ package aea import ( "encoding/binary" "errors" + "math" "os" ) @@ -66,6 +67,9 @@ func (pipe *UnixPipe) Read() ([]byte, error) { } func (pipe *UnixPipe) Write(data []byte) error { + if len(data) > math.MaxInt32 { + return errors.New("value too large") + } size := uint32(len(data)) buf := make([]byte, 4, 4+size) binary.BigEndian.PutUint32(buf, size) diff --git a/libs/go/libp2p_node/aea/pipe_windows.go b/libs/go/libp2p_node/aea/pipe_windows.go index b7e9591bfa..54efa060c7 100644 --- a/libs/go/libp2p_node/aea/pipe_windows.go +++ b/libs/go/libp2p_node/aea/pipe_windows.go @@ -24,6 +24,8 @@ package aea import ( "encoding/binary" + "errors" + "math" "net" "strconv" ) @@ -63,6 +65,9 @@ func (sock *TCPSocketChannel) Read() ([]byte, error) { func (sock *TCPSocketChannel) Write(data []byte) error { // TOFIX(LR) duplicated code to avoid circular dep // utils.WriteBytesConn(sock.conn, data) + if len(data) > math.MaxInt32 { + return errors.New("value too large") + } size := uint32(len(data)) buf := make([]byte, 4, 4+size) binary.BigEndian.PutUint32(buf, size) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 732f157728..382eb486b7 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -16,8 +16,8 @@ fingerprint: libp2p_node/aea/api.go: QmdFR5Rmkk2FGVDwzgHtjobjAKyLejqk2CAYBCvcF23AG7 libp2p_node/aea/envelope.pb.go: QmRfUNGpCeVJfsW3H1MzCN4pwDWgumfyWufVFp6xvUjjug libp2p_node/aea/envelope.proto: QmVuvesmfgzj5aKnbFoCocoGEv3T9MR7u6KWn7CT5yfjGi - libp2p_node/aea/pipe_unix.go: QmSzbAexrSUSA9ZE6VT6MmcWF5MhEcmceQjAbnfRoK7Ruv - libp2p_node/aea/pipe_windows.go: QmdYpfjgdgFbA6Pothg65NYM45ubfpZeKr8cVQoqbFzgUK + libp2p_node/aea/pipe_unix.go: QmYC7pExTkBBNHnbBeyd4HKmj9fDTnouS4apbwZoGXNWJz + libp2p_node/aea/pipe_windows.go: QmNcuPLMsbdWqUn1SFx2V2RTkujjtvBfYQmavz6QiZx6av libp2p_node/dht/dhtclient/dhtclient.go: QmZyZiFp29cSyQLiCurJhv6wfBaTJYkTgpY7tbooKt4N7F libp2p_node/dht/dhtclient/dhtclient_test.go: QmS9SiLAojXYobqV9m5yeRpyPzt6JcSbQD78RNvSp6LPx6 libp2p_node/dht/dhtclient/options.go: QmaoJiavHescgx98inQjVUipPFRvcFFrodrScZ3fvrXJ4z diff --git a/packages/fetchai/connections/p2p_libp2p/libp2p_node/aea/pipe_unix.go b/packages/fetchai/connections/p2p_libp2p/libp2p_node/aea/pipe_unix.go index 8e5082f278..9b03e165da 100644 --- a/packages/fetchai/connections/p2p_libp2p/libp2p_node/aea/pipe_unix.go +++ b/packages/fetchai/connections/p2p_libp2p/libp2p_node/aea/pipe_unix.go @@ -25,6 +25,7 @@ package aea import ( "encoding/binary" "errors" + "math" "os" ) @@ -66,6 +67,9 @@ func (pipe *UnixPipe) Read() ([]byte, error) { } func (pipe *UnixPipe) Write(data []byte) error { + if len(data) > math.MaxInt32 { + return errors.New("value too large") + } size := uint32(len(data)) buf := make([]byte, 4, 4+size) binary.BigEndian.PutUint32(buf, size) diff --git a/packages/fetchai/connections/p2p_libp2p/libp2p_node/aea/pipe_windows.go b/packages/fetchai/connections/p2p_libp2p/libp2p_node/aea/pipe_windows.go index b7e9591bfa..54efa060c7 100644 --- a/packages/fetchai/connections/p2p_libp2p/libp2p_node/aea/pipe_windows.go +++ b/packages/fetchai/connections/p2p_libp2p/libp2p_node/aea/pipe_windows.go @@ -24,6 +24,8 @@ package aea import ( "encoding/binary" + "errors" + "math" "net" "strconv" ) @@ -63,6 +65,9 @@ func (sock *TCPSocketChannel) Read() ([]byte, error) { func (sock *TCPSocketChannel) Write(data []byte) error { // TOFIX(LR) duplicated code to avoid circular dep // utils.WriteBytesConn(sock.conn, data) + if len(data) > math.MaxInt32 { + return errors.New("value too large") + } size := uint32(len(data)) buf := make([]byte, 4, 4+size) binary.BigEndian.PutUint32(buf, size) diff --git a/packages/hashes.csv b/packages/hashes.csv index 8129b066e3..4648159904 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -41,7 +41,7 @@ fetchai/connections/http_server,QmZqiszQJuG7XA6LFWsMqXYSmViTrUkDfpkHwgYtDMbyXy fetchai/connections/ledger,QmT7ffwPzJ3isCMhN2qoj6NRyqinE2RkpSpUKNRFRXxpes fetchai/connections/local,QmUxLhmeE98S8BcuRDB7W7pRsJzpC3wVJV5ELLxVeEkoKC fetchai/connections/oef,QmaHQhxryQLaBk5TvC4iJFZtFvkPp4CoHxHg1iLnh2PAdm -fetchai/connections/p2p_libp2p,QmfJnTnQ6MGqVu8fFBnJ7KfbQ33FbxYHnVEp3wj5pHxqtz +fetchai/connections/p2p_libp2p,QmPny2KMEwNa1QuvPhW44u8NSE5TfmLHNmoKNe7TefUnzi fetchai/connections/p2p_libp2p_client,QmYq2U83xTTnv1WDy1quvEz47QaV2uw1ZmB6sFHoCL31GC fetchai/connections/p2p_stub,QmToCExj3ZpdxUu3vYSMo4jZRJiMTkXMbyhbh18Hq6Nc4b fetchai/connections/prometheus,QmeRmgrccxRh8UG4AfTMct6D5wSuK9BXYoLwnohut2LJaB From 0ec11d73b4d4c9a965d1f295078931dfe6a4bc17 Mon Sep 17 00:00:00 2001 From: MarcoFavorito Date: Tue, 4 May 2021 15:16:31 +0200 Subject: [PATCH 13/25] test: update p2p_libp2p test add decorator 'libp2p_log_on_failure_all' to TestLibp2pEnvelopeOrder class --- .../test_connections/test_p2p_libp2p/test_communication.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_communication.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_communication.py index 7090828308..593647bc4e 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_communication.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_communication.py @@ -851,6 +851,7 @@ def test_envelope_routed(self): assert envelope.message == msg +@libp2p_log_on_failure_all class TestLibp2pEnvelopeOrder(BaseTestP2PLibp2p): """ Test message ordering. From eb0e178aba88da016626aa1c98424e03457c54d5 Mon Sep 17 00:00:00 2001 From: Amit Solanki Date: Tue, 4 May 2021 23:00:19 +0530 Subject: [PATCH 14/25] Added steps for prod deployment --- .github/workflows/publish.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 32e1d497ee..db5011679d 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -4,6 +4,7 @@ on: push: branches: - develop + - main jobs: publish: @@ -21,6 +22,13 @@ jobs: project_id: ${{ secrets.GCLOUD_FETCH_AI_SANDBOX_PROJECT }} service_account_key: ${{ secrets.GCLOUD_FETCH_AI_SANDBOX_KEY }} + - name: Setup GCloud - production + uses: GoogleCloudPlatform/github-actions/setup-gcloud@master + if: github.ref == 'refs/heads/main' + with: + project_id: ${{ secrets.GCLOUD_FETCH_AI_PROD_PROJECT }} + service_account_key: ${{ secrets.GCLOUD_FETCH_AI_PROD_KEY }} + - name: Configure Docker run: | gcloud auth configure-docker @@ -39,6 +47,11 @@ jobs: ./scripts/acn/build_upload_img.sh fi + if [ ${{ github.ref }} == 'refs/heads/main' ] + then + ./scripts/acn/build_upload_img.sh prod + fi + - name: Repository Dispatch env: IMAGE_TAG: ${{ steps.vars.outputs.sha_short }} @@ -51,3 +64,12 @@ jobs: --data '{"event_type": "agents-dht-testnet", "client_payload": {"image": "gcr.io/fetch-ai-sandbox/acn_node", "tag": "'"$IMAGE_TAG"'"}}' \ https://api.github.com/repos/fetchai/infra-sandbox-london-b-deployment/dispatches fi + + if [ ${{ github.ref }} == 'refs/heads/main' ] + then + curl -H "Accept: application/vnd.github.everest-preview+json" \ + -H "Authorization: token ${{ secrets.GH_PAT }}" \ + --request POST \ + --data '{"event_type": "agents-dht", "client_payload": {"image": "gcr.io/fetch-ai-images/acn_node", "tag": "'"$IMAGE_TAG"'"}}' \ + https://api.github.com/repos/fetchai/infra-mainnet-v2-deployment/dispatches + fi From 76c6c76029e773cf5560aa2d3d0a98185fbc6301 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Tue, 4 May 2021 15:19:46 +0300 Subject: [PATCH 15/25] libp2p utils writebytes better error handling --- aea/helpers/base.py | 6 +-- examples/tac_deploy/Dockerfile | 1 + .../go/libp2p_node/dht/dhtclient/dhtclient.go | 2 + libs/go/libp2p_node/libp2p_node.go | 11 ++---- libs/go/libp2p_node/utils/utils.go | 38 ++++++++++++++++-- .../connections/p2p_libp2p/connection.py | 39 ++++++++++++++++++- .../connections/p2p_libp2p/connection.yaml | 8 ++-- .../libp2p_node/dht/dhtclient/dhtclient.go | 2 + .../p2p_libp2p/libp2p_node/libp2p_node.go | 11 ++---- .../p2p_libp2p/libp2p_node/utils/utils.go | 38 ++++++++++++++++-- packages/hashes.csv | 2 +- scripts/check_imports_and_dependencies.py | 12 +++--- .../test_p2p_libp2p/test_errors.py | 30 ++++++++++++++ 13 files changed, 162 insertions(+), 38 deletions(-) diff --git a/aea/helpers/base.py b/aea/helpers/base.py index d6ba4dcd30..83881bb1d8 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -154,7 +154,7 @@ def sigint_crossplatform(process: subprocess.Popen) -> None: # pragma: nocover if os.name == "posix": process.send_signal(signal.SIGINT) # pylint: disable=no-member elif os.name == "nt": - process.send_signal(signal.CTRL_C_EVENT) # pylint: disable=no-member + process.send_signal(signal.CTRL_C_EVENT) # type: ignore # pylint: disable=no-member else: raise ValueError("Other platforms not supported.") @@ -190,7 +190,7 @@ def send_control_c( if platform.system() == "Windows": if process.stdin: # cause ctrl-c event will be handled with stdin process.stdin.close() - os.kill(process.pid, signal.CTRL_C_EVENT) # pylint: disable=no-member + os.kill(process.pid, signal.CTRL_C_EVENT) # type: ignore # pylint: disable=no-member elif kill_group: pgid = os.getpgid(process.pid) os.killpg(pgid, signal.SIGINT) @@ -488,7 +488,7 @@ def find_topological_order(adjacency_list: Dict[T, Set[T]]) -> List[T]: # compute the topological order queue: Deque[T] = deque() order = [] - queue.extendleft(sorted(roots)) + queue.extendleft(sorted(roots)) # type: ignore while len(queue) > 0: current = queue.pop() order.append(current) diff --git a/examples/tac_deploy/Dockerfile b/examples/tac_deploy/Dockerfile index adf8632d15..6bb96e7290 100644 --- a/examples/tac_deploy/Dockerfile +++ b/examples/tac_deploy/Dockerfile @@ -20,6 +20,7 @@ RUN pip install --upgrade --force-reinstall aea[all]==1.0.0 # directories and aea cli config COPY /.aea /home/.aea + WORKDIR /home/agents COPY ./packages /home/agents/packages diff --git a/libs/go/libp2p_node/dht/dhtclient/dhtclient.go b/libs/go/libp2p_node/dht/dhtclient/dhtclient.go index 1bd3d623f6..e627a21684 100644 --- a/libs/go/libp2p_node/dht/dhtclient/dhtclient.go +++ b/libs/go/libp2p_node/dht/dhtclient/dhtclient.go @@ -646,6 +646,8 @@ func (dhtClient *DHTClient) RouteEnvelope(envel *aea.Envelope) error { Str("target", target). Msg("couldn't send envelope") errReset := stream.Reset() + lerror(errReset). + Msg("stream.Reset error") ignore(errReset) return err } diff --git a/libs/go/libp2p_node/libp2p_node.go b/libs/go/libp2p_node/libp2p_node.go index 3c0a12b681..c92151c514 100644 --- a/libs/go/libp2p_node/libp2p_node.go +++ b/libs/go/libp2p_node/libp2p_node.go @@ -37,7 +37,6 @@ import ( const ( libp2pNodePanicError = "LIBP2P_NODE_PANIC_ERROR" - libp2pNodeIgnored = "IGNORED" libp2pMultiaddrsListStart = "MULTIADDRS_LIST_START" libp2pMultiaddrsListEnd = "MULTIADDRS_LIST_END" ) @@ -52,12 +51,6 @@ func check(err error) { } } -func ignore(err error) { - if err != nil { - fmt.Println(libp2pNodeIgnored, ":", err) - } -} - func main() { var err error @@ -158,7 +151,9 @@ func main() { envelope := envel logger.Info().Msgf("received envelope from agent: %s", envelope) err := node.RouteEnvelope(envelope) - ignore(err) + if err != nil { + logger.Error().Msgf("Route envelope error: %s", err.Error()) + } } }() diff --git a/libs/go/libp2p_node/utils/utils.go b/libs/go/libp2p_node/utils/utils.go index 66f7424893..c5cb0d3719 100644 --- a/libs/go/libp2p_node/utils/utils.go +++ b/libs/go/libp2p_node/utils/utils.go @@ -30,6 +30,7 @@ import ( "errors" "fmt" "io" + "math" "net" "os" "sync" @@ -612,6 +613,14 @@ func FetchAIPublicKeyFromFetchAIPrivateKey(privateKey string) (string, error) { // WriteBytesConn send bytes to `conn` func WriteBytesConn(conn net.Conn, data []byte) error { + if len(data) > math.MaxInt32 { + logger.Error().Msg("data size too large") + return errors.New("data size too large") + } + if len(data) == 0 { + logger.Error().Msg("No data to write") + return nil + } size := uint32(len(data)) buf := make([]byte, 4, 4+size) binary.BigEndian.PutUint32(buf, size) @@ -685,6 +694,15 @@ func ReadBytes(s network.Stream) ([]byte, error) { // WriteBytes to a network stream func WriteBytes(s network.Stream, data []byte) error { + if len(data) > math.MaxInt32 { + logger.Error().Msg("data size too large") + return errors.New("data size too large") + } + if len(data) == 0 { + logger.Error().Msg("No data to write") + return nil + } + wstream := bufio.NewWriter(s) size := uint32(len(data)) @@ -699,9 +717,20 @@ func WriteBytes(s network.Stream, data []byte) error { return err } - //logger.Debug().Msgf("writing %d", len(data)) _, err = wstream.Write(data) - wstream.Flush() + if err != nil { + logger.Error(). + Str("err", err.Error()). + Msg("Error on data write") + return err + } + err = wstream.Flush() + if err != nil { + logger.Error(). + Str("err", err.Error()). + Msg("Error on stream flush") + return err + } return err } @@ -734,7 +763,10 @@ func WriteEnvelope(envel *aea.Envelope, s network.Stream) error { return err } - wstream.Flush() + err = wstream.Flush() + if err != nil { + return err + } return nil } diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index 83ea39917a..2f7419b5d0 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -22,7 +22,7 @@ import os import platform import subprocess # nosec -from asyncio import AbstractEventLoop, CancelledError +from asyncio import AbstractEventLoop, CancelledError, events from ipaddress import ip_address from pathlib import Path from socket import gethostbyname @@ -106,7 +106,6 @@ def _golang_module_run( cmd.extend(args) env = os.environ - try: logger.debug(cmd) proc = subprocess.Popen( # nosec @@ -272,6 +271,7 @@ def __init__( ) self._max_restarts = max_restarts self._restart_counter: int = 0 + self._is_on_stop: bool = False def _make_env_file(self, pipe_in_path: str, pipe_out_path: str) -> str: # setup config @@ -337,12 +337,25 @@ def get_client(self) -> NodeClient: return NodeClient(self.pipe) + def _child_watcher_callback(self, *_) -> None: # type:ignore + """Log if process was terminated before stop was called.""" + if self._is_on_stop: + return + if self.proc is None: + return + self.proc.poll() + returncode = self.proc.returncode + self.logger.error( + f"Node process with pid {self.proc.pid} was terminated with returncode {returncode}" + ) + async def start(self) -> None: """ Start the node. :return: None """ + self._is_on_stop = False if self._loop is None: self._loop = asyncio.get_event_loop() @@ -362,10 +375,28 @@ async def start(self) -> None: self.source, LIBP2P_NODE_MODULE_NAME, [self.env_file], self._log_file_desc ) + if platform.system() == "Windows": + try: + asyncio.get_event_loop()._proactor.wait_for_handle( # type: ignore # pylint: disable=protected-access + int(self.proc._handle) # type: ignore # pylint: disable=no-member,protected-access + ).add_done_callback( + self._child_watcher_callback + ) + except Exception as e: # pragma: nocover # pylint:disable=broad-except + self.logger.debug(f"failed to set process monitoring on windows: {e}") + else: + with events.get_child_watcher() as watcher: + if watcher: + watcher.attach_loop(asyncio.get_event_loop()) + watcher.add_child_handler( + self.proc.pid, self._child_watcher_callback + ) + self.logger.info("Connecting to libp2p node...") try: connected = await self._set_connection_to_node() + if not connected: raise Exception("Couldn't connect to libp2p process within timeout") except Exception as e: @@ -482,6 +513,8 @@ async def stop(self) -> None: """ if self.proc is not None: self.logger.debug("Terminating node process {}...".format(self.proc.pid)) + self._is_on_stop = True + self.proc.poll() self.proc.terminate() self.logger.debug( "Waiting for node process {} to terminate...".format(self.proc.pid) @@ -492,6 +525,7 @@ async def stop(self) -> None: self._log_file_desc.close() else: self.logger.debug("Called stop when process not set!") # pragma: no cover + if self.pipe is not None: try: await self.pipe.close() @@ -500,6 +534,7 @@ async def stop(self) -> None: self.pipe = None else: self.logger.debug("Called stop when pipe not set!") # pragma: no cover + if os.path.exists(self.env_file): os.remove(self.env_file) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index 382eb486b7..ee63a5b13f 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -11,14 +11,14 @@ fingerprint: README.md: QmVRxZcTquf2m7j5ZpvCogKRuGWiHnLxBPx55CpyTxWdey __init__.py: QmYQuLNyQ8WTjgRYAoKAzoJEb7ocKXvM2hTyK4hsGch5D6 check_dependencies.py: QmP14nkQ8senwzdPdrZJLsA6EQ7zaKKEaLGDELhT42gp1P - connection.py: Qmb1o2qD6jgpqarBwuMYDqvySETUyc1MyxHHcRv7SPHuwc + connection.py: QmZTvLjt1EkGkszXpRVGtVzfGSweFxqyjZcmstQ2GiKRsC libp2p_node/README.md: QmSNJEQQwh25QSHgQPM4C84xKU6FczRpmJdN7HkCQyDPuC libp2p_node/aea/api.go: QmdFR5Rmkk2FGVDwzgHtjobjAKyLejqk2CAYBCvcF23AG7 libp2p_node/aea/envelope.pb.go: QmRfUNGpCeVJfsW3H1MzCN4pwDWgumfyWufVFp6xvUjjug libp2p_node/aea/envelope.proto: QmVuvesmfgzj5aKnbFoCocoGEv3T9MR7u6KWn7CT5yfjGi libp2p_node/aea/pipe_unix.go: QmYC7pExTkBBNHnbBeyd4HKmj9fDTnouS4apbwZoGXNWJz libp2p_node/aea/pipe_windows.go: QmNcuPLMsbdWqUn1SFx2V2RTkujjtvBfYQmavz6QiZx6av - libp2p_node/dht/dhtclient/dhtclient.go: QmZyZiFp29cSyQLiCurJhv6wfBaTJYkTgpY7tbooKt4N7F + libp2p_node/dht/dhtclient/dhtclient.go: QmYFj4YghhVA7E976xLNQHRTGrik1uHLc6ZQ9Tw1p9ECzk libp2p_node/dht/dhtclient/dhtclient_test.go: QmS9SiLAojXYobqV9m5yeRpyPzt6JcSbQD78RNvSp6LPx6 libp2p_node/dht/dhtclient/options.go: QmaoJiavHescgx98inQjVUipPFRvcFFrodrScZ3fvrXJ4z libp2p_node/dht/dhtnode/dhtnode.go: QmbyhgbCSAbQ1QsDw7FM7Nt5sZcvhbupA1jv5faxutbV7N @@ -36,8 +36,8 @@ fingerprint: libp2p_node/dht/monitoring/service.go: QmT47y2LHZECYcoE2uJ9QCGh3Kq8ePhYedo8dQE7X7v6YV libp2p_node/go.mod: QmT2LQXkSen6Th4QLS2RU3tRVxJSLjdZTxRULUcnQFXfEL libp2p_node/go.sum: QmZc6oiDqQhB6ZKVojN26JxoTNH1QetyuYd9zjSksQnTFt - libp2p_node/libp2p_node.go: QmVgVVxGDKsycgfTFu7TPPoXfgwEA2anh7ye89Kz1rgMXq - libp2p_node/utils/utils.go: QmawKkyn8nZtmgULM7vYWfHqMD4o49TQZGF4arwVxyYeCL + libp2p_node/libp2p_node.go: QmSu21WBmAAfBbZFi3MZZgTrUMK5FYruSunxghpeUhxBMA + libp2p_node/utils/utils.go: QmZQAY5CMRsqK36Zgy4ukgGzu7W5EVCQJGFhf6gJ9XCYNg fingerprint_ignore_patterns: [] build_entrypoint: check_dependencies.py connections: [] diff --git a/packages/fetchai/connections/p2p_libp2p/libp2p_node/dht/dhtclient/dhtclient.go b/packages/fetchai/connections/p2p_libp2p/libp2p_node/dht/dhtclient/dhtclient.go index 1bd3d623f6..e627a21684 100644 --- a/packages/fetchai/connections/p2p_libp2p/libp2p_node/dht/dhtclient/dhtclient.go +++ b/packages/fetchai/connections/p2p_libp2p/libp2p_node/dht/dhtclient/dhtclient.go @@ -646,6 +646,8 @@ func (dhtClient *DHTClient) RouteEnvelope(envel *aea.Envelope) error { Str("target", target). Msg("couldn't send envelope") errReset := stream.Reset() + lerror(errReset). + Msg("stream.Reset error") ignore(errReset) return err } diff --git a/packages/fetchai/connections/p2p_libp2p/libp2p_node/libp2p_node.go b/packages/fetchai/connections/p2p_libp2p/libp2p_node/libp2p_node.go index 3c0a12b681..c92151c514 100644 --- a/packages/fetchai/connections/p2p_libp2p/libp2p_node/libp2p_node.go +++ b/packages/fetchai/connections/p2p_libp2p/libp2p_node/libp2p_node.go @@ -37,7 +37,6 @@ import ( const ( libp2pNodePanicError = "LIBP2P_NODE_PANIC_ERROR" - libp2pNodeIgnored = "IGNORED" libp2pMultiaddrsListStart = "MULTIADDRS_LIST_START" libp2pMultiaddrsListEnd = "MULTIADDRS_LIST_END" ) @@ -52,12 +51,6 @@ func check(err error) { } } -func ignore(err error) { - if err != nil { - fmt.Println(libp2pNodeIgnored, ":", err) - } -} - func main() { var err error @@ -158,7 +151,9 @@ func main() { envelope := envel logger.Info().Msgf("received envelope from agent: %s", envelope) err := node.RouteEnvelope(envelope) - ignore(err) + if err != nil { + logger.Error().Msgf("Route envelope error: %s", err.Error()) + } } }() diff --git a/packages/fetchai/connections/p2p_libp2p/libp2p_node/utils/utils.go b/packages/fetchai/connections/p2p_libp2p/libp2p_node/utils/utils.go index 66f7424893..c5cb0d3719 100644 --- a/packages/fetchai/connections/p2p_libp2p/libp2p_node/utils/utils.go +++ b/packages/fetchai/connections/p2p_libp2p/libp2p_node/utils/utils.go @@ -30,6 +30,7 @@ import ( "errors" "fmt" "io" + "math" "net" "os" "sync" @@ -612,6 +613,14 @@ func FetchAIPublicKeyFromFetchAIPrivateKey(privateKey string) (string, error) { // WriteBytesConn send bytes to `conn` func WriteBytesConn(conn net.Conn, data []byte) error { + if len(data) > math.MaxInt32 { + logger.Error().Msg("data size too large") + return errors.New("data size too large") + } + if len(data) == 0 { + logger.Error().Msg("No data to write") + return nil + } size := uint32(len(data)) buf := make([]byte, 4, 4+size) binary.BigEndian.PutUint32(buf, size) @@ -685,6 +694,15 @@ func ReadBytes(s network.Stream) ([]byte, error) { // WriteBytes to a network stream func WriteBytes(s network.Stream, data []byte) error { + if len(data) > math.MaxInt32 { + logger.Error().Msg("data size too large") + return errors.New("data size too large") + } + if len(data) == 0 { + logger.Error().Msg("No data to write") + return nil + } + wstream := bufio.NewWriter(s) size := uint32(len(data)) @@ -699,9 +717,20 @@ func WriteBytes(s network.Stream, data []byte) error { return err } - //logger.Debug().Msgf("writing %d", len(data)) _, err = wstream.Write(data) - wstream.Flush() + if err != nil { + logger.Error(). + Str("err", err.Error()). + Msg("Error on data write") + return err + } + err = wstream.Flush() + if err != nil { + logger.Error(). + Str("err", err.Error()). + Msg("Error on stream flush") + return err + } return err } @@ -734,7 +763,10 @@ func WriteEnvelope(envel *aea.Envelope, s network.Stream) error { return err } - wstream.Flush() + err = wstream.Flush() + if err != nil { + return err + } return nil } diff --git a/packages/hashes.csv b/packages/hashes.csv index 4648159904..42bfa1f4df 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -41,7 +41,7 @@ fetchai/connections/http_server,QmZqiszQJuG7XA6LFWsMqXYSmViTrUkDfpkHwgYtDMbyXy fetchai/connections/ledger,QmT7ffwPzJ3isCMhN2qoj6NRyqinE2RkpSpUKNRFRXxpes fetchai/connections/local,QmUxLhmeE98S8BcuRDB7W7pRsJzpC3wVJV5ELLxVeEkoKC fetchai/connections/oef,QmaHQhxryQLaBk5TvC4iJFZtFvkPp4CoHxHg1iLnh2PAdm -fetchai/connections/p2p_libp2p,QmPny2KMEwNa1QuvPhW44u8NSE5TfmLHNmoKNe7TefUnzi +fetchai/connections/p2p_libp2p,QmPcqryGVtDArm1cFEf5DiEj9JU8XiKPJZBApgP4qBQ9Zi fetchai/connections/p2p_libp2p_client,QmYq2U83xTTnv1WDy1quvEz47QaV2uw1ZmB6sFHoCL31GC fetchai/connections/p2p_stub,QmToCExj3ZpdxUu3vYSMo4jZRJiMTkXMbyhbh18Hq6Nc4b fetchai/connections/prometheus,QmeRmgrccxRh8UG4AfTMct6D5wSuK9BXYoLwnohut2LJaB diff --git a/scripts/check_imports_and_dependencies.py b/scripts/check_imports_and_dependencies.py index 110b9379f8..db99690919 100755 --- a/scripts/check_imports_and_dependencies.py +++ b/scripts/check_imports_and_dependencies.py @@ -58,14 +58,14 @@ class DependenciesTool: """Tool to work with setup.py dependencies.""" @staticmethod - def get_package_files(package_name: str) -> List[str]: + def get_package_files(package_name: str) -> List[str]: # type: ignore """Get package files list.""" packages_info = list(search_packages_info([package_name])) if len(packages_info) == 0: raise Exception(f"package {package_name} not found") files = packages_info[0]["files"] location = packages_info[0]["location"] - return [Path(location) / i for i in files] + return [Path(location) / i for i in files] # type: ignore @staticmethod def clean_dependency_name(dependecy_specification: str) -> str: @@ -246,12 +246,12 @@ def _find_dependency_for_module( dict ) for section, modules in sections_imports.items(): - for module, pyfile in modules: + for module, pyfile in modules: # type: ignore package = _find_dependency_for_module( - sections_dependencies.get(section, {}), pyfile + sections_dependencies.get(section, {}), pyfile # type: ignore ) - if module not in IGNORE: - sections_imports_packages[section][module] = package + if module not in IGNORE: # type: ignore + sections_imports_packages[section][module] = package # type: ignore all_dependencies_set = set( sum((list(i.keys()) for _, i in sections_dependencies.items()), []) diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py index 49be308e41..d85a4d3b44 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py @@ -17,8 +17,10 @@ # # ------------------------------------------------------------------------------ """This test module contains Negative tests for Libp2p connection.""" +import asyncio import os import shutil +import subprocess # nosec import tempfile from asyncio.futures import Future from unittest.mock import Mock, patch @@ -298,3 +300,31 @@ async def test_max_restarts(): node = Libp2pNode(Mock(), Mock(), "tmp", "tmp", max_restarts=0) with pytest.raises(ValueError, match="Max restarts attempts reached:"): await node.restart() + + +@pytest.mark.asyncio +async def test_node_stopped_callback(): + """Test node stopped callback called.""" + host = "127.0.0.1" + port = "10000" + + with tempfile.TemporaryDirectory() as data_dir: + con = _make_libp2p_connection( + port=port, host=host, data_dir=data_dir, build_directory=data_dir + ) + con.node.logger.error = Mock() + await con.node.start() + subprocess.Popen.terminate(con.node.proc) + await asyncio.sleep(2) + con.node.logger.error.assert_called_once() + await con.node.stop() + + with tempfile.TemporaryDirectory() as data_dir: + con = _make_libp2p_connection( + port=port, host=host, data_dir=data_dir, build_directory=data_dir + ) + con.node.logger.error = Mock() + await con.node.start() + await con.node.stop() + await asyncio.sleep(2) + con.node.logger.error.assert_not_called() From e5b7625809f438f98ad5089d771616526cd87087 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Wed, 5 May 2021 13:47:09 +0100 Subject: [PATCH 16/25] test: extend public dht tests to include staging --- tests/conftest.py | 10 ++ .../test_p2p_libp2p/test_public_dht.py | 162 ++++++++++++++---- 2 files changed, 134 insertions(+), 38 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2f94ad344d..074b00e2fd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -249,6 +249,16 @@ PUBLIC_DHT_P2P_PUBLIC_KEY_2 = ( "03fa7cfae1037cba5218f0f5743802eced8de3247c55ecebaae46c7d3679e3f91d" ) +PUBLIC_STAGING_DHT_P2P_MADDR_1 = "/dns4/acn.fetch-ai.com/tcp/9003/p2p/16Uiu2HAmQo6EHbmwhkMJkyhjz1DCxE8Ahsy5zFZtw97tWCFckLUp" +PUBLIC_STAGING_DHT_P2P_MADDR_2 = "/dns4/acn.fetch-ai.com/tcp/9004/p2p/16Uiu2HAmEvey5siPHzdEb5QcTYCkh16squbeFHYHvRCWP9Jzp4bV" +PUBLIC_STAGING_DHT_DELEGATE_URI_1 = "acn.fetch-ai.com:11003" +PUBLIC_STAGING_DHT_DELEGATE_URI_2 = "acn.fetch-ai.com:11004" +PUBLIC_STAGING_DHT_P2P_PUBLIC_KEY_1 = ( + "03b45f898bde437ace4728b3ba097988306930b1600b7991d384e6d08452e340e1" +) +PUBLIC_STAGING_DHT_P2P_PUBLIC_KEY_2 = ( + "0321bac023b7f7cf655cf5e0f988a4c1cf758f7b530528362c4ba8d563f7b090c4" +) # testnets COSMOS_TESTNET_CONFIG = {"address": COSMOS_DEFAULT_ADDRESS} diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py index cbb2a2c1d1..b097d03796 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_public_dht.py @@ -28,7 +28,7 @@ from aea.helpers.base import CertRequest from aea.mail.base import Envelope from aea.multiplexer import Multiplexer -from aea.test_tools.test_cases import AEATestCaseEmpty +from aea.test_tools.test_cases import AEATestCaseMany from packages.fetchai.connections.p2p_libp2p.connection import ( PUBLIC_ID as P2P_CONNECTION_PUBLIC_ID, @@ -45,6 +45,12 @@ PUBLIC_DHT_P2P_MADDR_2, PUBLIC_DHT_P2P_PUBLIC_KEY_1, PUBLIC_DHT_P2P_PUBLIC_KEY_2, + PUBLIC_STAGING_DHT_DELEGATE_URI_1, + PUBLIC_STAGING_DHT_DELEGATE_URI_2, + PUBLIC_STAGING_DHT_P2P_MADDR_1, + PUBLIC_STAGING_DHT_P2P_MADDR_2, + PUBLIC_STAGING_DHT_P2P_PUBLIC_KEY_1, + PUBLIC_STAGING_DHT_P2P_PUBLIC_KEY_2, _make_libp2p_client_connection, _make_libp2p_connection, libp2p_log_on_failure, @@ -56,10 +62,34 @@ PUBLIC_DHT_MADDRS = [PUBLIC_DHT_P2P_MADDR_1, PUBLIC_DHT_P2P_MADDR_2] PUBLIC_DHT_DELEGATE_URIS = [PUBLIC_DHT_DELEGATE_URI_1, PUBLIC_DHT_DELEGATE_URI_2] PUBLIC_DHT_PUBLIC_KEYS = [PUBLIC_DHT_P2P_PUBLIC_KEY_1, PUBLIC_DHT_P2P_PUBLIC_KEY_2] +PUBLIC_STAGING_DHT_MADDRS = [ + PUBLIC_STAGING_DHT_P2P_MADDR_1, + PUBLIC_STAGING_DHT_P2P_MADDR_2, +] +PUBLIC_STAGING_DHT_DELEGATE_URIS = [ + PUBLIC_STAGING_DHT_DELEGATE_URI_1, + PUBLIC_STAGING_DHT_DELEGATE_URI_2, +] +PUBLIC_STAGING_DHT_PUBLIC_KEYS = [ + PUBLIC_STAGING_DHT_P2P_PUBLIC_KEY_1, + PUBLIC_STAGING_DHT_P2P_PUBLIC_KEY_2, +] AEA_DEFAULT_LAUNCH_TIMEOUT = 20 AEA_LIBP2P_LAUNCH_TIMEOUT = 20 +@pytest.fixture +def maddrs(request): + """Fixture for multi addresses.""" + return request.param + + +@pytest.fixture +def delegate_uris_public_keys(request): + """Fixture for delegate uris and public keys.""" + return request.param + + @pytest.mark.integration @libp2p_log_on_failure_all class TestLibp2pConnectionPublicDHTRelay: @@ -73,9 +103,12 @@ def setup(self): self.log_files = [] - def test_connectivity(self): + @pytest.mark.parametrize( + "maddrs", [PUBLIC_DHT_MADDRS, PUBLIC_STAGING_DHT_MADDRS], indirect=True + ) + def test_connectivity(self, maddrs): """Test connectivity.""" - for i, maddr in enumerate(PUBLIC_DHT_MADDRS): + for i, maddr in enumerate(maddrs): temp_dir = os.path.join(self.t, f"dir_{i}") os.mkdir(temp_dir) connection = _make_libp2p_connection( @@ -97,9 +130,12 @@ def test_connectivity(self): finally: multiplexer.disconnect() - def test_communication_direct(self): + @pytest.mark.parametrize( + "maddrs", [PUBLIC_DHT_MADDRS, PUBLIC_STAGING_DHT_MADDRS], indirect=True + ) + def test_communication_direct(self, maddrs): """Test communication direct.""" - for i, maddr in enumerate(PUBLIC_DHT_MADDRS): + for i, maddr in enumerate(maddrs): multiplexers = [] try: temp_dir_1 = os.path.join(self.t, f"dir_{i}_1") @@ -161,11 +197,14 @@ def test_communication_direct(self): for mux in multiplexers: mux.disconnect() - def test_communication_indirect(self): + @pytest.mark.parametrize( + "maddrs", [PUBLIC_DHT_MADDRS, PUBLIC_STAGING_DHT_MADDRS], indirect=True + ) + def test_communication_indirect(self, maddrs): """Test communication indirect.""" - assert len(PUBLIC_DHT_MADDRS) > 1, "Test requires at least 2 public dht node" + assert len(maddrs) > 1, "Test requires at least 2 public dht node" - for i in range(len(PUBLIC_DHT_MADDRS)): + for i in range(len(maddrs)): multiplexers = [] try: temp_dir_1 = os.path.join(self.t, f"dir_{i}__") @@ -173,7 +212,7 @@ def test_communication_indirect(self): connection1 = _make_libp2p_connection( port=DEFAULT_PORT + 1, relay=False, - entry_peers=[PUBLIC_DHT_MADDRS[i]], + entry_peers=[maddrs[i]], data_dir=temp_dir_1, ) multiplexer1 = Multiplexer([connection1]) @@ -182,7 +221,7 @@ def test_communication_indirect(self): multiplexers.append(multiplexer1) addr_1 = connection1.node.address - for j in range(len(PUBLIC_DHT_MADDRS)): + for j in range(len(maddrs)): if j == i: continue @@ -191,7 +230,7 @@ def test_communication_indirect(self): connection2 = _make_libp2p_connection( port=DEFAULT_PORT + 2, relay=False, - entry_peers=[PUBLIC_DHT_MADDRS[j]], + entry_peers=[maddrs[j]], data_dir=temp_dir_2, ) multiplexer2 = Multiplexer([connection2]) @@ -252,11 +291,20 @@ def setup(self): self.t = tempfile.mkdtemp() os.chdir(self.t) - def test_connectivity(self): + @pytest.mark.parametrize( + "delegate_uris_public_keys", + [ + (PUBLIC_DHT_DELEGATE_URIS, PUBLIC_DHT_PUBLIC_KEYS), + (PUBLIC_STAGING_DHT_DELEGATE_URIS, PUBLIC_STAGING_DHT_PUBLIC_KEYS), + ], + indirect=True, + ) + def test_connectivity(self, delegate_uris_public_keys): """Test connectivity.""" - for i in range(len(PUBLIC_DHT_DELEGATE_URIS)): - uri = PUBLIC_DHT_DELEGATE_URIS[i] - peer_public_key = PUBLIC_DHT_PUBLIC_KEYS[i] + delegate_uris, public_keys = delegate_uris_public_keys + for i in range(len(delegate_uris)): + uri = delegate_uris[i] + peer_public_key = public_keys[i] temp_dir = os.path.join(self.t, f"dir_{i}") os.mkdir(temp_dir) connection = _make_libp2p_client_connection( @@ -274,11 +322,20 @@ def test_connectivity(self): finally: multiplexer.disconnect() - def test_communication_direct(self): + @pytest.mark.parametrize( + "delegate_uris_public_keys", + [ + (PUBLIC_DHT_DELEGATE_URIS, PUBLIC_DHT_PUBLIC_KEYS), + (PUBLIC_STAGING_DHT_DELEGATE_URIS, PUBLIC_STAGING_DHT_PUBLIC_KEYS), + ], + indirect=True, + ) + def test_communication_direct(self, delegate_uris_public_keys): """Test communication direct (i.e. both clients registered to same peer).""" - for i in range(len(PUBLIC_DHT_DELEGATE_URIS)): - uri = PUBLIC_DHT_DELEGATE_URIS[i] - peer_public_key = PUBLIC_DHT_PUBLIC_KEYS[i] + delegate_uris, public_keys = delegate_uris_public_keys + for i in range(len(delegate_uris)): + uri = delegate_uris[i] + peer_public_key = public_keys[i] multiplexers = [] try: temp_dir_1 = os.path.join(self.t, f"dir_{i}_1") @@ -332,20 +389,27 @@ def test_communication_direct(self): for mux in multiplexers: mux.disconnect() - def test_communication_indirect(self): + @pytest.mark.parametrize( + "delegate_uris_public_keys", + [ + (PUBLIC_DHT_DELEGATE_URIS, PUBLIC_DHT_PUBLIC_KEYS), + (PUBLIC_STAGING_DHT_DELEGATE_URIS, PUBLIC_STAGING_DHT_PUBLIC_KEYS), + ], + indirect=True, + ) + def test_communication_indirect(self, delegate_uris_public_keys): """Test communication indirect (i.e. clients registered to different peers).""" - assert ( - len(PUBLIC_DHT_DELEGATE_URIS) > 1 - ), "Test requires at least 2 public dht node" + delegate_uris, public_keys = delegate_uris_public_keys + assert len(delegate_uris) > 1, "Test requires at least 2 public dht node" - for i in range(len(PUBLIC_DHT_DELEGATE_URIS)): + for i in range(len(delegate_uris)): multiplexers = [] try: temp_dir_1 = os.path.join(self.t, f"dir_{i}__") os.mkdir(temp_dir_1) connection1 = _make_libp2p_client_connection( - peer_public_key=PUBLIC_DHT_PUBLIC_KEYS[i], - uri=PUBLIC_DHT_DELEGATE_URIS[i], + peer_public_key=public_keys[i], + uri=delegate_uris[i], data_dir=temp_dir_1, ) multiplexer1 = Multiplexer([connection1]) @@ -354,15 +418,15 @@ def test_communication_indirect(self): addr_1 = connection1.address - for j in range(len(PUBLIC_DHT_DELEGATE_URIS)): + for j in range(len(delegate_uris)): if j == i: continue temp_dir_2 = os.path.join(self.t, f"dir_{i}_{j}") os.mkdir(temp_dir_2) connection2 = _make_libp2p_client_connection( - peer_public_key=PUBLIC_DHT_PUBLIC_KEYS[j], - uri=PUBLIC_DHT_DELEGATE_URIS[j], + peer_public_key=public_keys[j], + uri=delegate_uris[j], data_dir=temp_dir_2, ) multiplexer2 = Multiplexer([connection2]) @@ -412,13 +476,19 @@ def teardown(self): @pytest.mark.integration -class TestLibp2pConnectionPublicDHTRelayAEACli(AEATestCaseEmpty): +class TestLibp2pConnectionPublicDHTRelayAEACli(AEATestCaseMany): """Test that public DHT's relay service is working properly, using aea cli""" @libp2p_log_on_failure - def test_connectivity(self): + @pytest.mark.parametrize( + "maddrs", [PUBLIC_DHT_MADDRS, PUBLIC_STAGING_DHT_MADDRS], indirect=True + ) + def test_connectivity(self, maddrs): """Test connectivity.""" self.log_files = [] + self.agent_name = "some" + self.create_agents(self.agent_name) + self.set_agent_context(self.agent_name) self.conn_key_file = os.path.join( os.path.abspath(os.getcwd()), "./conn_key.txt" ) @@ -440,7 +510,7 @@ def test_connectivity(self): config_path, { "local_uri": "127.0.0.1:{}".format(DEFAULT_PORT), - "entry_peers": PUBLIC_DHT_MADDRS, + "entry_peers": maddrs, "log_file": log_file, }, ) @@ -463,29 +533,43 @@ def test_connectivity(self): assert self.is_successfully_terminated( process ), "AEA wasn't successfully terminated." + self.unset_agent_context() + self.run_cli_command("delete", self.agent_name) @pytest.mark.integration -class TestLibp2pConnectionPublicDHTDelegateAEACli(AEATestCaseEmpty): +class TestLibp2pConnectionPublicDHTDelegateAEACli(AEATestCaseMany): """Test that public DHT's delegate service is working properly, using aea cli""" - def test_connectivity(self): + @pytest.mark.parametrize( + "delegate_uris_public_keys", + [ + (PUBLIC_DHT_DELEGATE_URIS, PUBLIC_DHT_PUBLIC_KEYS), + (PUBLIC_STAGING_DHT_DELEGATE_URIS, PUBLIC_STAGING_DHT_PUBLIC_KEYS), + ], + indirect=True, + ) + def test_connectivity(self, delegate_uris_public_keys): """Test connectivity.""" + delegate_uris, public_keys = delegate_uris_public_keys + self.agent_name = "some" + self.create_agents(self.agent_name) + self.set_agent_context(self.agent_name) self.generate_private_key() self.add_private_key() self.add_item("connection", str(P2P_CLIENT_CONNECTION_PUBLIC_ID)) config_path = "vendor.fetchai.connections.p2p_libp2p_client.config" self.nested_set_config( config_path, - {"nodes": [{"uri": "{}".format(uri)} for uri in PUBLIC_DHT_DELEGATE_URIS]}, + {"nodes": [{"uri": "{}".format(uri)} for uri in delegate_uris]}, ) conn_path = "vendor.fetchai.connections.p2p_libp2p_client" self.nested_set_config( conn_path + ".config", { "nodes": [ - {"uri": uri, "public_key": PUBLIC_DHT_PUBLIC_KEYS[i]} - for i, uri in enumerate(PUBLIC_DHT_DELEGATE_URIS) + {"uri": uri, "public_key": public_keys[i]} + for i, uri in enumerate(delegate_uris) ] }, ) @@ -503,7 +587,7 @@ def test_connectivity(self): message_format="{public_key}", save_path=f"./cli_test_cert_{public_key}.txt", ) - for public_key in PUBLIC_DHT_PUBLIC_KEYS + for public_key in public_keys ], ) self.run_cli_command("issue-certificates", cwd=self._get_cwd()) @@ -516,3 +600,5 @@ def test_connectivity(self): assert self.is_successfully_terminated( process ), "AEA wasn't successfully terminated." + self.unset_agent_context() + self.run_cli_command("delete", self.agent_name) From 997a182159a8cf67556bdee0580d9d828556b17c Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Wed, 5 May 2021 14:00:14 +0100 Subject: [PATCH 17/25] fix: mypy error --- scripts/check_imports_and_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_imports_and_dependencies.py b/scripts/check_imports_and_dependencies.py index 110b9379f8..3250ee502e 100755 --- a/scripts/check_imports_and_dependencies.py +++ b/scripts/check_imports_and_dependencies.py @@ -65,7 +65,7 @@ def get_package_files(package_name: str) -> List[str]: raise Exception(f"package {package_name} not found") files = packages_info[0]["files"] location = packages_info[0]["location"] - return [Path(location) / i for i in files] + return [str(Path(location) / i) for i in files] @staticmethod def clean_dependency_name(dependecy_specification: str) -> str: From f892e0767a2aa308a1d0713d70755b0b056a5653 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Wed, 5 May 2021 14:08:16 +0100 Subject: [PATCH 18/25] fix: mypy again --- scripts/check_imports_and_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check_imports_and_dependencies.py b/scripts/check_imports_and_dependencies.py index 3250ee502e..e65075974b 100755 --- a/scripts/check_imports_and_dependencies.py +++ b/scripts/check_imports_and_dependencies.py @@ -65,7 +65,7 @@ def get_package_files(package_name: str) -> List[str]: raise Exception(f"package {package_name} not found") files = packages_info[0]["files"] location = packages_info[0]["location"] - return [str(Path(location) / i) for i in files] + return [Path(location) / i for i in files] # type: ignore @staticmethod def clean_dependency_name(dependecy_specification: str) -> str: From 256220eb0d93f7bd92faa7ca08283ee0413f4c13 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 5 May 2021 16:10:10 +0300 Subject: [PATCH 19/25] revert type ignore --- aea/helpers/base.py | 6 +++--- scripts/check_imports_and_dependencies.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/aea/helpers/base.py b/aea/helpers/base.py index 83881bb1d8..d6ba4dcd30 100644 --- a/aea/helpers/base.py +++ b/aea/helpers/base.py @@ -154,7 +154,7 @@ def sigint_crossplatform(process: subprocess.Popen) -> None: # pragma: nocover if os.name == "posix": process.send_signal(signal.SIGINT) # pylint: disable=no-member elif os.name == "nt": - process.send_signal(signal.CTRL_C_EVENT) # type: ignore # pylint: disable=no-member + process.send_signal(signal.CTRL_C_EVENT) # pylint: disable=no-member else: raise ValueError("Other platforms not supported.") @@ -190,7 +190,7 @@ def send_control_c( if platform.system() == "Windows": if process.stdin: # cause ctrl-c event will be handled with stdin process.stdin.close() - os.kill(process.pid, signal.CTRL_C_EVENT) # type: ignore # pylint: disable=no-member + os.kill(process.pid, signal.CTRL_C_EVENT) # pylint: disable=no-member elif kill_group: pgid = os.getpgid(process.pid) os.killpg(pgid, signal.SIGINT) @@ -488,7 +488,7 @@ def find_topological_order(adjacency_list: Dict[T, Set[T]]) -> List[T]: # compute the topological order queue: Deque[T] = deque() order = [] - queue.extendleft(sorted(roots)) # type: ignore + queue.extendleft(sorted(roots)) while len(queue) > 0: current = queue.pop() order.append(current) diff --git a/scripts/check_imports_and_dependencies.py b/scripts/check_imports_and_dependencies.py index db99690919..110b9379f8 100755 --- a/scripts/check_imports_and_dependencies.py +++ b/scripts/check_imports_and_dependencies.py @@ -58,14 +58,14 @@ class DependenciesTool: """Tool to work with setup.py dependencies.""" @staticmethod - def get_package_files(package_name: str) -> List[str]: # type: ignore + def get_package_files(package_name: str) -> List[str]: """Get package files list.""" packages_info = list(search_packages_info([package_name])) if len(packages_info) == 0: raise Exception(f"package {package_name} not found") files = packages_info[0]["files"] location = packages_info[0]["location"] - return [Path(location) / i for i in files] # type: ignore + return [Path(location) / i for i in files] @staticmethod def clean_dependency_name(dependecy_specification: str) -> str: @@ -246,12 +246,12 @@ def _find_dependency_for_module( dict ) for section, modules in sections_imports.items(): - for module, pyfile in modules: # type: ignore + for module, pyfile in modules: package = _find_dependency_for_module( - sections_dependencies.get(section, {}), pyfile # type: ignore + sections_dependencies.get(section, {}), pyfile ) - if module not in IGNORE: # type: ignore - sections_imports_packages[section][module] = package # type: ignore + if module not in IGNORE: + sections_imports_packages[section][module] = package all_dependencies_set = set( sum((list(i.keys()) for _, i in sections_dependencies.items()), []) From 416fc82c984a139257dd6b8d5f43f04df9100ef1 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 5 May 2021 16:20:01 +0300 Subject: [PATCH 20/25] mypy fixes for check_imports script --- scripts/check_imports_and_dependencies.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/check_imports_and_dependencies.py b/scripts/check_imports_and_dependencies.py index 110b9379f8..1be16a7047 100755 --- a/scripts/check_imports_and_dependencies.py +++ b/scripts/check_imports_and_dependencies.py @@ -58,7 +58,7 @@ class DependenciesTool: """Tool to work with setup.py dependencies.""" @staticmethod - def get_package_files(package_name: str) -> List[str]: + def get_package_files(package_name: str) -> List[Path]: """Get package files list.""" packages_info = list(search_packages_info([package_name])) if len(packages_info) == 0: @@ -142,7 +142,7 @@ class CheckTool: """Tool to check imports in sources match dependencies in setup.py.""" @classmethod - def get_section_dependencies_from_setup(cls) -> Dict[str, Dict[str, List[str]]]: + def get_section_dependencies_from_setup(cls) -> Dict[str, Dict[str, List[Path]]]: """Get sections with dependencies with files lists.""" spec = importlib.util.spec_from_file_location( "setup", str(AEA_ROOT_DIR / "setup.py") @@ -167,9 +167,11 @@ def get_section_dependencies_from_setup(cls) -> Dict[str, Dict[str, List[str]]]: @staticmethod def sections_dependencies_add_files( sections_dependencies: Dict[str, List[str]] - ) -> Dict[str, Dict[str, List[str]]]: + ) -> Dict[str, Dict[str, List[Path]]]: """Add packages file lists to dependencies in sections.""" - result: Dict[str, Dict[str, List[str]]] = defaultdict(lambda: defaultdict(list)) + result: Dict[str, Dict[str, List[Path]]] = defaultdict( + lambda: defaultdict(list) + ) for section, deps in sections_dependencies.items(): for dep in deps: dep = DependenciesTool.clean_dependency_name(dep) From 7af3817b964598a8089f92a4ac1812335db703cf Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 5 May 2021 17:05:44 +0300 Subject: [PATCH 21/25] fixes for windowes and older pythons --- .../connections/p2p_libp2p/connection.py | 21 +++++++------------ .../connections/p2p_libp2p/connection.yaml | 2 +- packages/hashes.csv | 2 +- .../test_p2p_libp2p/test_errors.py | 10 +++++++++ 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index 2f7419b5d0..2c572466aa 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -13,15 +13,14 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------------ + """This module contains the p2p libp2p connection.""" import asyncio import logging import os import platform import subprocess # nosec +import sys from asyncio import AbstractEventLoop, CancelledError, events from ipaddress import ip_address from pathlib import Path @@ -375,19 +374,13 @@ async def start(self) -> None: self.source, LIBP2P_NODE_MODULE_NAME, [self.env_file], self._log_file_desc ) - if platform.system() == "Windows": - try: - asyncio.get_event_loop()._proactor.wait_for_handle( # type: ignore # pylint: disable=protected-access - int(self.proc._handle) # type: ignore # pylint: disable=no-member,protected-access - ).add_done_callback( - self._child_watcher_callback - ) - except Exception as e: # pragma: nocover # pylint:disable=broad-except - self.logger.debug(f"failed to set process monitoring on windows: {e}") - else: + if ( + platform.system() != "Windows" + and sys.version_info.major == 3 + and sys.version_info.minor >= 8 + ): with events.get_child_watcher() as watcher: if watcher: - watcher.attach_loop(asyncio.get_event_loop()) watcher.add_child_handler( self.proc.pid, self._child_watcher_callback ) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index ee63a5b13f..a105b56107 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -11,7 +11,7 @@ fingerprint: README.md: QmVRxZcTquf2m7j5ZpvCogKRuGWiHnLxBPx55CpyTxWdey __init__.py: QmYQuLNyQ8WTjgRYAoKAzoJEb7ocKXvM2hTyK4hsGch5D6 check_dependencies.py: QmP14nkQ8senwzdPdrZJLsA6EQ7zaKKEaLGDELhT42gp1P - connection.py: QmZTvLjt1EkGkszXpRVGtVzfGSweFxqyjZcmstQ2GiKRsC + connection.py: QmesMcdZHfZJ9v4H6RmmFJJwcS84e4sVZeTiHyhTAt5oUJ libp2p_node/README.md: QmSNJEQQwh25QSHgQPM4C84xKU6FczRpmJdN7HkCQyDPuC libp2p_node/aea/api.go: QmdFR5Rmkk2FGVDwzgHtjobjAKyLejqk2CAYBCvcF23AG7 libp2p_node/aea/envelope.pb.go: QmRfUNGpCeVJfsW3H1MzCN4pwDWgumfyWufVFp6xvUjjug diff --git a/packages/hashes.csv b/packages/hashes.csv index 42bfa1f4df..9442a0fa1c 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -41,7 +41,7 @@ fetchai/connections/http_server,QmZqiszQJuG7XA6LFWsMqXYSmViTrUkDfpkHwgYtDMbyXy fetchai/connections/ledger,QmT7ffwPzJ3isCMhN2qoj6NRyqinE2RkpSpUKNRFRXxpes fetchai/connections/local,QmUxLhmeE98S8BcuRDB7W7pRsJzpC3wVJV5ELLxVeEkoKC fetchai/connections/oef,QmaHQhxryQLaBk5TvC4iJFZtFvkPp4CoHxHg1iLnh2PAdm -fetchai/connections/p2p_libp2p,QmPcqryGVtDArm1cFEf5DiEj9JU8XiKPJZBApgP4qBQ9Zi +fetchai/connections/p2p_libp2p,QmUvdCP5RqTBmpEwpM84MzACk1homdeZSV3WqbEz9rRCdN fetchai/connections/p2p_libp2p_client,QmYq2U83xTTnv1WDy1quvEz47QaV2uw1ZmB6sFHoCL31GC fetchai/connections/p2p_stub,QmToCExj3ZpdxUu3vYSMo4jZRJiMTkXMbyhbh18Hq6Nc4b fetchai/connections/prometheus,QmeRmgrccxRh8UG4AfTMct6D5wSuK9BXYoLwnohut2LJaB diff --git a/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py b/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py index d85a4d3b44..f6c639015d 100644 --- a/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py +++ b/tests/test_packages/test_connections/test_p2p_libp2p/test_errors.py @@ -19,8 +19,10 @@ """This test module contains Negative tests for Libp2p connection.""" import asyncio import os +import platform import shutil import subprocess # nosec +import sys import tempfile from asyncio.futures import Future from unittest.mock import Mock, patch @@ -305,6 +307,14 @@ async def test_max_restarts(): @pytest.mark.asyncio async def test_node_stopped_callback(): """Test node stopped callback called.""" + if not ( + platform.system() != "Windows" + and sys.version_info.major == 3 + and sys.version_info.minor >= 8 + ): + pytest.skip( + "Not supported on this platform. Unix and python >= 3.8 supported only" + ) host = "127.0.0.1" port = "10000" From d52880e997b036ec916df5223648e815049214a3 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Wed, 5 May 2021 15:20:52 +0100 Subject: [PATCH 22/25] fix: reintroduces accidentally deleted license --- packages/fetchai/connections/p2p_libp2p/connection.py | 3 +++ packages/fetchai/connections/p2p_libp2p/connection.yaml | 2 +- packages/hashes.csv | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index 2c572466aa..2baac21dd7 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -13,6 +13,9 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------------ """This module contains the p2p libp2p connection.""" import asyncio diff --git a/packages/fetchai/connections/p2p_libp2p/connection.yaml b/packages/fetchai/connections/p2p_libp2p/connection.yaml index a105b56107..cc76f70f57 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.yaml +++ b/packages/fetchai/connections/p2p_libp2p/connection.yaml @@ -11,7 +11,7 @@ fingerprint: README.md: QmVRxZcTquf2m7j5ZpvCogKRuGWiHnLxBPx55CpyTxWdey __init__.py: QmYQuLNyQ8WTjgRYAoKAzoJEb7ocKXvM2hTyK4hsGch5D6 check_dependencies.py: QmP14nkQ8senwzdPdrZJLsA6EQ7zaKKEaLGDELhT42gp1P - connection.py: QmesMcdZHfZJ9v4H6RmmFJJwcS84e4sVZeTiHyhTAt5oUJ + connection.py: QmR88oYU4YfnspuU23QWngC5JdvqX1Y3VE6HSFRG2PNDNp libp2p_node/README.md: QmSNJEQQwh25QSHgQPM4C84xKU6FczRpmJdN7HkCQyDPuC libp2p_node/aea/api.go: QmdFR5Rmkk2FGVDwzgHtjobjAKyLejqk2CAYBCvcF23AG7 libp2p_node/aea/envelope.pb.go: QmRfUNGpCeVJfsW3H1MzCN4pwDWgumfyWufVFp6xvUjjug diff --git a/packages/hashes.csv b/packages/hashes.csv index 9442a0fa1c..5e1a0ae025 100644 --- a/packages/hashes.csv +++ b/packages/hashes.csv @@ -41,7 +41,7 @@ fetchai/connections/http_server,QmZqiszQJuG7XA6LFWsMqXYSmViTrUkDfpkHwgYtDMbyXy fetchai/connections/ledger,QmT7ffwPzJ3isCMhN2qoj6NRyqinE2RkpSpUKNRFRXxpes fetchai/connections/local,QmUxLhmeE98S8BcuRDB7W7pRsJzpC3wVJV5ELLxVeEkoKC fetchai/connections/oef,QmaHQhxryQLaBk5TvC4iJFZtFvkPp4CoHxHg1iLnh2PAdm -fetchai/connections/p2p_libp2p,QmUvdCP5RqTBmpEwpM84MzACk1homdeZSV3WqbEz9rRCdN +fetchai/connections/p2p_libp2p,QmQXRHJLwJNa8nVy4YM5J9TAyQRPPe19n5MXxGh65Dn6nc fetchai/connections/p2p_libp2p_client,QmYq2U83xTTnv1WDy1quvEz47QaV2uw1ZmB6sFHoCL31GC fetchai/connections/p2p_stub,QmToCExj3ZpdxUu3vYSMo4jZRJiMTkXMbyhbh18Hq6Nc4b fetchai/connections/prometheus,QmeRmgrccxRh8UG4AfTMct6D5wSuK9BXYoLwnohut2LJaB From f956b80a06a51d0ca16e45cfae68a5215c248132 Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 5 May 2021 17:21:52 +0300 Subject: [PATCH 23/25] fix --- packages/fetchai/connections/p2p_libp2p/connection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/fetchai/connections/p2p_libp2p/connection.py b/packages/fetchai/connections/p2p_libp2p/connection.py index 2c572466aa..6d25b6a4ca 100644 --- a/packages/fetchai/connections/p2p_libp2p/connection.py +++ b/packages/fetchai/connections/p2p_libp2p/connection.py @@ -13,7 +13,8 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and - +# +# ------------------------------------------------------------------------------ """This module contains the p2p libp2p connection.""" import asyncio import logging From 29bc3d86096ab11090994ddc8a68adc457bcffc0 Mon Sep 17 00:00:00 2001 From: David Minarsch Date: Wed, 5 May 2021 17:09:35 +0100 Subject: [PATCH 24/25] feat: bump packages versions --- docs/aggregation-demo.md | 8 +- docs/aries-cloud-agent-demo.md | 14 ++-- docs/car-park-skills.md | 26 +++---- docs/cli-vs-programmatic-aeas.md | 2 +- docs/config.md | 2 +- docs/connect-a-frontend.md | 2 +- docs/erc1155-skills.md | 22 +++--- docs/generic-skills-step-by-step.md | 18 ++--- docs/generic-skills.md | 22 +++--- docs/ml-skills.md | 24 +++--- docs/multi-agent-manager.md | 8 +- docs/oracle-demo.md | 6 +- docs/orm-integration.md | 22 +++--- docs/p2p-connection.md | 20 ++--- docs/simple-oef-usage.md | 2 +- docs/skill-guide.md | 12 +-- docs/tac-skills-contract.md | 42 +++++----- docs/tac-skills.md | 42 +++++----- docs/thermometer-skills.md | 22 +++--- docs/weather-skills.md | 22 +++--- .../agents/aries_alice/aea-config.yaml | 10 +-- .../agents/aries_faber/aea-config.yaml | 10 +-- .../agents/car_data_buyer/aea-config.yaml | 10 +-- .../agents/car_detector/aea-config.yaml | 10 +-- .../agents/coin_price_oracle/aea-config.yaml | 8 +- .../confirmation_aea_aw1/aea-config.yaml | 14 ++-- .../confirmation_aea_aw2/aea-config.yaml | 14 ++-- .../confirmation_aea_aw3/aea-config.yaml | 14 ++-- .../confirmation_aea_aw5/aea-config.yaml | 14 ++-- .../agents/erc1155_client/aea-config.yaml | 14 ++-- .../agents/erc1155_deployer/aea-config.yaml | 14 ++-- .../agents/fipa_dummy_buyer/aea-config.yaml | 8 +- .../agents/generic_buyer/aea-config.yaml | 10 +-- .../agents/generic_seller/aea-config.yaml | 10 +-- .../agents/ml_data_provider/aea-config.yaml | 10 +-- .../agents/ml_model_trainer/aea-config.yaml | 10 +-- .../registration_aea_aw1/aea-config.yaml | 14 ++-- .../agents/simple_aggregator/aea-config.yaml | 8 +- .../agents/simple_buyer_aw2/aea-config.yaml | 14 ++-- .../agents/simple_buyer_aw5/aea-config.yaml | 14 ++-- .../agents/simple_seller_aw2/aea-config.yaml | 14 ++-- .../agents/simple_seller_aw5/aea-config.yaml | 14 ++-- .../aea-config.yaml | 10 +-- .../simple_service_search/aea-config.yaml | 10 +-- .../agents/tac_controller/aea-config.yaml | 12 +-- .../tac_controller_contract/aea-config.yaml | 20 ++--- .../agents/tac_participant/aea-config.yaml | 14 ++-- .../tac_participant_contract/aea-config.yaml | 22 +++--- .../agents/thermometer_aea/aea-config.yaml | 10 +-- .../agents/thermometer_client/aea-config.yaml | 10 +-- .../agents/weather_client/aea-config.yaml | 10 +-- .../agents/weather_station/aea-config.yaml | 10 +-- .../fetchai/connections/p2p_libp2p/README.md | 2 +- .../connections/p2p_libp2p/connection.py | 2 +- .../connections/p2p_libp2p/connection.yaml | 6 +- packages/fetchai/connections/soef/README.md | 2 +- .../fetchai/connections/soef/connection.py | 2 +- .../fetchai/connections/soef/connection.yaml | 6 +- .../fetchai/skills/tac_control/__init__.py | 2 +- .../fetchai/skills/tac_control/skill.yaml | 4 +- .../skills/tac_control_contract/__init__.py | 2 +- .../skills/tac_control_contract/skill.yaml | 6 +- .../skills/tac_negotiation/__init__.py | 2 +- .../fetchai/skills/tac_negotiation/skill.yaml | 6 +- .../skills/tac_participation/__init__.py | 2 +- .../skills/tac_participation/skill.yaml | 4 +- packages/hashes.csv | 76 +++++++++---------- tests/data/dummy_aea/aea-config.yaml | 2 +- tests/data/hashes.csv | 2 +- tests/test_cli/test_upgrade.py | 8 +- .../md_files/bash-aggregation-demo.md | 8 +- .../md_files/bash-aries-cloud-agent-demo.md | 12 +-- .../md_files/bash-car-park-skills.md | 22 +++--- .../md_files/bash-cli-vs-programmatic-aeas.md | 2 +- .../test_bash_yaml/md_files/bash-config.md | 2 +- .../md_files/bash-erc1155-skills.md | 26 +++---- .../bash-generic-skills-step-by-step.md | 18 ++--- .../md_files/bash-generic-skills.md | 22 +++--- .../test_bash_yaml/md_files/bash-ml-skills.md | 22 +++--- .../md_files/bash-oracle-demo.md | 6 +- .../md_files/bash-orm-integration.md | 22 +++--- .../md_files/bash-p2p-connection.md | 14 ++-- .../md_files/bash-skill-guide.md | 10 +-- .../md_files/bash-tac-skills-contract.md | 40 +++++----- .../md_files/bash-tac-skills.md | 44 +++++------ .../md_files/bash-thermometer-skills.md | 22 +++--- .../md_files/bash-weather-skills.md | 22 +++--- .../test_cli_vs_programmatic_aea.py | 2 +- .../test_orm_integration.py | 14 ++-- .../test_skill_guide/test_skill_guide.py | 10 +-- .../test_skills_integration/test_carpark.py | 32 ++++---- .../test_skills_integration/test_erc1155.py | 18 ++--- .../test_skills_integration/test_generic.py | 32 ++++---- .../test_skills_integration/test_ml_skills.py | 32 ++++---- .../test_simple_aggregation.py | 8 +- .../test_simple_oracle.py | 10 +-- .../test_skills_integration/test_tac.py | 50 ++++++------ .../test_thermometer.py | 32 ++++---- .../test_skills_integration/test_weather.py | 34 ++++----- 99 files changed, 724 insertions(+), 724 deletions(-) diff --git a/docs/aggregation-demo.md b/docs/aggregation-demo.md index 0514feb161..2473fad3b1 100644 --- a/docs/aggregation-demo.md +++ b/docs/aggregation-demo.md @@ -19,7 +19,7 @@ Repeat the following process four times in four different terminals (for each {` Fetch the aggregator AEA: ``` bash agent_name="agg$i" -aea fetch fetchai/simple_aggregator:0.1.0 --alias $agent_name +aea fetch fetchai/simple_aggregator:0.2.0 --alias $agent_name cd $agent_name aea install aea build @@ -36,13 +36,13 @@ aea create agent_name cd agent_name aea add connection fetchai/http_client:0.22.0 aea add connection fetchai/http_server:0.21.0 -aea add connection fetchai/p2p_libp2p:0.22.0 -aea add connection fetchai/soef:0.23.0 +aea add connection fetchai/p2p_libp2p:0.23.0 +aea add connection fetchai/soef:0.24.0 aea add connection fetchai/prometheus:0.7.0 aea add skill fetchai/advanced_data_request:0.5.0 aea add skill fetchai/simple_aggregation:0.1.0 -aea config set agent.default_connection fetchai/p2p_libp2p:0.22.0 +aea config set agent.default_connection fetchai/p2p_libp2p:0.23.0 aea install aea build ``` diff --git a/docs/aries-cloud-agent-demo.md b/docs/aries-cloud-agent-demo.md index 69e1a599d2..4ad22936fc 100644 --- a/docs/aries-cloud-agent-demo.md +++ b/docs/aries-cloud-agent-demo.md @@ -180,7 +180,7 @@ Now you can create **Alice_AEA** and **Faber_AEA** in terminals 3 and 4 respecti In the third terminal, fetch **Alice_AEA** and move into its project folder: ``` bash -aea fetch fetchai/aries_alice:0.28.0 +aea fetch fetchai/aries_alice:0.29.0 cd aries_alice ``` @@ -191,8 +191,8 @@ The following steps create Alice_AEA from scratch: ``` bash aea create aries_alice cd aries_alice -aea add connection fetchai/p2p_libp2p:0.22.0 -aea add connection fetchai/soef:0.23.0 +aea add connection fetchai/p2p_libp2p:0.23.0 +aea add connection fetchai/soef:0.24.0 aea add connection fetchai/http_client:0.22.0 aea add connection fetchai/webhook:0.18.0 aea add skill fetchai/aries_alice:0.22.0 @@ -257,14 +257,14 @@ Finally run **Alice_AEA**: aea run ``` -Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.22.0 -u public_uri` to retrieve the address.) We will refer to this as **Alice_AEA's P2P address**. +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.23.0 -u public_uri` to retrieve the address.) We will refer to this as **Alice_AEA's P2P address**. ### Faber_AEA In the fourth terminal, fetch **Faber_AEA** and move into its project folder: ``` bash -aea fetch fetchai/aries_faber:0.28.0 +aea fetch fetchai/aries_faber:0.29.0 cd aries_faber ``` @@ -275,8 +275,8 @@ The following steps create Faber_AEA from scratch: ``` bash aea create aries_faber cd aries_faber -aea add connection fetchai/p2p_libp2p:0.22.0 -aea add connection fetchai/soef:0.23.0 +aea add connection fetchai/p2p_libp2p:0.23.0 +aea add connection fetchai/soef:0.24.0 aea add connection fetchai/http_client:0.22.0 aea add connection fetchai/webhook:0.18.0 aea add skill fetchai/aries_faber:0.20.0 diff --git a/docs/car-park-skills.md b/docs/car-park-skills.md index ed0a9c9bf2..52410c5d79 100644 --- a/docs/car-park-skills.md +++ b/docs/car-park-skills.md @@ -57,9 +57,9 @@ Install the AEA ManagerAgentLand block explorer and request some test tokens via `Get Funds`. @@ -97,7 +97,7 @@ Follow the Preliminaries and =1.0.0"} }' -aea config set agent.default_connection fetchai/p2p_libp2p:0.22.0 +aea config set agent.default_connection fetchai/p2p_libp2p:0.23.0 aea config set --type dict agent.default_routing \ '{ "fetchai/ledger_api:1.0.0": "fetchai/ledger:0.18.0", - "fetchai/oef_search:1.0.0": "fetchai/soef:0.23.0" + "fetchai/oef_search:1.0.0": "fetchai/soef:0.24.0" }' aea install aea build @@ -135,7 +135,7 @@ aea build Then, fetch the car data client AEA: ``` bash -aea fetch fetchai/car_data_buyer:0.29.0 +aea fetch fetchai/car_data_buyer:0.30.0 cd car_data_buyer aea install aea build @@ -148,19 +148,19 @@ The following steps create the car data client from scratch: ``` bash aea create car_data_buyer cd car_data_buyer -aea add connection fetchai/p2p_libp2p:0.22.0 -aea add connection fetchai/soef:0.23.0 +aea add connection fetchai/p2p_libp2p:0.23.0 +aea add connection fetchai/soef:0.24.0 aea add connection fetchai/ledger:0.18.0 aea add skill fetchai/carpark_client:0.25.0 aea config set --type dict agent.dependencies \ '{ "aea-ledger-fetchai": {"version": "<2.0.0,>=1.0.0"} }' -aea config set agent.default_connection fetchai/p2p_libp2p:0.22.0 +aea config set agent.default_connection fetchai/p2p_libp2p:0.23.0 aea config set --type dict agent.default_routing \ '{ "fetchai/ledger_api:1.0.0": "fetchai/ledger:0.18.0", - "fetchai/oef_search:1.0.0": "fetchai/soef:0.23.0" + "fetchai/oef_search:1.0.0": "fetchai/soef:0.24.0" }' aea install aea build @@ -225,7 +225,7 @@ First, run the car data seller AEA: aea run ``` -Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.22.0 -u public_uri` to retrieve the address.) +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.23.0 -u public_uri` to retrieve the address.) This is the entry peer address for the local agent communication network created by the car data seller. Then, in the car data buyer, run this command (replace `SOME_ADDRESS` with the correct value as described above): diff --git a/docs/cli-vs-programmatic-aeas.md b/docs/cli-vs-programmatic-aeas.md index df7c8685cf..cad4efe3b9 100644 --- a/docs/cli-vs-programmatic-aeas.md +++ b/docs/cli-vs-programmatic-aeas.md @@ -33,7 +33,7 @@ If you want to create the weather station AEA step by step you can follow this g Fetch the weather station AEA with the following command : ``` bash -aea fetch fetchai/weather_station:0.28.0 +aea fetch fetchai/weather_station:0.29.0 cd weather_station aea install aea build diff --git a/docs/config.md b/docs/config.md index 0caaca2295..392026562a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -27,7 +27,7 @@ protocols: # The list of protocol public id - fetchai/default:1.0.0 skills: # The list of skill public ids the AEA project depends on (each public id must satisfy PUBLIC_ID_REGEX). - fetchai/error:0.16.0 -default_connection: fetchai/p2p_libp2p:0.22.0 # The default connection used for envelopes sent by the AEA (must satisfy PUBLIC_ID_REGEX). +default_connection: fetchai/p2p_libp2p:0.23.0 # The default connection used for envelopes sent by the AEA (must satisfy PUBLIC_ID_REGEX). default_ledger: fetchai # The default ledger identifier the AEA project uses (must satisfy LEDGER_ID_REGEX) required_ledgers: [fetchai] # the list of identifiers of ledgers that the AEA project requires key pairs for (each item must satisfy LEDGER_ID_REGEX) default_routing: {} # The default routing scheme applied to envelopes sent by the AEA, it maps from protocol public ids to connection public ids (both keys and values must satisfy PUBLIC_ID_REGEX) diff --git a/docs/connect-a-frontend.md b/docs/connect-a-frontend.md index 0ccebe43bd..89880fe01c 100644 --- a/docs/connect-a-frontend.md +++ b/docs/connect-a-frontend.md @@ -6,4 +6,4 @@ This page lays out two options for connecting a front-end to an AEA. The followi The first option is to create a `HTTP Server` connection that handles incoming requests from a REST API. In this scenario, the REST API communicates with the AEA and requests are handled by the `HTTP Server` connection package. The REST API should send CRUD requests to the `HTTP Server` connection (`fetchai/http_server:0.21.0`) which translates these into Envelopes to be consumed by the correct skill. ## Case 2 -The second option is to create a front-end comprising a stand-alone `Multiplexer` with a `P2P` connection (`fetchai/p2p_libp2p:0.22.0`). In this scenario the Agent Communication Network can be used to send Envelopes from the AEA to the front-end. \ No newline at end of file +The second option is to create a front-end comprising a stand-alone `Multiplexer` with a `P2P` connection (`fetchai/p2p_libp2p:0.23.0`). In this scenario the Agent Communication Network can be used to send Envelopes from the AEA to the front-end. \ No newline at end of file diff --git a/docs/erc1155-skills.md b/docs/erc1155-skills.md index b1a4b840bb..aecd50a6ca 100644 --- a/docs/erc1155-skills.md +++ b/docs/erc1155-skills.md @@ -25,7 +25,7 @@ The scope of this guide is demonstrating how you can deploy a smart contract and Fetch the AEA that will deploy the contract: ``` bash -aea fetch fetchai/erc1155_deployer:0.30.0 +aea fetch fetchai/erc1155_deployer:0.31.0 cd erc1155_deployer aea install aea build @@ -39,8 +39,8 @@ Create the AEA that will deploy the contract. ``` bash aea create erc1155_deployer cd erc1155_deployer -aea add connection fetchai/p2p_libp2p:0.22.0 -aea add connection fetchai/soef:0.23.0 +aea add connection fetchai/p2p_libp2p:0.23.0 +aea add connection fetchai/soef:0.24.0 aea add connection fetchai/ledger:0.18.0 aea add skill fetchai/erc1155_deploy:0.28.0 aea config set --type dict agent.dependencies \ @@ -49,12 +49,12 @@ aea config set --type dict agent.dependencies \ "aea-ledger-ethereum": {"version": "<2.0.0,>=1.0.0"}, "aea-ledger-cosmos": {"version": "<2.0.0,>=1.0.0"} }' -aea config set agent.default_connection fetchai/p2p_libp2p:0.22.0 +aea config set agent.default_connection fetchai/p2p_libp2p:0.23.0 aea config set --type dict agent.default_routing \ '{ "fetchai/contract_api:1.0.0": "fetchai/ledger:0.18.0", "fetchai/ledger_api:1.0.0": "fetchai/ledger:0.18.0", - "fetchai/oef_search:1.0.0": "fetchai/soef:0.23.0" + "fetchai/oef_search:1.0.0": "fetchai/soef:0.24.0" }' aea config set --type list vendor.fetchai.connections.p2p_libp2p.cert_requests \ '[{"identifier": "acn", "ledger_id": "ethereum", "not_after": "2022-01-01", "not_before": "2021-01-01", "public_key": "fetchai", "save_path": ".certs/conn_cert.txt"}]' @@ -95,7 +95,7 @@ aea issue-certificates In another terminal, fetch the client AEA which will receive some tokens from the deployer. ``` bash -aea fetch fetchai/erc1155_client:0.30.0 +aea fetch fetchai/erc1155_client:0.31.0 cd erc1155_client aea install aea build @@ -109,8 +109,8 @@ Create the AEA that will get some tokens from the deployer. ``` bash aea create erc1155_client cd erc1155_client -aea add connection fetchai/p2p_libp2p:0.22.0 -aea add connection fetchai/soef:0.23.0 +aea add connection fetchai/p2p_libp2p:0.23.0 +aea add connection fetchai/soef:0.24.0 aea add connection fetchai/ledger:0.18.0 aea add skill fetchai/erc1155_client:0.26.0 aea config set --type dict agent.dependencies \ @@ -119,12 +119,12 @@ aea config set --type dict agent.dependencies \ "aea-ledger-ethereum": {"version": "<2.0.0,>=1.0.0"}, "aea-ledger-cosmos": {"version": "<2.0.0,>=1.0.0"} }' -aea config set agent.default_connection fetchai/p2p_libp2p:0.22.0 +aea config set agent.default_connection fetchai/p2p_libp2p:0.23.0 aea config set --type dict agent.default_routing \ '{ "fetchai/contract_api:1.0.0": "fetchai/ledger:0.18.0", "fetchai/ledger_api:1.0.0": "fetchai/ledger:0.18.0", - "fetchai/oef_search:1.0.0": "fetchai/soef:0.23.0" + "fetchai/oef_search:1.0.0": "fetchai/soef:0.24.0" }' aea config set --type list vendor.fetchai.connections.p2p_libp2p.cert_requests \ '[{"identifier": "acn", "ledger_id": "ethereum", "not_after": "2022-01-01", "not_before": "2021-01-01", "public_key": "fetchai", "save_path": ".certs/conn_cert.txt"}]' @@ -199,7 +199,7 @@ aea run Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of this address. -Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.22.0 -u public_uri` to retrieve the address. The output will be something like `/dns4/127.0.0.1/tcp/9000/p2p/16Uiu2HAm2JPsUX1Su59YVDXJQizYkNSe8JCusqRpLeeTbvY76fE5`. +Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.23.0 -u public_uri` to retrieve the address. The output will be something like `/dns4/127.0.0.1/tcp/9000/p2p/16Uiu2HAm2JPsUX1Su59YVDXJQizYkNSe8JCusqRpLeeTbvY76fE5`. This is the entry peer address for the local agent communication network created by the deployer. diff --git a/docs/generic-skills-step-by-step.md b/docs/generic-skills-step-by-step.md index 6f29619ee2..edea186f98 100644 --- a/docs/generic-skills-step-by-step.md +++ b/docs/generic-skills-step-by-step.md @@ -11,14 +11,14 @@ Follow the Preliminaries and Preliminaries and =1.0.0"} }' -aea config set agent.default_connection fetchai/p2p_libp2p:0.22.0 +aea config set agent.default_connection fetchai/p2p_libp2p:0.23.0 aea config set --type dict agent.default_routing \ '{ "fetchai/ledger_api:1.0.0": "fetchai/ledger:0.18.0", - "fetchai/oef_search:1.0.0": "fetchai/soef:0.23.0" + "fetchai/oef_search:1.0.0": "fetchai/soef:0.24.0" }' aea install aea build @@ -96,7 +96,7 @@ aea build Then, in another terminal fetch the buyer AEA: ``` bash -aea fetch fetchai/generic_buyer:0.26.0 --alias my_buyer_aea +aea fetch fetchai/generic_buyer:0.27.0 --alias my_buyer_aea cd my_buyer_aea aea install aea build @@ -109,19 +109,19 @@ The following steps create the buyer from scratch: ``` bash aea create my_buyer_aea cd my_buyer_aea -aea add connection fetchai/p2p_libp2p:0.22.0 -aea add connection fetchai/soef:0.23.0 +aea add connection fetchai/p2p_libp2p:0.23.0 +aea add connection fetchai/soef:0.24.0 aea add connection fetchai/ledger:0.18.0 aea add skill fetchai/generic_buyer:0.25.0 aea config set --type dict agent.dependencies \ '{ "aea-ledger-fetchai": {"version": "<2.0.0,>=1.0.0"} }' -aea config set agent.default_connection fetchai/p2p_libp2p:0.22.0 +aea config set agent.default_connection fetchai/p2p_libp2p:0.23.0 aea config set --type dict agent.default_routing \ '{ "fetchai/ledger_api:1.0.0": "fetchai/ledger:0.18.0", - "fetchai/oef_search:1.0.0": "fetchai/soef:0.23.0" + "fetchai/oef_search:1.0.0": "fetchai/soef:0.24.0" }' aea install aea build @@ -252,7 +252,7 @@ First, run the seller AEA: aea run ``` -Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of this address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.22.0 -u public_uri` to retrieve the address.) +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of this address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.23.0 -u public_uri` to retrieve the address.) This is the entry peer address for the local agent communication network created by the seller. Then, configure the buyer to connect to this same local ACN by running the following command in the buyer terminal, replacing `SOME_ADDRESS` with the value you noted above: diff --git a/docs/ml-skills.md b/docs/ml-skills.md index f460619035..98aeee6907 100644 --- a/docs/ml-skills.md +++ b/docs/ml-skills.md @@ -62,7 +62,7 @@ Follow the Preliminaries and =1.0.0"} }' -aea config set agent.default_connection fetchai/p2p_libp2p:0.22.0 +aea config set agent.default_connection fetchai/p2p_libp2p:0.23.0 aea config set --type dict agent.default_routing \ '{ "fetchai/ledger_api:1.0.0": "fetchai/ledger:0.18.0", - "fetchai/oef_search:1.0.0": "fetchai/soef:0.23.0" + "fetchai/oef_search:1.0.0": "fetchai/soef:0.24.0" }' aea install aea build @@ -100,7 +100,7 @@ aea build Then, fetch the model trainer AEA: ``` bash -aea fetch fetchai/ml_model_trainer:0.29.0 +aea fetch fetchai/ml_model_trainer:0.30.0 cd ml_model_trainer aea install aea build @@ -113,19 +113,19 @@ The following steps create the model trainer from scratch: ``` bash aea create ml_model_trainer cd ml_model_trainer -aea add connection fetchai/p2p_libp2p:0.22.0 -aea add connection fetchai/soef:0.23.0 +aea add connection fetchai/p2p_libp2p:0.23.0 +aea add connection fetchai/soef:0.24.0 aea add connection fetchai/ledger:0.18.0 aea add skill fetchai/ml_train:0.26.0 aea config set --type dict agent.dependencies \ '{ "aea-ledger-fetchai": {"version": "<2.0.0,>=1.0.0"} }' -aea config set agent.default_connection fetchai/p2p_libp2p:0.22.0 +aea config set agent.default_connection fetchai/p2p_libp2p:0.23.0 aea config set --type dict agent.default_routing \ '{ "fetchai/ledger_api:1.0.0": "fetchai/ledger:0.18.0", - "fetchai/oef_search:1.0.0": "fetchai/soef:0.23.0" + "fetchai/oef_search:1.0.0": "fetchai/soef:0.24.0" }' aea install aea build @@ -190,7 +190,7 @@ First, run the data provider AEA: aea run ``` -Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.22.0 -u public_uri` to retrieve the address.) +Once you see a message of the form `To join its network use multiaddr 'SOME_ADDRESS'` take note of the address. (Alternatively, use `aea get-multiaddress fetchai -c -i fetchai/p2p_libp2p:0.23.0 -u public_uri` to retrieve the address.) This is the entry peer address for the local agent communication network created by the ML data provider.